when: event: [cron, manual] variables: FORCE_REBUILD: description: >- Rebuild every targeted R version even when the artifact is already in the bucket. Needed after a builder-image change that alters the resulting R binary, because artifact filenames carry only the R version and would otherwise dedup to the stale build forever. options: - "false" - "true" default: "false" matrix: include: - PLATFORM: rhel-9 PLATFORM_ID: el9 ARCH: arm64 ARCH_ID: aarch64 - PLATFORM: rhel-9 PLATFORM_ID: el9 ARCH: amd64 ARCH_ID: x86_64 - PLATFORM: rhel-10 PLATFORM_ID: el10 ARCH: arm64 ARCH_ID: aarch64 - PLATFORM: rhel-10 PLATFORM_ID: el10 ARCH: amd64 ARCH_ID: x86_64 - PLATFORM: centos-8 PLATFORM_ID: el8 ARCH: arm64 ARCH_ID: aarch64 - PLATFORM: centos-8 PLATFORM_ID: el8 ARCH: amd64 ARCH_ID: x86_64 - PLATFORM: ubuntu-2204 PLATFORM_ID: 2204 ARCH: arm64 ARCH_ID: arm64 - PLATFORM: ubuntu-2204 PLATFORM_ID: 2204 ARCH: amd64 ARCH_ID: amd64 - PLATFORM: ubuntu-2404 PLATFORM_ID: 2404 ARCH: arm64 ARCH_ID: arm64 - PLATFORM: ubuntu-2404 PLATFORM_ID: 2404 ARCH: amd64 ARCH_ID: amd64 - PLATFORM: ubuntu-2604 PLATFORM_ID: 2604 ARCH: arm64 ARCH_ID: arm64 - PLATFORM: ubuntu-2604 PLATFORM_ID: 2604 ARCH: amd64 ARCH_ID: amd64 # - PLATFORM: alpine-320 # PLATFORM_ID: alpine320 # ARCH: arm64 # ARCH_ID: aarch64 # INSTANCE_TYPE: cax31 - PLATFORM: alpine-324 PLATFORM_ID: alpine324 ARCH: arm64 ARCH_ID: aarch64 - PLATFORM: alpine-324 PLATFORM_ID: alpine324 ARCH: amd64 ARCH_ID: x86_64 - PLATFORM: alpine-323 PLATFORM_ID: alpine323 ARCH: arm64 ARCH_ID: aarch64 - PLATFORM: alpine-323 PLATFORM_ID: alpine323 ARCH: amd64 ARCH_ID: x86_64 - PLATFORM: alpine-322 PLATFORM_ID: alpine322 ARCH: arm64 ARCH_ID: aarch64 - PLATFORM: alpine-322 PLATFORM_ID: alpine322 ARCH: amd64 ARCH_ID: x86_64 steps: - name: Get R versions to build image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl jq && break; sleep 5; done # Latest patch release of each of the last 4 minor R versions (e.g. 4.6.0, 4.5.3, 4.4.3, 4.3.3). - | curl -sf https://cdn.posit.co/r/versions.json | jq -r ' .r_versions | map(select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))) | group_by(split(".")[0:2] | join(".")) | map(max_by(split(".") | map(tonumber))) | sort_by(split(".") | map(tonumber)) | reverse | .[0:4] | .[]' > r-versions-to-build.txt # To pin specific versions for testing, overwrite the file, e.g.: # - printf '4.1.3\n' > r-versions-to-build.txt - cat r-versions-to-build.txt backend_options: kubernetes: resources: requests: memory: 50Mi cpu: 200m limits: memory: 200Mi cpu: 500m nodeSelector: kubernetes.io/arch: "${ARCH}" - name: Check which R versions already exist in s3 image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done - | # Backblaze B2's list-bucket API requires authentication, but the upload # credentials are scoped to the plugin-s3 image and aren't available here. # Individual objects are public-read, so probe each expected artifact URL # with an anonymous HEAD instead of listing the bucket. The filename mirrors # what nfpm produces in builder/package.${PLATFORM} (version/release pinned # to 1): r-_1_.{apk,deb} for alpine/ubuntu, R--1-1..rpm # for el*. Shell variables must use bare $name, not ${name}: crow performs # its own ${VAR} substitution on these commands before the shell runs, so # ${VAR} for a shell var would be blanked out (only matrix vars resolve there). case "${PLATFORM}" in alpine-*) FMT="r-%s_1_%s.apk" ;; ubuntu-*) FMT="r-%s_1_%s.deb" ;; *) FMT="R-%s-1-1.%s.rpm" ;; esac BASE="https://s3.eu-central-003.backblazeb2.com/devxy-r-builds/${PLATFORM_ID}" : > r-versions-existing.txt # FORCE_REBUILD is declared as a top-level variable, so the UI offers it # as a dropdown on a manual run. The two trigger paths deliver it # differently, so accept either: # UI, or crow pipeline create --var FORCE_REBUILD=true # -> a substitution variable, expanded into ${FORCE_REBUILD} below # crow pipeline start -p FORCE_REBUILD=true # -> injected into the step environment, read as bare $FORCE_REBUILD # The declared default is the string "false", so test the value rather # than emptiness; an empty value means neither path supplied one. FORCE_VALUE="${FORCE_REBUILD}" [ -n "$FORCE_VALUE" ] || FORCE_VALUE="$FORCE_REBUILD" case "$FORCE_VALUE" in true|TRUE|True|1|yes|YES) FORCE_ON=1 ;; *) FORCE_ON="" ;; esac if [ -n "$FORCE_ON" ]; then echo "FORCE_REBUILD=$FORCE_VALUE, ignoring artifacts already in the bucket" else for VERSION in $(cat r-versions-to-build.txt); do URL="$BASE/$(printf "$FMT" "$VERSION" "${ARCH_ID}")" if [ "$(curl -s -o /dev/null -w '%{http_code}' -I "$URL")" = "200" ]; then echo "$VERSION" >> r-versions-existing.txt fi done fi - echo "Already present in s3:"; cat r-versions-existing.txt backend_options: kubernetes: nodeSelector: kubernetes.io/arch: "${ARCH}" # ref: https://github.com/woodpecker-ci/woodpecker/discussions/2721 - name: Build image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true environment: DOCKER_HOST: tcp://docker:2375 commands: | ip link set dev eth0 mtu 1280 2>/dev/null || true TO_BUILD="" for VERSION in $(cat r-versions-to-build.txt); do if grep -qxF "$VERSION" r-versions-existing.txt; then echo "R $VERSION already exists for ${PLATFORM} (${ARCH_ID}), skipping" else TO_BUILD="$TO_BUILD $VERSION" fi done if [ -n "$TO_BUILD" ]; then for i in 1 2 3 4 5; do apk add -q --no-cache make just docker-compose && break; sleep 5; done for VERSION in $TO_BUILD; do echo "Building R $VERSION for ${PLATFORM}" just build-r-${PLATFORM} "$VERSION" done fi backend_options: kubernetes: resources: requests: memory: 200Mi cpu: 500m ephemeral-storage: 2Gi limits: memory: 1000Mi cpu: 4000m ephemeral-storage: 4Gi nodeSelector: kubernetes.io/arch: "${ARCH}" - name: Upload to Backblaze S3 image: reg.devxy.io/docker.io/woodpeckerci/plugin-s3:1.5.2 pull: true settings: bucket: devxy-r-builds strip_prefix: /tmp/${PLATFORM}/ source: /tmp/${PLATFORM}/* target: ${PLATFORM_ID}/ region: eu-central-003 acl: public-read endpoint: https://s3.eu-central-003.backblazeb2.com access_key: from_secret: B2_S3_ACCESS_KEY_ID secret_key: from_secret: B2_S3_SECRET_KEY # Must stay true for FORCE_REBUILD to mean anything: with overwrite=false # the plugin logs "File already exists, skipping upload" and still exits # green, so a rebuild would be discarded. Normal runs are unaffected, # since dedup means only not-yet-published versions are ever built. overwrite: true volumes: - ci-tmp-rwx:/tmp # agent on k8s # - /tmp:/tmp # agent on VM backend_options: kubernetes: resources: requests: memory: 100Mi cpu: 200m limits: memory: 1000Mi cpu: 1000m nodeSelector: kubernetes.io/arch: "${ARCH}" services: docker: environment: DOCKER_TLS_CERTDIR: "" image: reg.devxy.io/docker.io/library/docker:29-dind commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - mkdir -p /tmp/${PLATFORM}/${ARCH} - dockerd --tls=false --mtu 1280 --host=tcp://0.0.0.0:2375 privileged: true ports: - 2375 volumes: - ci-tmp-rwx:/tmp # agent on k8s backend_options: kubernetes: nodeSelector: kubernetes.io/arch: "${ARCH}"