diff --git a/.crow/build.yaml b/.crow/build.yaml index b10a930..b970235 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -1,6 +1,18 @@ when: event: [cron, manual] +variables: + FORCE_REBUILD: + description: >- + Rebuild every targeted R version even when the artifact is already in the + bucket. Needed after a builder-image change that alters the resulting R + binary, because artifact filenames carry only the R version and would + otherwise dedup to the stale build forever. + options: + - "false" + - "true" + default: "false" + matrix: include: - PLATFORM: rhel-9 @@ -82,46 +94,6 @@ matrix: ARCH_ID: x86_64 steps: - # Ensures that we're always using the latest available upstream build chain - # - name: Pull upstream - # image: reg.devxy.io/docker.io/library/alpine:3.20 - # commands: - # - apk add -q --no-cache git - # - git remote add upstream https://github.com/cynkra/r-builds.git - # - git config --global user.email "you@example.com" - # - git config --global user.name "Your Name" - # - git config pull.ff only - # - git fetch upstream - # - git merge upstream/main --allow-unrelated-histories - #. - - - name: Net debug - image: reg.devxy.io/docker.io/library/alpine:3.23 - privileged: true - commands: - - ip link set dev eth0 mtu 1280 2>/dev/null || true - - echo "=== pod interface ===" - - cat /sys/class/net/eth0/mtu - - ip -o link show eth0 - - echo "=== route ===" - - ip route - - echo "=== resolv.conf ===" - - cat /etc/resolv.conf - - for i in 1 2 3; do apk add -q --no-cache iputils && break; sleep 5; done || echo "apk failed; falling back to busybox ping (no -M do)" - - | - echo "=== PMTUD probes (DF=1) to dl-cdn.alpinelinux.org ===" - for sz in 1472 1392 1352 1252 1200; do - echo "--- payload=$sz (wire ~$((sz+28))) ---" - ping -c 2 -W 3 -M do -s $sz dl-cdn.alpinelinux.org || echo "(size=$sz failed)" - done - - ping -c 3 -W 5 dl-cdn.alpinelinux.org || true - - wget --tries=1 --timeout=10 -O /dev/null -nv https://dl-cdn.alpinelinux.org/ 2>&1 || true - backend_options: - kubernetes: - nodeSelector: - kubernetes.io/arch: "${ARCH}" - - - name: Get R versions to build image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true @@ -178,22 +150,23 @@ steps: esac BASE="https://s3.eu-central-003.backblazeb2.com/devxy-r-builds/${PLATFORM_ID}" : > r-versions-existing.txt - # Escape hatch for when the builder image itself changes in a way that - # alters the resulting R binary (a newly enabled graphics device, a - # different BLAS). The artifact filename carries only the R version, so - # dedup would otherwise keep the stale binaries forever. Uploads - # overwrite in place. - # - # The two ways to trigger a manual run pass the flag differently: - # crow pipeline create --branch main --var FORCE_REBUILD=1 - # -> a crow substitution variable, read below as ${FORCE_REBUILD}, - # which crow blanks out when it was not supplied - # crow pipeline start -p FORCE_REBUILD=1 + # FORCE_REBUILD is declared as a top-level variable, so the UI offers it + # as a dropdown on a manual run. The two trigger paths deliver it + # differently, so accept either: + # UI, or crow pipeline create --var FORCE_REBUILD=true + # -> a substitution variable, expanded into ${FORCE_REBUILD} below + # crow pipeline start -p FORCE_REBUILD=true # -> injected into the step environment, read as bare $FORCE_REBUILD - # Accept either, so the flag cannot be silently ignored. - FORCE_REBUILD_VAR="${FORCE_REBUILD}" - if [ -n "$FORCE_REBUILD_VAR" ] || [ -n "$FORCE_REBUILD" ]; then - echo "FORCE_REBUILD is set, ignoring artifacts already in the bucket" + # The declared default is the string "false", so test the value rather + # than emptiness; an empty value means neither path supplied one. + FORCE_VALUE="${FORCE_REBUILD}" + [ -n "$FORCE_VALUE" ] || FORCE_VALUE="$FORCE_REBUILD" + case "$FORCE_VALUE" in + true|TRUE|True|1|yes|YES) FORCE_ON=1 ;; + *) FORCE_ON="" ;; + esac + if [ -n "$FORCE_ON" ]; then + echo "FORCE_REBUILD=$FORCE_VALUE, ignoring artifacts already in the bucket" else for VERSION in $(cat r-versions-to-build.txt); do URL="$BASE/$(printf "$FMT" "$VERSION" "${ARCH_ID}")" @@ -217,7 +190,6 @@ steps: DOCKER_HOST: tcp://docker:2375 commands: | ip link set dev eth0 mtu 1280 2>/dev/null || true - # docker info TO_BUILD="" for VERSION in $(cat r-versions-to-build.txt); do if grep -qxF "$VERSION" r-versions-existing.txt; then @@ -263,7 +235,11 @@ steps: from_secret: B2_S3_ACCESS_KEY_ID secret_key: from_secret: B2_S3_SECRET_KEY - overwrite: false + # Must stay true for FORCE_REBUILD to mean anything: with overwrite=false + # the plugin logs "File already exists, skipping upload" and still exits + # green, so a rebuild would be discarded. Normal runs are unaffected, + # since dedup means only not-yet-published versions are ever built. + overwrite: true volumes: - ci-tmp-rwx:/tmp # agent on k8s # - /tmp:/tmp # agent on VM