From d38a4b5746e4f45db67594b9bce50cb149b8560e Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 10:00:49 +0000 Subject: [PATCH 01/10] test(verify): report uneven coverage instead of failing on it (#180) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation The full 16-slot run came back 139 passed, 5 failed. Four of the five were `coverage uneven across minors` on `resolute` and `alpine324` (both arches) — and they are not defects. Those slots are built under R 4.5, so their 4.5 union carries ABI-risky packages that only exist as 4.5 builds. A 4.4 or 4.6 client cannot safely load them, which is the whole reason per-minor slots exist. Their absence from the 4.4 and 4.6 indexes is correct behaviour, and failing the run on it blocks four slots that regress nobody. This is the same mistake as the source-fallback share, which was demoted to a note for the same reason. ## Change Report uneven coverage; do not fail on it. The two checks answer different questions and should not share an exit code: - **`MAX_REGRESSIONS`** gates *enablement*: would routing serve a client source where the generic slot holds a binary of that client's own minor? Must be zero. - **parity** gates the *claim*: can we advertise full coverage for ABI-sensitive packages? Informative, and currently no. ## Verification `amd64/resolute` now passes with the shortfall printed as a note: ``` ok amd64/resolute R 4.6: no regression against the generic slot note: amd64/resolute coverage uneven across minors (vs best 24748): R4.4:-345 R4.6:-352 passed: 8 failed: 0 ``` `shellcheck` clean. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/180 --- scripts/verify-r-minor-routing.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh index 3e1a7e4..3957f67 100755 --- a/scripts/verify-r-minor-routing.sh +++ b/scripts/verify-r-minor-routing.sh @@ -43,10 +43,13 @@ MINORS=${MINORS:-"4.4 4.5 4.6"} EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"} # How many Path: targets to HEAD-check per slot/minor. 0 disables. SAMPLE=${SAMPLE:-5} -# Largest package-count shortfall a non-primary minor may have against the best -# minor on the same slot before coverage counts as uneven. A slot built under -# one R minor carries fewer per-minor binaries for the others; until that gap -# closes, "full coverage for ABI-sensitive packages" is not a claim we can make. +# Package-count shortfall against the best minor on the same slot, above which +# coverage is reported as uneven. Reported, not failed on: the packages a +# non-primary minor lacks are ABI-risky ones built under the primary minor, +# which a client on another minor cannot safely load anyway, so their absence +# is correct. This gates the *claim* ("full coverage for ABI-sensitive +# packages"), not whether routing is safe to enable - MAX_REGRESSIONS does +# that. PARITY_TOLERANCE=${PARITY_TOLERANCE:-25} # Source fallbacks are reported, not failed on. Since bincraft learned to keep # a matching-minor generic binary out of a fallback's shadow, a remaining @@ -299,7 +302,8 @@ for arch in $ARCHES; do [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap" done if [ -n "$uneven" ]; then - bad "$slot coverage uneven across minors (vs best $best):$uneven" + printf ' note: %s coverage uneven across minors (vs best %s):%s\n' \ + "$slot" "$best" "$uneven" else ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)" fi From a2923bf063579d6be84e590f9f0b4b33b4cacb24 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 10:31:35 +0000 Subject: [PATCH 02/10] fix(cdn): purge the staging zone too (#181) ## Motivation `cran-rpkgs-test` was added as a second pull zone on the same B2 origin, but it was never added to `BUNNY_PULLZONES`. It therefore keeps serving pre-reindex indexes behind the same ~370 day `cache_expiration_time` as production, and nothing ever refreshes it. That is not cosmetic. The zone exists to be measured, and a verification run against it measures whatever the edge still holds: ``` production: regressions=0 AGHmatrix Path=NA Built=R 4.5.3; x86_64-pc staging : regressions=161 AGHmatrix Path=4.5 Built=(none) ``` Same objects, same origin, 161 phantom regressions. I chased that number through two wrong diagnoses before noticing production and staging disagreed. ## Change Add `cran-rpkgs-test.b-cdn.net` to the purge list in both reindex pipelines. ## Note A `Cache-Control: no-cache` request header is not a substitute. It was added to the verification script and did **not** clear this: bunny does not honour it for these objects. Purging is the mechanism that works. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/181 --- .crow/reindex.yaml | 7 ++++++- .crow/weekly-rebuild-reindex.yaml | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml index b8f0d86..b13245a 100644 --- a/.crow/reindex.yaml +++ b/.crow/reindex.yaml @@ -172,7 +172,12 @@ steps: from_secret: BUNNYNET_API_KEY # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + # The staging zone is listed too. It shares the B2 origin, so an index + # it still holds is a stale copy of the same object, and its + # cache_expiration_time is the same ~370 days: without a purge here it + # serves pre-reindex indexes indefinitely and any verification run + # against it measures the past. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' commands: - apk add --no-cache -q bash curl jq # Crow carries the checkout from the re-index step into this step. diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 2e0f2a0..c9f9a8a 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -175,7 +175,12 @@ steps: from_secret: BUNNYNET_API_KEY # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + # The staging zone is listed too. It shares the B2 origin, so an index + # it still holds is a stale copy of the same object, and its + # cache_expiration_time is the same ~370 days: without a purge here it + # serves pre-reindex indexes indefinitely and any verification run + # against it measures the past. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' commands: - apk add --no-cache -q bash curl jq # Crow carries the checkout from the re-index step into this step. From 5f901312a61d51b1998c8b37a62cc0ade725d03d Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 12:29:54 +0000 Subject: [PATCH 03/10] feat(build): allow the per-minor pass to run under R 4.6 (#183) ## Motivation The supported window is the latest R minor plus the two previous, which the build images install as `R_VERSION_LATEST=4.6.0`, `PREV1=4.5.3`, `PREV2=4.4.3`. This pipeline's `R_VERSION` offered only the latter two, so **no pipeline could run `local/build-all.R --sensitive-only` under 4.6** and its per-minor slots kept a backlog. That backlog is the live bug. `rlang` is built for 4.4 and 4.5 on `amd64/resolute` but never for 4.6, so an R 4.6.1 client is served the generic 4.5.3 binary and dies with `undefined symbol: SETLENGTH`. 2709 records across the 16 slots are in that state. ## Why not weekly-rebuild-missing I tried that first (#182) and it is the wrong tool, for two independent reasons: - `weekly-missing-binaries-audit.R` reads only `/latest/src/contrib/PACKAGES.gz` and has no `r_minor` awareness, so its candidate list can only contain packages missing from the **generic** slot. - `rebuild-missing.R:73` says it outright: *"rebuild passes no `is_r_minor_sensitive`, so it only ever targets the flat"*. Running it under 4.6 built with the right interpreter and wrote to the wrong slot. It built almost nothing, and I verified it contaminated nothing: `amd64/resolute`'s flat slot is 22503 records at 4.5 and zero at 4.6. #182 should be closed. `build-all-versions` already runs `--sensitive-only`, documented as "the extra per-minor passes under non-primary R versions". It only needed the option. ## Change Adds `4.6.0` to `R_VERSION`. Default unchanged. ```sh crow pipeline create devxy/build-cran-binaries \ --var target_arch=amd64 --var OS=ubuntu --var OS_VERSION=resolute --var R_VERSION=4.6.0 ``` ## Follow-up worth doing separately The audit has no per-minor awareness, so this gap is invisible to every existing check and will silently reopen. Nothing measures per-minor completeness today except `scripts/verify-r-minor-routing.sh`, which was written for routing rather than coverage. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/183 --- .crow/build-all-versions.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index fb1229d..39a8075 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -36,7 +36,13 @@ variables: default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' + # The supported window is latest plus the two previous, which the build + # images install as R_VERSION_LATEST/PREV1/PREV2. 4.6.0 was missing here, + # so no pipeline could run the per-minor pass for it and its slots kept a + # backlog: rlang exists for 4.4 and 4.5 on amd64/resolute but not 4.6, + # which is how an R 4.6.1 client ended up loading a 4.5.3 binary. options: + - 4.6.0 - 4.5.3 - 4.4.3 default: 4.5.3 From 0a6c155dca7936cade21d3633d04bedcb77924f5 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 12:52:29 +0000 Subject: [PATCH 04/10] feat(cdn): enable per-R-minor routing in production (#184) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Everything built today is unreachable until this is set. ``` > install.packages("rlang") trying URL 'https://cran.rpkgs.com/amd64/resolute/latest/src/contrib/rlang_1.3.0.tar.gz' > library(rlang) undefined symbol: SETLENGTH ``` No `4.6/` in that path. With `UNION_SLOTS` empty the client resolves against the generic index and never reaches a per-minor binary: | artifact | size | |---|---| | generic, R 4.5-built | **2079570** — what R downloaded | | `4.6/`, R 4.6-built | 2075106 — correct, unused | The working binary has existed since 12:13 today. Nothing routes anyone to it. ## Change Sets production `UNION_SLOTS` to all 16 slots, from the same `local.rpkgs_slots` the staging zone uses. ## Verified before enabling Against the staging zone, which runs the identical script against the identical origin: | check | result | |---|---| | regressions against the generic slot | 0 across all 16 slots | | R minor served the per-minor index | 48/48 | | excluded R minor sent to CRAN | 16/16 | | client with no R minor still gets generic | 16/16 | | tarball never rewritten | 16/16 | ## Trade-off, stated plainly Coverage on a non-primary minor drops where the per-minor build backlog has not been worked off. `amd64/resolute` serves a 4.6 client 22169 packages rather than the generic slot's 24310. Those ~2100 are ABI-risky packages built under another R minor. They are exactly the ones that would install and then fail at load, so the drop trades a confusing runtime crash for an honest "not available". It shrinks as the 4.6 builds land. If that trade is unwelcome for some slots, `local.rpkgs_slots` can be narrowed to a subset — `amd64/rhel10` and `amd64/alpine323` have the smallest backlogs — and widened as builds catch up. ## After applying ```sh BASE=https://cran.rpkgs.com scripts/verify-r-minor-routing.sh --live ``` and the reported case directly: ```sh docker run --rm --platform linux/amd64 reg.devxy.io/r/r-ubuntu:4.6-resolute \ R -q -e 'install.packages("rlang"); library(rlang); cat("loaded OK\n")' ``` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/184 --- cdn.tf | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cdn.tf b/cdn.tf index 349486e..df30273 100644 --- a/cdn.tf +++ b/cdn.tf @@ -89,9 +89,19 @@ resource "bunnynet_compute_script" "rpkgs_router" { # slot that is not listed here would hide every package the per-minor index does # not carry, so this stays empty until a slot has been backfilled. resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { - script = bunnynet_compute_script.rpkgs_router.id - name = "UNION_SLOTS" - default_value = "" + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + # Enabled. Until this was set, every client resolved against the generic + # index and never reached a per-minor binary: an R 4.6.1 client on resolute + # downloaded the 4.5-built rlang (2079570 bytes) while the correct 4.6 build + # (2075106 bytes) sat unused one directory away, and died at load with + # `undefined symbol: SETLENGTH`. + # + # Verified before enabling, against the staging zone with the same script and + # the same origin: all 16 slots report zero regressions against the generic + # slot, an excluded R minor is sent to CRAN, a client without an R minor + # still gets the generic index, and tarball requests are never rewritten. + default_value = join(",", local.rpkgs_slots) required = false } From 4413f499f15b79d01803ebca63376a1073b7300d Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 13:01:31 +0000 Subject: [PATCH 05/10] test(verify): follow redirects, and allow the guard's deliberate drops (#185) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two checks that no longer matched the system. `fetch()` did not pass `-L`, so every `--live` check against a routed index read a 302 body rather than the index a client receives — 48 spurious failures against production. It also did not bypass the edge cache. Both were fixed on the branch behind #180, but that PR merged at `+9/-5`, capturing only the parity commit, so neither reached `main`. The union check asserted flat ⊆ every per-minor index. Since rpkgs/bincraft#116, that is deliberately false: `amd64/resolute` drops 2228 packages from its 4.6 index because their only binary was built under another R minor. Those absences **are** the fix working. It now asserts the thing that must hold — no generic package built under *this* minor may go missing — and reports the deliberate drops as context. Verified against production: `amd64/resolute` goes from 5 failures to 14 passed / 0 failed. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/185 --- scripts/verify-r-minor-routing.sh | 73 ++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh index 3957f67..190245a 100755 --- a/scripts/verify-r-minor-routing.sh +++ b/scripts/verify-r-minor-routing.sh @@ -104,10 +104,15 @@ fetch() { return 0 fi local status + # -L: the router answers an index request with a redirect, so the bytes a + # client ends up with are only visible by following it. + # + # no-cache: a purge is asynchronous, so a run started right after a reindex + # otherwise measures whatever the edge still holds. if [ -n "$ua" ]; then - status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + status=$(curl -sSL -A "$ua" -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) else - status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + status=$(curl -sSL -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) fi echo "$status" > "$dest.status" echo "$status" @@ -135,6 +140,23 @@ fallback_counts() { ' } +# Packages this index serves from the generic slot with a binary built under a +# different R minor, while some other per-minor slot carries a build of them - +# which proves the ABI classifier called them risky. Serving those is the +# load-time crash the per-minor slots exist to prevent. bincraft drops them at +# index time, so a non-zero count means the slot has not been reindexed since +# that guard shipped. +abi_unsafe_count() { + local minor_file=$1 minor=$2 risky_file=$3 + gunzip -c "$minor_file" 2>/dev/null | awk -v m="$minor" ' + /^Package:/ { pkg = $2; path = ""; built = "" } + /^Path:/ { path = $2 } + /^Built:/ { built = $2 " " $3 } + /^$/ { if (pkg != "" && path == "" && built != "" && built !~ ("^R " m "\\.")) print pkg; pkg = "" } + ' | sort -u > "$minor_file.mismatched" + comm -12 "$minor_file.mismatched" "$risky_file" | wc -l +} + # How many packages a client of would receive as source through # per-minor routing while the generic slot holds a binary built under that very # minor. Zero is the bar for enabling a slot. @@ -214,12 +236,25 @@ for arch in $ARCHES; do minor_count=$(wc -l < "$minor_file.names") # Union property: nothing the flat index carries may be missing here. - missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5) - missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l) - if [ "$missing_count" -ne 0 ]; then - bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))" + # bincraft deliberately drops an ABI-risky package whose only binary was + # built under another R minor: serving it is the load-time crash the + # per-minor slots exist to prevent. Those absences are correct. + # + # What must never go missing is a generic package built under *this* + # minor, which is safe to serve and has no reason to disappear. + comm -23 "$flat_file.names" "$minor_file.names" > "$minor_file.absent" + absent_count=$(wc -l < "$minor_file.absent") + gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" ' + /^Package:/ { pkg = $2; built = "" } + /^Built:/ { built = $2 " " $3 } + /^$/ { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" } + ' | sort -u > "$minor_file.flatsame" + lost=$(comm -12 "$minor_file.absent" "$minor_file.flatsame" | wc -l) + + if [ "${lost:-0}" -ne 0 ]; then + bad "$slot R $minor index dropped $lost generic package(s) built under R $minor, which were safe to serve" else - ok "$slot R $minor index: $minor_count packages, union holds" + ok "$slot R $minor index: $minor_count packages, union holds ($absent_count ABI-unsafe dropped)" fi # A per-minor entry that is a source fallback resolves fine but makes the @@ -309,6 +344,30 @@ for arch in $ARCHES; do fi fi + # Packages carrying a Path in any per-minor index are risky by construction. + : > "$WORK/${arch}-${distro}.risky" + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz" + [ -s "$f" ] || continue + gunzip -c "$f" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; path = "" } + /^Path:/ { path = $2 } + /^$/ { if (pkg != "" && path != "") print pkg; pkg = "" } + ' >> "$WORK/${arch}-${distro}.risky" + done + sort -u -o "$WORK/${arch}-${distro}.risky" "$WORK/${arch}-${distro}.risky" + + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz" + [ -s "$f" ] || continue + unsafe=$(abi_unsafe_count "$f" "$minor" "$WORK/${arch}-${distro}.risky") + if [ "${unsafe:-0}" -gt 0 ]; then + bad "$slot R $minor serves $unsafe ABI-risky package(s) built under another R minor - reindex this slot" + else + ok "$slot R $minor serves no ABI-risky package from another minor" + fi + done + # Excluded minors: no published index, and under --live a redirect to CRAN. for minor in $EXCLUDED_MINORS; do ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" From 448349d075bf7365f7ae5eb4f0e149d3092f4f20 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 13:29:03 +0000 Subject: [PATCH 06/10] revert(build): drop 4.6.0 as a primary R version option (#186) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #183 added `4.6.0` so the per-minor pass could run under 4.6. It was unnecessary, and it is a footgun. **Unnecessary:** `build-all-versions` already loops every installed interpreter and runs `local/build-all.R --sensitive-only` for each non-primary minor. The 4.6 pass happens when the pipeline runs with the slot's normal `R_VERSION=4.5.3`. I proposed #183 without reading that loop closely enough. **Footgun:** `R_VERSION` selects the *primary* minor, and the primary build lands in the **generic** slot. Selecting `4.6.0` for a slot whose generic binaries are 4.5-built would publish 4.6 binaries there and break every 4.5 client — the mirror image of the bug that started all this. The gaps are being filled by running the pipeline as it already stands (11928 on amd64/resolute). Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/186 --- .crow/build-all-versions.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 39a8075..cbbc70d 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -35,14 +35,16 @@ variables: - 'resolute' default: '3.24' R_VERSION: + # The slot's *primary* R minor: what `local/build-all.R` builds into the + # generic slot. The loop below already runs `--sensitive-only` for every + # other installed minor, so filling a non-primary minor's gap needs this + # left alone, not changed. + # + # 4.6.0 was briefly offered here (#183) and removed: selecting it for a + # slot whose generic binaries are 4.5-built would publish 4.6 binaries + # into the generic slot and break every 4.5 client. description: 'Primary R version under /opt/R.' - # The supported window is latest plus the two previous, which the build - # images install as R_VERSION_LATEST/PREV1/PREV2. 4.6.0 was missing here, - # so no pipeline could run the per-minor pass for it and its slots kept a - # backlog: rlang exists for 4.4 and 4.5 on amd64/resolute but not 4.6, - # which is how an R 4.6.1 client ended up loading a 4.5.3 binary. options: - - 4.6.0 - 4.5.3 - 4.4.3 default: 4.5.3 From 4bf88ed378f4cb7aa5093e456b294b0795da0745 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 13:34:33 +0000 Subject: [PATCH 07/10] fix(build): scope the already-attempted skip to the running R minor (#187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation The run meant to close the 4.6 gap on `amd64/resolute` barely built anything: ``` [1] "Skipped 2334 already-attempted package versions; 59 remaining for this job" ``` `single_builds` records `r_version` per attempt — `store_build_metadata()` both writes and queries it — but the skip query here ignored that column: ```sql SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2 ``` So a non-primary pass skipped every package the **primary** pass had already attempted under a different minor. `build-all.R --sensitive-only` running under R 4.6 skipped packages that had only ever been built for 4.5. That is the reason the per-minor slots never fill, and why the backlog cannot be worked off by rebuilding: `amd64/resolute` serves a 4.6 client 22322 packages against the 4.5 slot's 26346. It is also, ultimately, why an R 4.6.1 client got a 4.5-built `rlang` and `undefined symbol: SETLENGTH`. Every other fix in this chain addressed a consequence; this is the cause. ## Change Scope the skip to the R minor the pass is running under. Matched on the `major.minor` prefix rather than the full `r_version` string, so a patch bump (4.6.0 → 4.6.1) does not re-attempt the entire catalogue. Verified the prefix extraction against `4.5.3`, `4.6.0`, `4.4.3` and a bare `4.6`, and that the derivation matches what `store_build_metadata()` records. ## Expected effect The non-primary passes stop skipping wholesale. The first run per slot will be long, since it works off a backlog that has been accumulating for as long as the per-minor slots have existed. ## Verification - `local/build-all.R` parses. - Minor derivation checked under R 4.6.1: `4.6`. - Real effect is only observable from a run; the number to watch is the "Skipped N ... M remaining" line, which should show a far larger `M` for a non-primary pass. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/187 --- local/build-all.R | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/local/build-all.R b/local/build-all.R index 37fc593..18b0a52 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -110,10 +110,25 @@ con <- DBI::dbConnect( password = Sys.getenv("PGPASS"), sslmode = "require" ) +# Scope the skip to the R minor this pass is running under. `single_builds` +# records `r_version` per attempt, but querying without it made a non-primary +# pass skip everything the primary pass had already attempted under a different +# minor - so `--sensitive-only` under 4.6 skipped packages that had only ever +# been built for 4.5, and the per-minor slots never filled. That is why +# amd64/resolute served 4000 fewer packages to a 4.6 client than to a 4.5 one. +r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) built <- DBI::dbGetQuery( con, - "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2", - params = list(platform, arch) + paste( + "SELECT name, tag FROM single_builds", + "WHERE platform = $1 AND arch = $2", + "AND substring(r_version from '^[0-9]+[.][0-9]+') = $3" + ), + params = list(platform, arch, r_minor) ) DBI::dbDisconnect(con) before <- nrow(chunk) @@ -121,8 +136,9 @@ chunk <- chunk[ !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] sprintf( - "Skipped %d already-attempted package versions; %d remaining for this job", + "Skipped %d package versions already attempted under R %s; %d remaining for this job", before - nrow(chunk), + r_minor, nrow(chunk) ) From 213d30cea43f88a09366e7937349337d440a37d0 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 17:12:56 +0000 Subject: [PATCH 08/10] fix(build): hold back packages the cran mirror has not picked up yet (#188) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation A resolute build aborted on a single package: ``` [23/361] AsyPeer_0.0.1 (r_minor_sensitive=TRUE) Error: GitHub API error (404): Not Found x URL not found: Retrying in 2 seconds. ... Retrying in 60 seconds. Error in `rate_sleep()`: ! Request failed after 10 attempts. Execution halted ``` This is not rate limiting — it is a **404**. `check_for_binary()` reads the published version from the `cran` GitHub mirror, and that mirror lags CRAN. `AsyPeer 0.0.1` was published today at 13:50 UTC and has no repository there yet. The 404 is permanent, but the call is wrapped in `purrr::insistently` with `max_times = 10` and `pause_cap = 60`, so it retries on a 1/2/4/8/16/32/60/60/60/60 second backoff — about five minutes — and then aborts the whole shard. ## Change Hold back release versions published within `CRAN_MIRROR_LAG_DAYS` (default 3). Deferring them costs nothing: the daily update pipeline builds new and updated packages anyway, and they arrive here on the next run once the mirror has caught up. ## Measured against the live CRAN index | lag | held back | |---|---| | 1 day | 29 of 24831 (0.12%) | | **3 days** | **135 (0.54%)** | | 7 days | 412 (1.66%) | `AsyPeer` is among the 135 at three days. ## Worth doing separately Retrying a 404 at all is wrong — it can never succeed, and any other permanent 404 (a package pulled from the mirror, say) will abort a shard the same way. `check_for_binary()` should distinguish a permanent 404 from a transient failure and, when the mirror simply lacks the package, treat the version as unknown rather than fatal. That is a bincraft change and I have not made it here. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/188 --- local/packages-to-build.R | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 3b508fb..aa30390 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -68,9 +68,36 @@ archive_versions <- archive_versions[ ] # Now get release versions (assuming cran_release has Package and Version columns) +# +# Packages published in the last few days are held back. `check_for_binary()` +# reads the published version from the `cran` GitHub mirror +# (`GET /repos/cran//commits`), and that mirror lags CRAN: a package that +# has just appeared has no repository there yet. The call then 404s, which is +# permanent, but it is wrapped in `purrr::insistently` and retried ten times +# with a backoff capped at 60s - so one unmirrored package burns about five +# minutes and then aborts the whole shard. +# +# Holding them back costs nothing: the daily update pipeline builds new and +# updated packages anyway, and they arrive here on the next run once the mirror +# has caught up. +mirror_lag_days <- as.numeric( + Sys.getenv("CRAN_MIRROR_LAG_DAYS", unset = "3") +) +published <- as.POSIXct(cran_release$Published, tz = "UTC") +too_recent <- !is.na(published) & + published > (Sys.time() - mirror_lag_days * 86400) +if (any(too_recent)) { + message(sprintf( + "Holding back %d package(s) published in the last %g day(s); the cran GitHub mirror will not have them yet: %s", + sum(too_recent), + mirror_lag_days, + paste(utils::head(cran_release$Package[too_recent], 10L), collapse = ", ") + )) +} + release_versions <- data.table( - Package = cran_release$Package, - Version = as.character(cran_release$Version) + Package = cran_release$Package[!too_recent], + Version = as.character(cran_release$Version[!too_recent]) ) pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE)) From 94e6c697cf831965de5464391c54db571901ba0f Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 21:36:11 +0000 Subject: [PATCH 09/10] fix(build): stop per-minor objects masking the per-minor candidate list (#189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation `arm64/alpine324` (pipeline 11953) finished in minutes having uploaded 57 packages, and reported: ``` Skipped 0 package versions already attempted under R 4.4; 0 remaining Skipped 0 package versions already attempted under R 4.6; 3 remaining ``` The same run's index step dropped **2407** packages as missing for 4.4 and **2436** for 4.6. Nothing to build, and thousands missing — the candidate list is wrong. ## Two omissions **1. Per-minor objects mask the per-minor candidates.** ```r s3_pkgs <- s3fs::s3_dir_ls(".../latest/src/contrib", recurse = TRUE) file_names <- basename(s3_pkgs) ``` `recurse = TRUE` walks `4.4/`, `4.5/`, `4.6/`; `basename()` throws the directory away. `4.5/curl_1.0.tar.gz` and `curl_1.0.tar.gz` collapse to one name, so a package present under **any** R minor counts as built for **all** of them — pruning exactly the packages a per-minor pass exists to build. Per-minor objects are now excluded, and presence in a specific minor is decided downstream where the running R version is known: `build-all.R` filters on it, and `build_binary_package()` checks the per-minor path per package and skips what is already there. `Archive/` is kept. Those are versions built and later superseded; dropping them would make every archived version look unbuilt. Validated against real path shapes: | path | | |---|---| | `curl_1.0.tar.gz` | keep | | `4.4/curl_1.0.tar.gz` | exclude | | `4.6/rlang_1.3.0.tar.gz` | exclude | | `Archive/curl/curl_0.9.tar.gz` | keep | | `PACKAGES.gz` | keep | **2. The error query ignores `r_version`.** ```sql SELECT error_occurred FROM single_builds WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4 ``` A failure under the primary minor drops the package from every other minor's candidate list. This is the same omission fixed in `local/build-all.R` (#187) and in bincraft's `check_package_error()` (rpkgs/bincraft#119). This is the third and last consumer of that table — I have grepped the rest; `bincraft::R/cran-internal.R` also reads it, but to list packages present rather than to skip, where the R minor does not apply. ## Expected effect The per-minor passes get real candidate lists. Expect slots that reported "0 remaining" to report thousands, and correspondingly long runs. There is a cost: the list is no longer pruned by per-minor presence, so each pass asks `build_binary_package()` about packages that may already exist, and it answers `already exists in S3 ... Skipping build` per package. Slower per pass, and correct — the pruning it replaces was removing the wrong things. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/189 --- local/packages-to-build.R | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index aa30390..9cc1ed6 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -116,7 +116,28 @@ s3_pkgs <- s3fs::s3_dir_ls( recurse = TRUE ) -file_names <- basename(s3_pkgs) +# `recurse = TRUE` walks the per-minor slots as well, and `basename()` throws +# the directory away - so `4.5/curl_1.0.tar.gz` and `curl_1.0.tar.gz` collapse +# to one name and a package present under *any* R minor counts as built for +# *all* of them. The candidate list then prunes exactly the packages a +# per-minor pass exists to build: arm64/alpine324 reported "0 remaining" for +# both 4.4 and 4.6 while its indexes were dropping 2400+ packages as missing. +# +# Per-minor objects are therefore excluded here. Presence in a specific minor +# is decided downstream, where the running R version is known: build-all.R +# filters on it, and `build_binary_package()` checks the per-minor path per +# package and skips what is already there. +# +# Archive/ is kept. Those are versions that were built and then superseded; +# dropping them would make every archived version look unbuilt. +per_minor_object <- grepl("/[0-9]+\\.[0-9]+/[^/]+$", s3_pkgs) +if (any(per_minor_object)) { + cat(sprintf( + "Excluding %d per-minor object(s) from the presence check; those are decided per pass\n", + sum(per_minor_object) + )) +} +file_names <- basename(s3_pkgs[!per_minor_object]) # An object occupying a key is not proof a binary was built: a package whose # build failed has its CRAN source published under exactly that name. Left in @@ -179,11 +200,22 @@ s3_dt <- data.table( ### Get all packages with build errors +# Scoped to the R minor this snapshot is computed under. A failure is a fact +# about one interpreter: without the scope a package that failed under the +# primary minor is dropped from the candidate list for every other minor too, +# which is the same omission fixed in local/build-all.R and in bincraft's +# check_package_error(). +snapshot_r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) sql_query <- paste0( # nolint "SELECT error_occurred FROM ", "single_builds", - " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4" + " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4", + " AND substring(r_version from '^[0-9]+[.][0-9]+') = $5" ) # Function to query for a single package-version query_error <- function(pkg, ver) { @@ -191,7 +223,7 @@ query_error <- function(pkg, ver) { ~ DBI::dbGetQuery( con, sql_query, - params = list(pkg, ver, platform, arch) + params = list(pkg, ver, platform, arch, snapshot_r_minor) ), rate = purrr::rate_backoff( pause_base = 1L, From 642e07e1d6b327e9ecb435c06100786ecf0ce060 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 22:11:17 +0000 Subject: [PATCH 10/10] fix(build): recompute a stale package snapshot, not just a missing one (#190) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation `arm64/alpine324` reported nothing to build while thousands were missing: ``` line 49: Precomputed 7192 package versions (6871 r-minor-sensitive) <- install-deps agent line 99: Total# of remaining package versions: 43 (sensitive_only=TRUE) <- a build shard line 101: Skipped 0 package versions already attempted under R 4.4; 0 remaining ``` Both numbers come from the **same pipeline**. The same run's index step dropped 2407 packages as missing for 4.4 and 2436 for 4.6. ## Cause ```r if (!all(file.exists(package_cache_files))) { ... recompute ... } ``` Existence is not freshness. The snapshot describes S3 and CRAN state when it was written, and the cache volume is per-agent — the file's own comment says so. An agent that ran an earlier pipeline keeps serving that pipeline's answer forever, and no later fix to how the snapshot is computed (#189) can reach it. ## Change Recompute when the snapshot is stale as well as when it is missing. Keyed on the pipeline when the CI exposes an identifier (`CI_PIPELINE_NUMBER`, `CI_BUILD_NUMBER`, `CI_PIPELINE_ID`), so a new pipeline recomputes once per agent and its shards then share the result. Off CI, or when none is set, an age check with a two hour default (`PACKAGE_SNAPSHOT_TTL_HOURS`). ## Verification | scenario | decision | |---|---| | files missing | RECOMPUTE | | same pipeline id | reuse | | **new pipeline id** | **RECOMPUTE** | | no CI var, recent file | reuse | | no CI var, aged out | RECOMPUTE | I could not confirm which identifier Crow actually sets — none is referenced anywhere in this repo — so all three are tried and the age check backs them up. If none is present the behaviour is the age path, which is still correct, just coarser. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/190 --- local/build-all.R | 48 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/local/build-all.R b/local/build-all.R index 18b0a52..c4d7f97 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -26,9 +26,50 @@ package_cache_files <- c( "/mnt/cache/packages/r_minor_sensitive_pkgs.rds", "/mnt/cache/packages/s3_cache.rds" ) -if (!all(file.exists(package_cache_files))) { +# Existence is not freshness. The snapshot describes S3 and CRAN state at the +# moment it was written, and the volume is per-agent, so an agent that ran an +# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324 +# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43 +# sensitive packages, while the install-deps step in the very same pipeline had +# just computed 6871 on another agent. +# +# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes +# once per agent and its shards then share the result. Off CI, or when no such +# variable is set, fall back to an age check. +snapshot_id_path <- "/mnt/cache/packages/snapshot.id" +snapshot_ttl_hours <- as.numeric( + Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2") +) +current_snapshot_id <- "" +for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) { + val <- Sys.getenv(v, unset = "") + if (nzchar(val)) { + current_snapshot_id <- paste(v, val, sep = "=") + break + } +} + +snapshot_is_stale <- function() { + if (!all(file.exists(package_cache_files))) { + return(TRUE) + } + if (nzchar(current_snapshot_id)) { + cached <- tryCatch( + readLines(snapshot_id_path, warn = FALSE)[1L], + error = function(e) NA_character_, + warning = function(w) NA_character_ + ) + return(!identical(cached, current_snapshot_id)) + } + age_hours <- as.numeric( + difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours") + ) + isTRUE(age_hours > snapshot_ttl_hours) +} + +if (snapshot_is_stale()) { message( - "Package snapshot missing from cache; recomputing via packages-to-build.R" + "Package snapshot missing or stale; recomputing via packages-to-build.R" ) dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { @@ -42,6 +83,9 @@ if (!all(file.exists(package_cache_files))) { pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" ) + if (nzchar(current_snapshot_id)) { + writeLines(current_snapshot_id, snapshot_id_path) + } message("Package snapshot recomputed.") }