feat(rebuild): allow rebuilding for an explicit R minor

The matrix pins each slot's primary R minor, and the weekly cron only
ever builds that one. Non-primary minors are filled in incidentally, by
process-cran-updates looping the installed interpreters for new and
updated packages, so older packages accumulate a backlog there.

That is why rlang exists for 4.4 and 4.5 on amd64/resolute but not 4.6,
and why an R 4.6.1 client got a 4.5.3 binary and 'undefined symbol:
SETLENGTH'. 2709 records across the 16 slots are in that state.

rebuild_r_version selects the interpreter; rebuild-missing.R derives its
target slot from the R it runs under, so that is the whole mechanism.
'matrix' keeps today's behaviour. An R minor absent from the image fails
the step with the available list rather than silently rebuilding the
wrong slot.

.crow/build-all-versions.yaml

@@ -35,14 +35,6 @@ variables:
       - 'resolute'
     default: '3.24'
   R_VERSION:
-    # The slot's *primary* R minor: what `local/build-all.R` builds into the
-    # generic slot. The loop below already runs `--sensitive-only` for every
-    # other installed minor, so filling a non-primary minor's gap needs this
-    # left alone, not changed.
-    #
-    # 4.6.0 was briefly offered here (#183) and removed: selecting it for a
-    # slot whose generic binaries are 4.5-built would publish 4.6 binaries
-    # into the generic slot and break every 4.5 client.
     description: 'Primary R version under /opt/R.'
     options:
       - 4.5.3

.crow/reindex.yaml

@@ -172,12 +172,7 @@ steps:
         from_secret: BUNNYNET_API_KEY
       # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
       # Bunny pull zones, so both must be purged after the shared origin changes.
-      # The staging zone is listed too. It shares the B2 origin, so an index
+      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io'
-      # it still holds is a stale copy of the same object, and its
-      # cache_expiration_time is the same ~370 days: without a purge here it
-      # serves pre-reindex indexes indefinitely and any verification run
-      # against it measures the past.
-      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
     commands:
       - apk add --no-cache -q bash curl jq
       # Crow carries the checkout from the re-index step into this step.

.crow/weekly-rebuild-missing.yaml

@@ -20,6 +20,19 @@ variables:
   # Gates this pipeline. A manual pipeline creation instantiates every file in
   # .crow/, and a declared default is applied even when the run never passed
   # this variable, so the default must be a value that matches no matrix row.
+  # Which R minor to rebuild for. The matrix pins each slot's primary minor,
+  # which is the only one the weekly cron ever builds - that is why non-primary
+  # minors accumulate a backlog and why an ABI-risky package can exist for 4.4
+  # and 4.5 but not 4.6. Override this to work that backlog off.
+  rebuild_r_version:
+    description: "R version to rebuild with: 'matrix' for each slot's primary, or an explicit x.y.z."
+    options:
+      - matrix
+      - 4.6.0
+      - 4.5.3
+      - 4.4.3
+    default: matrix
+
   weekly_rebuild_missing:
     description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
     options:
@@ -410,6 +423,7 @@ steps:
       UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
       R_LIBS_USER: /mnt/cache/R-pkgs
       R_VERSION: ${R_VERSION}
+      REBUILD_R_VERSION: ${rebuild_r_version}
       CCACHE_DIR: /mnt/cache/ccache
       PLATFORM: ${OS}
       ARCH: ${ARCH}
@@ -424,12 +438,26 @@ steps:
       - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
       - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       - rm -rf /mnt/cache/R-pkgs/00LOCK-*
-      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
+      # `matrix` keeps the slot's primary minor; anything else overrides it.
-      - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
+      # rebuild-missing.R derives its target slot from the R it runs under, so
+      # selecting the interpreter is all that is needed to fill a minor's gap.
+      - |
+        if [ "$REBUILD_R_VERSION" = "matrix" ] || [ -z "$REBUILD_R_VERSION" ]; then
+          RV="$R_VERSION"
+        else
+          RV="$REBUILD_R_VERSION"
+        fi
+        if [ ! -x "/opt/R/$RV/bin/R" ]; then
+          echo "R $RV is not installed in this image; available: $(ls /opt/R)" >&2
+          exit 1
+        fi
+        echo "Rebuilding with R $RV"
+      - /opt/R/$RV/bin/Rscript local/install-bincraft.R
+      - /opt/R/$RV/bin/R -q -e 'packageVersion("bincraft")'
       - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
-      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2
+      - UVR_R_BIN=/opt/R/$RV/bin/R local/uvr-install.sh httr2
-      - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
+      - /opt/R/$RV/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
-      - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1
+      - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$RV/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1
     backend_options:
       docker:
         resources:

.crow/weekly-rebuild-reindex.yaml

@@ -175,12 +175,7 @@ steps:
         from_secret: BUNNYNET_API_KEY
       # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
       # Bunny pull zones, so both must be purged after the shared origin changes.
-      # The staging zone is listed too. It shares the B2 origin, so an index
+      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io'
-      # it still holds is a stale copy of the same object, and its
-      # cache_expiration_time is the same ~370 days: without a purge here it
-      # serves pre-reindex indexes indefinitely and any verification run
-      # against it measures the past.
-      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
     commands:
       - apk add --no-cache -q bash curl jq
       # Crow carries the checkout from the re-index step into this step.

cdn.tf

@@ -91,17 +91,7 @@ resource "bunnynet_compute_script" "rpkgs_router" {
 resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" {
   script        = bunnynet_compute_script.rpkgs_router.id
   name          = "UNION_SLOTS"
-  # Enabled. Until this was set, every client resolved against the generic
+  default_value = ""
-  # index and never reached a per-minor binary: an R 4.6.1 client on resolute
-  # downloaded the 4.5-built rlang (2079570 bytes) while the correct 4.6 build
-  # (2075106 bytes) sat unused one directory away, and died at load with
-  # `undefined symbol: SETLENGTH`.
-  #
-  # Verified before enabling, against the staging zone with the same script and
-  # the same origin: all 16 slots report zero regressions against the generic
-  # slot, an excluded R minor is sent to CRAN, a client without an R minor
-  # still gets the generic index, and tarball requests are never rewritten.
-  default_value = join(",", local.rpkgs_slots)
   required      = false
 }
 

local/build-all.R

@@ -26,50 +26,9 @@ package_cache_files <- c(
   "/mnt/cache/packages/r_minor_sensitive_pkgs.rds",
   "/mnt/cache/packages/s3_cache.rds"
 )
-# Existence is not freshness. The snapshot describes S3 and CRAN state at the
+if (!all(file.exists(package_cache_files))) {
-# moment it was written, and the volume is per-agent, so an agent that ran an
-# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324
-# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43
-# sensitive packages, while the install-deps step in the very same pipeline had
-# just computed 6871 on another agent.
-#
-# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes
-# once per agent and its shards then share the result. Off CI, or when no such
-# variable is set, fall back to an age check.
-snapshot_id_path <- "/mnt/cache/packages/snapshot.id"
-snapshot_ttl_hours <- as.numeric(
-  Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2")
-)
-current_snapshot_id <- ""
-for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) {
-  val <- Sys.getenv(v, unset = "")
-  if (nzchar(val)) {
-    current_snapshot_id <- paste(v, val, sep = "=")
-    break
-  }
-}
-
-snapshot_is_stale <- function() {
-  if (!all(file.exists(package_cache_files))) {
-    return(TRUE)
-  }
-  if (nzchar(current_snapshot_id)) {
-    cached <- tryCatch(
-      readLines(snapshot_id_path, warn = FALSE)[1L],
-      error = function(e) NA_character_,
-      warning = function(w) NA_character_
-    )
-    return(!identical(cached, current_snapshot_id))
-  }
-  age_hours <- as.numeric(
-    difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours")
-  )
-  isTRUE(age_hours > snapshot_ttl_hours)
-}
-
-if (snapshot_is_stale()) {
   message(
-    "Package snapshot missing or stale; recomputing via packages-to-build.R"
+    "Package snapshot missing from cache; recomputing via packages-to-build.R"
   )
   dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
   save_rds_atomic <- function(obj, path) {
@@ -83,9 +42,6 @@ if (snapshot_is_stale()) {
     pkgs[r_minor_sensitive == TRUE],
     "/mnt/cache/packages/r_minor_sensitive_pkgs.rds"
   )
-  if (nzchar(current_snapshot_id)) {
-    writeLines(current_snapshot_id, snapshot_id_path)
-  }
   message("Package snapshot recomputed.")
 }
 
@@ -154,25 +110,10 @@ con <- DBI::dbConnect(
   password = Sys.getenv("PGPASS"),
   sslmode = "require"
 )
-# Scope the skip to the R minor this pass is running under. `single_builds`
-# records `r_version` per attempt, but querying without it made a non-primary
-# pass skip everything the primary pass had already attempted under a different
-# minor - so `--sensitive-only` under 4.6 skipped packages that had only ever
-# been built for 4.5, and the per-minor slots never filled. That is why
-# amd64/resolute served 4000 fewer packages to a 4.6 client than to a 4.5 one.
-r_minor <- paste(
-  R.version$major,
-  strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L],
-  sep = "."
-)
 built <- DBI::dbGetQuery(
   con,
-  paste(
+  "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2",
-    "SELECT name, tag FROM single_builds",
+  params = list(platform, arch)
-    "WHERE platform = $1 AND arch = $2",
-    "AND substring(r_version from '^[0-9]+[.][0-9]+') = $3"
-  ),
-  params = list(platform, arch, r_minor)
 )
 DBI::dbDisconnect(con)
 before <- nrow(chunk)
@@ -180,9 +121,8 @@ chunk <- chunk[
   !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag),
 ]
 sprintf(
-  "Skipped %d package versions already attempted under R %s; %d remaining for this job",
+  "Skipped %d already-attempted package versions; %d remaining for this job",
   before - nrow(chunk),
-  r_minor,
   nrow(chunk)
 )
 

local/packages-to-build.R

@@ -68,36 +68,9 @@ archive_versions <- archive_versions[
 ]
 
 # Now get release versions (assuming cran_release has Package and Version columns)
-#
-# Packages published in the last few days are held back. `check_for_binary()`
-# reads the published version from the `cran` GitHub mirror
-# (`GET /repos/cran/<pkg>/commits`), and that mirror lags CRAN: a package that
-# has just appeared has no repository there yet. The call then 404s, which is
-# permanent, but it is wrapped in `purrr::insistently` and retried ten times
-# with a backoff capped at 60s - so one unmirrored package burns about five
-# minutes and then aborts the whole shard.
-#
-# Holding them back costs nothing: the daily update pipeline builds new and
-# updated packages anyway, and they arrive here on the next run once the mirror
-# has caught up.
-mirror_lag_days <- as.numeric(
-  Sys.getenv("CRAN_MIRROR_LAG_DAYS", unset = "3")
-)
-published <- as.POSIXct(cran_release$Published, tz = "UTC")
-too_recent <- !is.na(published) &
-  published > (Sys.time() - mirror_lag_days * 86400)
-if (any(too_recent)) {
-  message(sprintf(
-    "Holding back %d package(s) published in the last %g day(s); the cran GitHub mirror will not have them yet: %s",
-    sum(too_recent),
-    mirror_lag_days,
-    paste(utils::head(cran_release$Package[too_recent], 10L), collapse = ", ")
-  ))
-}
-
 release_versions <- data.table(
-  Package = cran_release$Package[!too_recent],
+  Package = cran_release$Package,
-  Version = as.character(cran_release$Version[!too_recent])
+  Version = as.character(cran_release$Version)
 )
 
 pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE))
@@ -116,28 +89,7 @@ s3_pkgs <- s3fs::s3_dir_ls(
   recurse = TRUE
 )
 
-# `recurse = TRUE` walks the per-minor slots as well, and `basename()` throws
+file_names <- basename(s3_pkgs)
-# the directory away - so `4.5/curl_1.0.tar.gz` and `curl_1.0.tar.gz` collapse
-# to one name and a package present under *any* R minor counts as built for
-# *all* of them. The candidate list then prunes exactly the packages a
-# per-minor pass exists to build: arm64/alpine324 reported "0 remaining" for
-# both 4.4 and 4.6 while its indexes were dropping 2400+ packages as missing.
-#
-# Per-minor objects are therefore excluded here. Presence in a specific minor
-# is decided downstream, where the running R version is known: build-all.R
-# filters on it, and `build_binary_package()` checks the per-minor path per
-# package and skips what is already there.
-#
-# Archive/ is kept. Those are versions that were built and then superseded;
-# dropping them would make every archived version look unbuilt.
-per_minor_object <- grepl("/[0-9]+\\.[0-9]+/[^/]+$", s3_pkgs)
-if (any(per_minor_object)) {
-  cat(sprintf(
-    "Excluding %d per-minor object(s) from the presence check; those are decided per pass\n",
-    sum(per_minor_object)
-  ))
-}
-file_names <- basename(s3_pkgs[!per_minor_object])
 
 # An object occupying a key is not proof a binary was built: a package whose
 # build failed has its CRAN source published under exactly that name. Left in
@@ -200,22 +152,11 @@ s3_dt <- data.table(
 
 ### Get all packages with build errors
 
-# Scoped to the R minor this snapshot is computed under. A failure is a fact
-# about one interpreter: without the scope a package that failed under the
-# primary minor is dropped from the candidate list for every other minor too,
-# which is the same omission fixed in local/build-all.R and in bincraft's
-# check_package_error().
-snapshot_r_minor <- paste(
-  R.version$major,
-  strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L],
-  sep = "."
-)
 sql_query <- paste0(
   # nolint
   "SELECT error_occurred FROM ",
   "single_builds",
-  " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4",
+  " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4"
-  " AND substring(r_version from '^[0-9]+[.][0-9]+') = $5"
 )
 # Function to query for a single package-version
 query_error <- function(pkg, ver) {
@@ -223,7 +164,7 @@ query_error <- function(pkg, ver) {
     ~ DBI::dbGetQuery(
       con,
       sql_query,
-      params = list(pkg, ver, platform, arch, snapshot_r_minor)
+      params = list(pkg, ver, platform, arch)
     ),
     rate = purrr::rate_backoff(
       pause_base = 1L,

scripts/verify-r-minor-routing.sh

@@ -43,13 +43,10 @@ MINORS=${MINORS:-"4.4 4.5 4.6"}
 EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"}
 # How many Path: targets to HEAD-check per slot/minor. 0 disables.
 SAMPLE=${SAMPLE:-5}
-# Package-count shortfall against the best minor on the same slot, above which
+# Largest package-count shortfall a non-primary minor may have against the best
-# coverage is reported as uneven. Reported, not failed on: the packages a
+# minor on the same slot before coverage counts as uneven. A slot built under
-# non-primary minor lacks are ABI-risky ones built under the primary minor,
+# one R minor carries fewer per-minor binaries for the others; until that gap
-# which a client on another minor cannot safely load anyway, so their absence
+# closes, "full coverage for ABI-sensitive packages" is not a claim we can make.
-# is correct. This gates the *claim* ("full coverage for ABI-sensitive
-# packages"), not whether routing is safe to enable - MAX_REGRESSIONS does
-# that.
 PARITY_TOLERANCE=${PARITY_TOLERANCE:-25}
 # Source fallbacks are reported, not failed on. Since bincraft learned to keep
 # a matching-minor generic binary out of a fallback's shadow, a remaining
@@ -104,15 +101,10 @@ fetch() {
     return 0
   fi
   local status
-  # -L: the router answers an index request with a redirect, so the bytes a
-  # client ends up with are only visible by following it.
-  #
-  # no-cache: a purge is asynchronous, so a run started right after a reindex
-  # otherwise measures whatever the edge still holds.
   if [ -n "$ua" ]; then
-    status=$(curl -sSL -A "$ua" -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
+    status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
   else
-    status=$(curl -sSL -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
+    status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
   fi
   echo "$status" > "$dest.status"
   echo "$status"
@@ -140,23 +132,6 @@ fallback_counts() {
   '
 }
 
-# Packages this index serves from the generic slot with a binary built under a
-# different R minor, while some other per-minor slot carries a build of them -
-# which proves the ABI classifier called them risky. Serving those is the
-# load-time crash the per-minor slots exist to prevent. bincraft drops them at
-# index time, so a non-zero count means the slot has not been reindexed since
-# that guard shipped.
-abi_unsafe_count() {
-  local minor_file=$1 minor=$2 risky_file=$3
-  gunzip -c "$minor_file" 2>/dev/null | awk -v m="$minor" '
-    /^Package:/ { pkg = $2; path = ""; built = "" }
-    /^Path:/    { path = $2 }
-    /^Built:/   { built = $2 " " $3 }
-    /^$/        { if (pkg != "" && path == "" && built != "" && built !~ ("^R " m "\\.")) print pkg; pkg = "" }
-  ' | sort -u > "$minor_file.mismatched"
-  comm -12 "$minor_file.mismatched" "$risky_file" | wc -l
-}
-
 # How many packages a client of <minor> would receive as source through
 # per-minor routing while the generic slot holds a binary built under that very
 # minor. Zero is the bar for enabling a slot.
@@ -236,25 +211,12 @@ for arch in $ARCHES; do
       minor_count=$(wc -l < "$minor_file.names")
 
       # Union property: nothing the flat index carries may be missing here.
-      # bincraft deliberately drops an ABI-risky package whose only binary was
+      missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5)
-      # built under another R minor: serving it is the load-time crash the
+      missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l)
-      # per-minor slots exist to prevent. Those absences are correct.
+      if [ "$missing_count" -ne 0 ]; then
-      #
+        bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))"
-      # What must never go missing is a generic package built under *this*
-      # minor, which is safe to serve and has no reason to disappear.
-      comm -23 "$flat_file.names" "$minor_file.names" > "$minor_file.absent"
-      absent_count=$(wc -l < "$minor_file.absent")
-      gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" '
-        /^Package:/ { pkg = $2; built = "" }
-        /^Built:/   { built = $2 " " $3 }
-        /^$/        { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" }
-      ' | sort -u > "$minor_file.flatsame"
-      lost=$(comm -12 "$minor_file.absent" "$minor_file.flatsame" | wc -l)
-
-      if [ "${lost:-0}" -ne 0 ]; then
-        bad "$slot R $minor index dropped $lost generic package(s) built under R $minor, which were safe to serve"
       else
-        ok "$slot R $minor index: $minor_count packages, union holds ($absent_count ABI-unsafe dropped)"
+        ok "$slot R $minor index: $minor_count packages, union holds"
       fi
 
       # A per-minor entry that is a source fallback resolves fine but makes the
@@ -337,37 +299,12 @@ for arch in $ARCHES; do
         [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap"
       done
       if [ -n "$uneven" ]; then
-        printf '        note: %s coverage uneven across minors (vs best %s):%s\n' \
+        bad "$slot coverage uneven across minors (vs best $best):$uneven"
-          "$slot" "$best" "$uneven"
       else
         ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)"
       fi
     fi
 
-    # Packages carrying a Path in any per-minor index are risky by construction.
-    : > "$WORK/${arch}-${distro}.risky"
-    for minor in $MINORS; do
-      f="$WORK/${arch}-${distro}-${minor}.gz"
-      [ -s "$f" ] || continue
-      gunzip -c "$f" 2>/dev/null | awk '
-        /^Package:/ { pkg = $2; path = "" }
-        /^Path:/    { path = $2 }
-        /^$/        { if (pkg != "" && path != "") print pkg; pkg = "" }
-      ' >> "$WORK/${arch}-${distro}.risky"
-    done
-    sort -u -o "$WORK/${arch}-${distro}.risky" "$WORK/${arch}-${distro}.risky"
-
-    for minor in $MINORS; do
-      f="$WORK/${arch}-${distro}-${minor}.gz"
-      [ -s "$f" ] || continue
-      unsafe=$(abi_unsafe_count "$f" "$minor" "$WORK/${arch}-${distro}.risky")
-      if [ "${unsafe:-0}" -gt 0 ]; then
-        bad "$slot R $minor serves $unsafe ABI-risky package(s) built under another R minor - reindex this slot"
-      else
-        ok "$slot R $minor serves no ABI-risky package from another minor"
-      fi
-    done
-
     # Excluded minors: no published index, and under --live a redirect to CRAN.
     for minor in $EXCLUDED_MINORS; do
       ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz"