test(verify): gate on regressions against the generic slot, not fallback rate

A source-fallback percentage stopped being a readiness signal once
bincraft learned to keep a matching-minor generic binary out of a
fallback's shadow. What survives now is the case where the generic
binary was built under a different minor: unsafe for this client anyway,
so serving source is correct, just slow. Failing on that share would
block slots that are in fact ready - amd64/noble sits at 53% for 4.5 and
4.6 while regressing nobody.

Gate on the thing that decides it instead: packages this client would
receive as source through per-minor routing while the generic slot holds
a binary built under its own minor. That is strictly worse than not
routing, and must be zero. Fallback share is still printed, as context
rather than a verdict.

Measured zero across every reindexed slot and minor.

.crow/build-all-versions.yaml

@@ -35,14 +35,6 @@ variables:
       - 'resolute'
     default: '3.24'
   R_VERSION:
-    # The slot's *primary* R minor: what `local/build-all.R` builds into the
-    # generic slot. The loop below already runs `--sensitive-only` for every
-    # other installed minor, so filling a non-primary minor's gap needs this
-    # left alone, not changed.
-    #
-    # 4.6.0 was briefly offered here (#183) and removed: selecting it for a
-    # slot whose generic binaries are 4.5-built would publish 4.6 binaries
-    # into the generic slot and break every 4.5 client.
     description: 'Primary R version under /opt/R.'
     options:
       - 4.5.3

.crow/reindex.yaml

@@ -1,193 +0,0 @@
-# Re-index every slot without rebuilding anything.
-#
-# `weekly-rebuild-reindex` exists to run after `weekly-rebuild-missing`, so it
-# depends on that workflow and shares its gate: triggering it manually also
-# starts hours of package rebuilds. That is the wrong tool when only the index
-# needs regenerating - after a bincraft release that changes how the index is
-# written, for instance, where the objects in the bucket are already correct
-# and only `PACKAGES*` is stale.
-#
-# This workflow does the index half on its own. It installs the latest bincraft
-# release, republishes the generic and per-R-minor indexes for each slot, and
-# purges the edge. No package is built.
-#
-# Trigger with the `reindex` variable set to `all` or to a single
-# `<os>-<arch>`, e.g.
-#
-#   crow pipeline create devxy/build-cran-binaries --var reindex=all
-
-variables:
-  # A manual pipeline creation instantiates every file in .crow/, so the
-  # default must match no matrix row.
-  reindex:
-    description: "Re-index target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
-    options:
-      - none
-      - all
-      - alpine-322-amd64
-      - alpine-322-arm64
-      - alpine-323-amd64
-      - alpine-323-arm64
-      - alpine-324-amd64
-      - alpine-324-arm64
-      - redhat-8-amd64
-      - redhat-8-arm64
-      - redhat-9-amd64
-      - redhat-9-arm64
-      - redhat-10-amd64
-      - redhat-10-arm64
-      - ubuntu-2204-amd64
-      - ubuntu-2204-arm64
-      - ubuntu-2404-amd64
-      - ubuntu-2404-arm64
-      - ubuntu-2604-amd64
-      - ubuntu-2604-arm64
-    default: none
-
-when:
-  - event: manual
-    evaluate: 'reindex == "all" || reindex == "${OS}-${ARCH}"'
-
-skip_clone: true
-
-labels:
-  group: rpkgs-${ARCH}
-
-matrix:
-  include:
-    - OS: alpine-322
-      ARCH: amd64
-      R_VERSION: 4.5.3
-      IMG: alpine:3.22
-    - OS: alpine-322
-      ARCH: arm64
-      R_VERSION: 4.5.3
-      IMG: alpine:3.22
-    - OS: alpine-323
-      ARCH: amd64
-      R_VERSION: 4.5.3
-      IMG: alpine:3.23
-    - OS: alpine-323
-      ARCH: arm64
-      R_VERSION: 4.5.3
-      IMG: alpine:3.23
-    - OS: alpine-324
-      ARCH: amd64
-      R_VERSION: 4.5.3
-      IMG: alpine:3.24
-    - OS: alpine-324
-      ARCH: arm64
-      R_VERSION: 4.5.3
-      IMG: alpine:3.24
-    - OS: redhat-8
-      ARCH: amd64
-      R_VERSION: 4.4.3
-      IMG: redhat:8
-    - OS: redhat-8
-      ARCH: arm64
-      R_VERSION: 4.4.3
-      IMG: redhat:8
-    - OS: redhat-9
-      ARCH: amd64
-      R_VERSION: 4.4.3
-      IMG: redhat:9
-    - OS: redhat-9
-      ARCH: arm64
-      R_VERSION: 4.4.3
-      IMG: redhat:9
-    - OS: redhat-10
-      ARCH: amd64
-      R_VERSION: 4.5.3
-      IMG: redhat:10
-    - OS: redhat-10
-      ARCH: arm64
-      R_VERSION: 4.5.3
-      IMG: redhat:10
-    - OS: ubuntu-2204
-      ARCH: amd64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:jammy
-    - OS: ubuntu-2204
-      ARCH: arm64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:jammy
-    - OS: ubuntu-2404
-      ARCH: amd64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:noble
-    - OS: ubuntu-2404
-      ARCH: arm64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:noble
-    - OS: ubuntu-2604
-      ARCH: amd64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:resolute
-    - OS: ubuntu-2604
-      ARCH: arm64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:resolute
-
-steps:
-  - name: 'Re-index the slot'
-    image: reg.devxy.io/rpkgs/build-env-${IMG}
-    pull: true
-    environment:
-      OTEL_R_TRACES_EXPORTER: none
-      OTEL_R_LOGS_EXPORTER: none
-      OTEL_R_METRICS_EXPORTER: none
-      RED_HAT_DEV_PW:
-        from_secret: RED_HAT_DEV_PW
-      B2_S3_ACCESS_KEY:
-        from_secret: B2_S3_ACCESS_KEY
-      B2_S3_SECRET_KEY:
-        from_secret: B2_S3_SECRET_KEY
-      REPO_RO_TOKEN:
-        from_secret: REPO_RO_TOKEN
-      GIT_USER: pat-s
-      R_LIBS_USER: /mnt/cache/R-pkgs
-      R_VERSION: ${R_VERSION}
-      PLATFORM: ${OS}
-      ARCH: ${ARCH}
-    commands:
-      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
-      - mkdir -p /mnt/cache/R-pkgs
-      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
-      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
-      # The codename is detected from the image's /etc/os-release.
-      - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
-      - |
-        for RBIN in /opt/R/[0-9]*/bin/R; do
-          RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
-          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
-        done
-
-  - name: Purge CDN cache
-    image: reg.devxy.io/docker.io/library/alpine:3.24
-    environment:
-      OTEL_R_TRACES_EXPORTER: none
-      OTEL_R_LOGS_EXPORTER: none
-      OTEL_R_METRICS_EXPORTER: none
-      BUNNYNET_API_KEY:
-        from_secret: BUNNYNET_API_KEY
-      # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
-      # Bunny pull zones, so both must be purged after the shared origin changes.
-      # The staging zone is listed too. It shares the B2 origin, so an index
-      # it still holds is a stale copy of the same object, and its
-      # cache_expiration_time is the same ~370 days: without a purge here it
-      # serves pre-reindex indexes indefinitely and any verification run
-      # against it measures the past.
-      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
-    commands:
-      - apk add --no-cache -q bash curl jq
-      # Crow carries the checkout from the re-index step into this step.
-      - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
-    # Runs on every row rather than on one designated slot: a cron fires only
-    # its own slot's row, so gating on a named slot would leave every other
-    # slot unpurged. A manual "all" run therefore purges the zone 18 times,
-    # which is a cheap API call and rare.
-    #
-    # Run it even when the re-index above failed: the objects were still
-    # replaced, and a stale edge is exactly what keeps them hidden.
-    when:
-      - status: [success, failure]

.crow/weekly-rebuild-reindex.yaml

@@ -175,12 +175,7 @@ steps:
         from_secret: BUNNYNET_API_KEY
       # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
       # Bunny pull zones, so both must be purged after the shared origin changes.
-      # The staging zone is listed too. It shares the B2 origin, so an index
-      # it still holds is a stale copy of the same object, and its
-      # cache_expiration_time is the same ~370 days: without a purge here it
-      # serves pre-reindex indexes indefinitely and any verification run
-      # against it measures the past.
-      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
+      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io'
     commands:
       - apk add --no-cache -q bash curl jq
       # Crow carries the checkout from the re-index step into this step.

cdn.tf

@@ -89,19 +89,9 @@ resource "bunnynet_compute_script" "rpkgs_router" {
 # slot that is not listed here would hide every package the per-minor index does
 # not carry, so this stays empty until a slot has been backfilled.
 resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" {
-  script = bunnynet_compute_script.rpkgs_router.id
-  name   = "UNION_SLOTS"
-  # Enabled. Until this was set, every client resolved against the generic
-  # index and never reached a per-minor binary: an R 4.6.1 client on resolute
-  # downloaded the 4.5-built rlang (2079570 bytes) while the correct 4.6 build
-  # (2075106 bytes) sat unused one directory away, and died at load with
-  # `undefined symbol: SETLENGTH`.
-  #
-  # Verified before enabling, against the staging zone with the same script and
-  # the same origin: all 16 slots report zero regressions against the generic
-  # slot, an excluded R minor is sent to CRAN, a client without an R minor
-  # still gets the generic index, and tarball requests are never rewritten.
-  default_value = join(",", local.rpkgs_slots)
+  script        = bunnynet_compute_script.rpkgs_router.id
+  name          = "UNION_SLOTS"
+  default_value = ""
   required      = false
 }
 

local/build-all.R

@@ -110,25 +110,10 @@ con <- DBI::dbConnect(
   password = Sys.getenv("PGPASS"),
   sslmode = "require"
 )
-# Scope the skip to the R minor this pass is running under. `single_builds`
-# records `r_version` per attempt, but querying without it made a non-primary
-# pass skip everything the primary pass had already attempted under a different
-# minor - so `--sensitive-only` under 4.6 skipped packages that had only ever
-# been built for 4.5, and the per-minor slots never filled. That is why
-# amd64/resolute served 4000 fewer packages to a 4.6 client than to a 4.5 one.
-r_minor <- paste(
-  R.version$major,
-  strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L],
-  sep = "."
-)
 built <- DBI::dbGetQuery(
   con,
-  paste(
-    "SELECT name, tag FROM single_builds",
-    "WHERE platform = $1 AND arch = $2",
-    "AND substring(r_version from '^[0-9]+[.][0-9]+') = $3"
-  ),
-  params = list(platform, arch, r_minor)
+  "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2",
+  params = list(platform, arch)
 )
 DBI::dbDisconnect(con)
 before <- nrow(chunk)
@@ -136,9 +121,8 @@ chunk <- chunk[
   !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag),
 ]
 sprintf(
-  "Skipped %d package versions already attempted under R %s; %d remaining for this job",
+  "Skipped %d already-attempted package versions; %d remaining for this job",
   before - nrow(chunk),
-  r_minor,
   nrow(chunk)
 )
 

local/packages-to-build.R

@@ -68,36 +68,9 @@ archive_versions <- archive_versions[
 ]
 
 # Now get release versions (assuming cran_release has Package and Version columns)
-#
-# Packages published in the last few days are held back. `check_for_binary()`
-# reads the published version from the `cran` GitHub mirror
-# (`GET /repos/cran/<pkg>/commits`), and that mirror lags CRAN: a package that
-# has just appeared has no repository there yet. The call then 404s, which is
-# permanent, but it is wrapped in `purrr::insistently` and retried ten times
-# with a backoff capped at 60s - so one unmirrored package burns about five
-# minutes and then aborts the whole shard.
-#
-# Holding them back costs nothing: the daily update pipeline builds new and
-# updated packages anyway, and they arrive here on the next run once the mirror
-# has caught up.
-mirror_lag_days <- as.numeric(
-  Sys.getenv("CRAN_MIRROR_LAG_DAYS", unset = "3")
-)
-published <- as.POSIXct(cran_release$Published, tz = "UTC")
-too_recent <- !is.na(published) &
-  published > (Sys.time() - mirror_lag_days * 86400)
-if (any(too_recent)) {
-  message(sprintf(
-    "Holding back %d package(s) published in the last %g day(s); the cran GitHub mirror will not have them yet: %s",
-    sum(too_recent),
-    mirror_lag_days,
-    paste(utils::head(cran_release$Package[too_recent], 10L), collapse = ", ")
-  ))
-}
-
 release_versions <- data.table(
-  Package = cran_release$Package[!too_recent],
-  Version = as.character(cran_release$Version[!too_recent])
+  Package = cran_release$Package,
+  Version = as.character(cran_release$Version)
 )
 
 pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE))

scripts/purge_cdn_zone.sh

@@ -47,31 +47,12 @@ resolve_zone_id() {
   fi
 
   response_file=$(mktemp)
-  local status
-  status=$(
-    curl -sS -o "${response_file}" -w '%{http_code}' \
-      -H "AccessKey: ${api_key}" \
-      "https://api.bunny.net/pullzone?perPage=1000"
-  )
-
-  if [[ "${status}" != "200" ]]; then
-    echo "Listing BunnyCDN pull zones failed with HTTP ${status}:" >&2
-    head -c 500 "${response_file}" >&2
-    echo >&2
-    rm -f "${response_file}"
-    exit 1
-  fi
-
-  # The endpoint answers with a bare array on some accounts and a paginated
-  # object on others. `.Items // .` looks like it covers both but does not:
-  # indexing an array with a string is an *error*, and `//` only substitutes
-  # for null, so the array case aborted with
-  # "Cannot index array with string" and the zone was never purged.
+  curl -sS -o "${response_file}" \
+    -H "AccessKey: ${api_key}" \
+    "https://api.bunny.net/pullzone"
   zone_id=$(
     jq -r --arg hostname "${zone}" \
-      '(if type == "object" then (.Items // []) else . end)[]
-         | select(any(.Hostnames[]?; .Value == $hostname))
-         | .Id' \
+      '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \
       "${response_file}"
   )
   rm -f "${response_file}"
@@ -81,12 +62,6 @@ resolve_zone_id() {
     exit 1
   fi
 
-  # Two zones sharing a hostname would purge only whichever jq emitted first.
-  if [[ $(wc -l <<<"${zone_id}") -gt 1 ]]; then
-    echo "Hostname ${zone} matched multiple pull zones: ${zone_id//$'\n'/ }" >&2
-    exit 1
-  fi
-
   echo "${zone_id}"
 }
 

scripts/verify-r-minor-routing.sh

@@ -43,13 +43,10 @@ MINORS=${MINORS:-"4.4 4.5 4.6"}
 EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"}
 # How many Path: targets to HEAD-check per slot/minor. 0 disables.
 SAMPLE=${SAMPLE:-5}
-# Package-count shortfall against the best minor on the same slot, above which
-# coverage is reported as uneven. Reported, not failed on: the packages a
-# non-primary minor lacks are ABI-risky ones built under the primary minor,
-# which a client on another minor cannot safely load anyway, so their absence
-# is correct. This gates the *claim* ("full coverage for ABI-sensitive
-# packages"), not whether routing is safe to enable - MAX_REGRESSIONS does
-# that.
+# Largest package-count shortfall a non-primary minor may have against the best
+# minor on the same slot before coverage counts as uneven. A slot built under
+# one R minor carries fewer per-minor binaries for the others; until that gap
+# closes, "full coverage for ABI-sensitive packages" is not a claim we can make.
 PARITY_TOLERANCE=${PARITY_TOLERANCE:-25}
 # Source fallbacks are reported, not failed on. Since bincraft learned to keep
 # a matching-minor generic binary out of a fallback's shadow, a remaining
@@ -104,15 +101,10 @@ fetch() {
     return 0
   fi
   local status
-  # -L: the router answers an index request with a redirect, so the bytes a
-  # client ends up with are only visible by following it.
-  #
-  # no-cache: a purge is asynchronous, so a run started right after a reindex
-  # otherwise measures whatever the edge still holds.
   if [ -n "$ua" ]; then
-    status=$(curl -sSL -A "$ua" -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
+    status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
   else
-    status=$(curl -sSL -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
+    status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
   fi
   echo "$status" > "$dest.status"
   echo "$status"
@@ -140,23 +132,6 @@ fallback_counts() {
   '
 }
 
-# Packages this index serves from the generic slot with a binary built under a
-# different R minor, while some other per-minor slot carries a build of them -
-# which proves the ABI classifier called them risky. Serving those is the
-# load-time crash the per-minor slots exist to prevent. bincraft drops them at
-# index time, so a non-zero count means the slot has not been reindexed since
-# that guard shipped.
-abi_unsafe_count() {
-  local minor_file=$1 minor=$2 risky_file=$3
-  gunzip -c "$minor_file" 2>/dev/null | awk -v m="$minor" '
-    /^Package:/ { pkg = $2; path = ""; built = "" }
-    /^Path:/    { path = $2 }
-    /^Built:/   { built = $2 " " $3 }
-    /^$/        { if (pkg != "" && path == "" && built != "" && built !~ ("^R " m "\\.")) print pkg; pkg = "" }
-  ' | sort -u > "$minor_file.mismatched"
-  comm -12 "$minor_file.mismatched" "$risky_file" | wc -l
-}
-
 # How many packages a client of <minor> would receive as source through
 # per-minor routing while the generic slot holds a binary built under that very
 # minor. Zero is the bar for enabling a slot.
@@ -236,25 +211,12 @@ for arch in $ARCHES; do
       minor_count=$(wc -l < "$minor_file.names")
 
       # Union property: nothing the flat index carries may be missing here.
-      # bincraft deliberately drops an ABI-risky package whose only binary was
-      # built under another R minor: serving it is the load-time crash the
-      # per-minor slots exist to prevent. Those absences are correct.
-      #
-      # What must never go missing is a generic package built under *this*
-      # minor, which is safe to serve and has no reason to disappear.
-      comm -23 "$flat_file.names" "$minor_file.names" > "$minor_file.absent"
-      absent_count=$(wc -l < "$minor_file.absent")
-      gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" '
-        /^Package:/ { pkg = $2; built = "" }
-        /^Built:/   { built = $2 " " $3 }
-        /^$/        { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" }
-      ' | sort -u > "$minor_file.flatsame"
-      lost=$(comm -12 "$minor_file.absent" "$minor_file.flatsame" | wc -l)
-
-      if [ "${lost:-0}" -ne 0 ]; then
-        bad "$slot R $minor index dropped $lost generic package(s) built under R $minor, which were safe to serve"
+      missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5)
+      missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l)
+      if [ "$missing_count" -ne 0 ]; then
+        bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))"
       else
-        ok "$slot R $minor index: $minor_count packages, union holds ($absent_count ABI-unsafe dropped)"
+        ok "$slot R $minor index: $minor_count packages, union holds"
       fi
 
       # A per-minor entry that is a source fallback resolves fine but makes the
@@ -337,37 +299,12 @@ for arch in $ARCHES; do
         [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap"
       done
       if [ -n "$uneven" ]; then
-        printf '        note: %s coverage uneven across minors (vs best %s):%s\n' \
-          "$slot" "$best" "$uneven"
+        bad "$slot coverage uneven across minors (vs best $best):$uneven"
       else
         ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)"
       fi
     fi
 
-    # Packages carrying a Path in any per-minor index are risky by construction.
-    : > "$WORK/${arch}-${distro}.risky"
-    for minor in $MINORS; do
-      f="$WORK/${arch}-${distro}-${minor}.gz"
-      [ -s "$f" ] || continue
-      gunzip -c "$f" 2>/dev/null | awk '
-        /^Package:/ { pkg = $2; path = "" }
-        /^Path:/    { path = $2 }
-        /^$/        { if (pkg != "" && path != "") print pkg; pkg = "" }
-      ' >> "$WORK/${arch}-${distro}.risky"
-    done
-    sort -u -o "$WORK/${arch}-${distro}.risky" "$WORK/${arch}-${distro}.risky"
-
-    for minor in $MINORS; do
-      f="$WORK/${arch}-${distro}-${minor}.gz"
-      [ -s "$f" ] || continue
-      unsafe=$(abi_unsafe_count "$f" "$minor" "$WORK/${arch}-${distro}.risky")
-      if [ "${unsafe:-0}" -gt 0 ]; then
-        bad "$slot R $minor serves $unsafe ABI-risky package(s) built under another R minor - reindex this slot"
-      else
-        ok "$slot R $minor serves no ABI-risky package from another minor"
-      fi
-    done
-
     # Excluded minors: no published index, and under --live a redirect to CRAN.
     for minor in $EXCLUDED_MINORS; do
       ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz"