fix(ci): install RPostgres for audit workflows (#172)

## Summary

- Install RPostgres before running the missing-binaries audit.
- Install RPostgres before running weekly patch proposal and automatic patch workflows.

## Validation

- `prek run --files .crow/auto-apply-patches.yaml .crow/weekly-audit-missing.yaml .crow/weekly-patch-proposals.yaml`
- `crow lint .crow/`
- `git diff --check`

Reviewed-on: #172

.crow/auto-apply-patches.yaml

@@ -58,7 +58,7 @@ steps:
       - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
       - mkdir -p /mnt/cache/R-pkgs
       - rm -rf /mnt/cache/R-pkgs/00LOCK-*
-      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
+      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite
       - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT
     backend_options:
       kubernetes:

.crow/process-updates.yaml

@@ -3,15 +3,15 @@
 # Routing is preserved 1:1:
 #   - cron:   each existing `process-cran-updates-<os>-<arch>` cron fires only
 #             its matching matrix row (via the per-row `cron:` name filter).
-#   - manual: pick a target from the `process_cran_updates` dropdown
+#   - manual: pick a target from the `process_cran_updates` dropdown;
-#             ("all" = every os/arch).
+#             "all" fans out every os/arch as parallel matrix workflows.
 # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64).
 variables:
   # Gates this pipeline. A manual pipeline creation instantiates every file in
   # .crow/, and a declared default is applied even when the run never passed
   # this variable, so the default must be a value that matches no matrix row.
   process_cran_updates:
-    description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
+    description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
     options:
       - none
       - all
@@ -203,6 +203,7 @@ steps:
       - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
       - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
+      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres
       - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
       # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
       - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
@@ -218,6 +219,7 @@ steps:
           LIB="/mnt/cache/R-pkgs-$RMINOR"
           mkdir -p "$LIB"
           R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
+          R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
           R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true
         done
       - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'

.crow/weekly-audit-missing.yaml

@@ -3,15 +3,15 @@
 # Routing is preserved 1:1:
 #   - cron:   each existing `weekly-audit-missing-<os>-<arch>` cron fires only
 #             its matching matrix row (via the per-row `cron:` name filter).
-#   - manual: pick a target from the `weekly_audit_missing` dropdown
+#   - manual: pick a target from the `weekly_audit_missing` dropdown;
-#             ("all" = every os/arch).
+#             "all" fans out every os/arch as parallel matrix workflows.
 # Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64).
 variables:
   # Gates this pipeline. A manual pipeline creation instantiates every file in
   # .crow/, and a declared default is applied even when the run never passed
   # this variable, so the default must be a value that matches no matrix row.
   weekly_audit_missing:
-    description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
+    description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
     options:
       - none
       - all
@@ -147,7 +147,7 @@ steps:
       - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs
       - rm -rf /mnt/cache/R-pkgs/00LOCK-*
       - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
-      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
+      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite
       - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")'
     backend_options:
       docker:

.crow/weekly-patch-proposals.yaml

@@ -53,7 +53,7 @@ steps:
       - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
       - mkdir -p /mnt/cache/R-pkgs
       - rm -rf /mnt/cache/R-pkgs/00LOCK-*
-      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
+      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite
       - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue
       - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue
     backend_options:

.crow/weekly-rebuild-missing.yaml

@@ -1,18 +1,27 @@
 # Consolidated weekly-rebuild-missing pipeline (all platforms, both arches).
-# One matrix row per OS/arch replaces the former per-platform files.
+# Three matrix rows per OS/arch, one per shard of that slot's rebuild list.
 # Routing is preserved 1:1:
 #   - cron:   each existing `weekly-rebuild-missing-<os>-<arch>` cron fires only
-#             its matching matrix row (via the per-row `cron:` name filter).
+#             its matching matrix rows (via the per-row `cron:` name filter),
-#   - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the
+#             which is now all three shards of that slot.
-#             previous bare manual trigger that ran every os/arch); pick a
+#   - manual: pick a target from the `weekly_rebuild_missing` dropdown;
-#             single <os>-<arch> to run just one.
+#             "all" fans out every os/arch and shard as parallel matrix
+#             workflows, while a single <os>-<arch> runs its three shards.
 # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64).
+#
+# The shard picks up its own slice and re-derives what is still outstanding
+# from the bucket, so a restart resumes rather than replaying; see
+# local/rebuild-missing.R.
+#
+# Re-indexing and the CDN purge deliberately do NOT live here. Three shards
+# writing one slot's PACKAGES concurrently would race, so they moved to
+# .crow/weekly-rebuild-reindex.yaml, which depends on this pipeline.
 variables:
   # Gates this pipeline. A manual pipeline creation instantiates every file in
   # .crow/, and a declared default is applied even when the run never passed
   # this variable, so the default must be a value that matches no matrix row.
   weekly_rebuild_missing:
-    description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
+    description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
     options:
       - none
       - all
@@ -53,74 +62,326 @@ matrix:
       ARCH: amd64
       R_VERSION: 4.5.3
       IMG: alpine:3.22
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: alpine-322
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: alpine-322
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: alpine-322
       ARCH: arm64
       R_VERSION: 4.5.3
       IMG: alpine:3.22
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: alpine-322
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: alpine-322
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: alpine-323
       ARCH: amd64
       R_VERSION: 4.5.3
       IMG: alpine:3.23
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: alpine-323
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: alpine-323
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: alpine-323
       ARCH: arm64
       R_VERSION: 4.5.3
       IMG: alpine:3.23
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: alpine-323
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: alpine-323
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: alpine-324
       ARCH: amd64
       R_VERSION: 4.5.3
       IMG: alpine:3.24
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: alpine-324
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: alpine-324
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: alpine-324
       ARCH: arm64
       R_VERSION: 4.5.3
       IMG: alpine:3.24
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: alpine-324
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: alpine-324
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: redhat-8
       ARCH: amd64
       R_VERSION: 4.4.3
       IMG: redhat:8
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: redhat-8
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: redhat-8
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: redhat-8
       ARCH: arm64
       R_VERSION: 4.4.3
       IMG: redhat:8
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: redhat-8
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: redhat-8
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: redhat-9
       ARCH: amd64
       R_VERSION: 4.4.3
       IMG: redhat:9
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: redhat-9
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: redhat-9
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: redhat-9
       ARCH: arm64
       R_VERSION: 4.4.3
       IMG: redhat:9
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: redhat-9
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: redhat-9
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: redhat-10
       ARCH: amd64
       R_VERSION: 4.5.3
       IMG: redhat:10
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: redhat-10
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: redhat-10
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: redhat-10
       ARCH: arm64
       R_VERSION: 4.5.3
       IMG: redhat:10
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: redhat-10
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: redhat-10
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: ubuntu-2204
       ARCH: amd64
       R_VERSION: 4.4.3
       IMG: ubuntu:jammy
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: ubuntu-2204
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: ubuntu-2204
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: ubuntu-2204
       ARCH: arm64
       R_VERSION: 4.4.3
       IMG: ubuntu:jammy
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: ubuntu-2204
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: ubuntu-2204
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: ubuntu-2404
       ARCH: amd64
       R_VERSION: 4.4.3
       IMG: ubuntu:noble
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: ubuntu-2404
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: ubuntu-2404
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: ubuntu-2404
       ARCH: arm64
       R_VERSION: 4.4.3
       IMG: ubuntu:noble
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: ubuntu-2404
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: ubuntu-2404
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: ubuntu-2604
       ARCH: amd64
       R_VERSION: 4.4.3
       IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: ubuntu-2604
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: ubuntu-2604
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
     - OS: ubuntu-2604
       ARCH: arm64
       R_VERSION: 4.4.3
       IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: ubuntu-2604
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: ubuntu-2604
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
 
 steps:
   - name: 'Rebuild missing binaries'
@@ -153,6 +414,12 @@ steps:
       PLATFORM: ${OS}
       ARCH: ${ARCH}
       NCPUS: 2
+      SPLIT_INTO: ${SPLIT_INTO}
+      SPLIT_INDEX: ${SPLIT_INDEX}
+      # Wall clock after which the shard stops cleanly instead of having to be
+      # killed. A kill matches neither `success` nor `failure`, so it would skip
+      # the dependent re-index and leave rebuilt binaries behind a stale edge.
+      REBUILD_BUDGET_HOURS: 20
     commands:
       - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
       - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
@@ -162,18 +429,7 @@ steps:
       - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
       - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2
       - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
-      - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1
+      - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1
-      # A rebuild replaces objects in place, so the slot's index still advertises
-      # the old MD5 and, for anything that had been served from source, no Built
-      # stamp. Re-index here rather than waiting for the next process-updates
-      # run, or the rebuilt binaries stay invisible to clients until then.
-      # The codename is detected from the image's /etc/os-release.
-      - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
-      - |
-        for RBIN in /opt/R/[0-9]*/bin/R; do
-          RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
-          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
-        done
     backend_options:
       docker:
         resources:
@@ -183,25 +439,3 @@ steps:
           limits:
             memory: 18Gi
             cpu: 3000m
-
-  - name: Purge CDN cache
-    image: reg.devxy.io/docker.io/library/alpine:3.24
-    environment:
-      OTEL_R_TRACES_EXPORTER: none
-      OTEL_R_LOGS_EXPORTER: none
-      OTEL_R_METRICS_EXPORTER: none
-      BUNNYNET_API_KEY:
-        from_secret: BUNNYNET_API_KEY
-      REPO_RO_TOKEN:
-        from_secret: REPO_RO_TOKEN
-      # All hostnames on the zone share this id, so one purge covers
-      # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com.
-      BUNNY_PULLZONE: '3857050'
-    commands:
-      - apk add --no-cache -q bash curl git
-      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
-      - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE"
-    # A rebuild that died part-way still replaced objects, and those are exactly
-    # the ones a stale edge would keep hiding, so purge either way.
-    when:
-      - status: [success, failure]

.crow/weekly-rebuild-reindex.yaml

@@ -0,0 +1,191 @@
+# Re-index and purge after weekly-rebuild-missing.
+#
+# weekly-rebuild-missing runs three shards per slot. Each of them replaces
+# objects in place, so the slot's index still advertises the old MD5 and, for
+# anything that had been served from source, no Built stamp. Re-indexing from
+# inside a shard would mean three concurrent `upload_package_index()` calls on
+# one prefix: `cranlike::update_PACKAGES()` lists the live bucket, so an early
+# lister that uploads last publishes an index missing its siblings' work.
+#
+# So it happens exactly once per slot, here, after every shard has finished.
+# `runs_on: [success, failure]` keeps that true when a shard fails; only an
+# explicit cancel skips it, and this pipeline can then be triggered on its own.
+
+variables:
+  # Mirrors the gate on weekly-rebuild-missing so a manual run re-indexes
+  # exactly the slots it rebuilt. A manual pipeline creation instantiates every
+  # file in .crow/, so the default must match no matrix row.
+  weekly_rebuild_missing:
+    description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
+    options:
+      - none
+      - all
+      - alpine-322-amd64
+      - alpine-322-arm64
+      - alpine-323-amd64
+      - alpine-323-arm64
+      - alpine-324-amd64
+      - alpine-324-arm64
+      - redhat-8-amd64
+      - redhat-8-arm64
+      - redhat-9-amd64
+      - redhat-9-arm64
+      - redhat-10-amd64
+      - redhat-10-arm64
+      - ubuntu-2204-amd64
+      - ubuntu-2204-arm64
+      - ubuntu-2404-amd64
+      - ubuntu-2404-arm64
+      - ubuntu-2604-amd64
+      - ubuntu-2604-arm64
+    default: none
+
+when:
+  - event: cron
+    cron: weekly-rebuild-missing-${OS}-${ARCH}
+  - event: manual
+    evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"'
+
+depends_on:
+  - weekly-rebuild-missing
+
+runs_on: [success, failure]
+
+skip_clone: true
+
+labels:
+  group: rpkgs-${ARCH}
+
+matrix:
+  include:
+    - OS: alpine-322
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+    - OS: alpine-322
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+    - OS: alpine-323
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+    - OS: alpine-323
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+    - OS: alpine-324
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: alpine-324
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: redhat-8
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+    - OS: redhat-8
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+    - OS: redhat-9
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+    - OS: redhat-9
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+    - OS: redhat-10
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+    - OS: redhat-10
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+    - OS: ubuntu-2204
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2204
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2404
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+    - OS: ubuntu-2404
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+    - OS: ubuntu-2604
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+    - OS: ubuntu-2604
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+
+steps:
+  - name: 'Re-index the slot'
+    image: reg.devxy.io/rpkgs/build-env-${IMG}
+    pull: true
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GIT_USER: pat-s
+      R_LIBS_USER: /mnt/cache/R-pkgs
+      R_VERSION: ${R_VERSION}
+      PLATFORM: ${OS}
+      ARCH: ${ARCH}
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - mkdir -p /mnt/cache/R-pkgs
+      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
+      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
+      # The codename is detected from the image's /etc/os-release.
+      - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
+      - |
+        for RBIN in /opt/R/[0-9]*/bin/R; do
+          RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
+          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
+        done
+
+  - name: Purge CDN cache
+    image: reg.devxy.io/docker.io/library/alpine:3.24
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      BUNNYNET_API_KEY:
+        from_secret: BUNNYNET_API_KEY
+      # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
+      # Bunny pull zones, so both must be purged after the shared origin changes.
+      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io'
+    commands:
+      - apk add --no-cache -q bash curl jq
+      # Crow carries the checkout from the re-index step into this step.
+      - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
+    # Runs on every row rather than on one designated slot: a cron fires only
+    # its own slot's row, so gating on a named slot would leave every other
+    # slot unpurged. A manual "all" run therefore purges the zone 18 times,
+    # which is a cheap API call and rare.
+    #
+    # Run it even when the re-index above failed: the objects were still
+    # replaced, and a stale edge is exactly what keeps them hidden.
+    when:
+      - status: [success, failure]

.pre-commit-config.yaml

@@ -36,7 +36,7 @@ repos:
     hooks:
       - id: air-format
   - repo: https://github.com/editorconfig-checker/editorconfig-checker
-    rev: v3.11.1
+    rev: v3.11.2
     hooks:
       - id: editorconfig-checker
         exclude: ^local/patches/.*\.patch$

.terraform.lock.hcl

@@ -2,79 +2,79 @@
 # Manual edits may be lost in future updates.
 
 provider "registry.opentofu.org/hashicorp/http" {
-  version = "3.6.0"
+  version = "3.6.1"
   hashes = [
-    "h1:0n4RBz9zNw6TTddh5+x7E8L2+qzPXNwKhK4uoZ/DUwE=",
+    "h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=",
-    "h1:22Ob7lpzMBSqdrCvoFN5EgmhGPHPBovV/9qo0c/Cd+A=",
+    "h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=",
-    "h1:2IRBvmWOYrq/ooaYYn2i86jZb7iIUvlg0KlmOMfDHoQ=",
+    "h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=",
-    "h1:5mucXikk4OcW3un3u94QnMx4AB4Wfih+sXeMd5QxSNk=",
+    "h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=",
-    "h1:5oU7Zm+2gAVGmxqtJ9E8uTudUkYy/DEn/y3IWphdv4k=",
+    "h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=",
-    "h1:5w0R4b1/VSzpqQF1tXXPr/qmaQLPVRXamOmPKWFcTk4=",
+    "h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=",
-    "h1:AEVeJr8xGmwad+JUUQ833C3x5d4W+W2szF5DfwxYppw=",
+    "h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=",
-    "h1:CPHJ+0zQbS/cX1m55Y90jIOgf1jV3ocUUnqsXAh+9Eg=",
+    "h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=",
-    "h1:JPewnGDOJudNer5+ghqwXoaJkfot3QRq9uiEYvo+JHU=",
+    "h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=",
-    "h1:QzbluV2vQLxsJYxjpziQCmPndIoJ/UGS4/UHH/GpwUM=",
+    "h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=",
-    "h1:TjUNbUdqweRBq/ycQ4ixpNkx5qaYwpXEOn9QCpqNZP8=",
+    "h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=",
-    "h1:XNbcODP60ajj21N/OO7af8bBg1ltIsYkq9egn7BYbiY=",
+    "h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=",
-    "h1:tgrbgmX7WYQz9G9ncgu7TkpVB+RlLjJA/Rvp9KPlZH8=",
+    "h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=",
-    "h1:vLxthX/ZWsOZ+aHKbAMqmNKqD0K5f4nJ8ppy0Ioyup0=",
+    "h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=",
-    "h1:wZOdGBAZkY8OKEPjKz82j1HloAKOmmvtjWyTxM+I110=",
+    "h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=",
-    "zh:0f719fa5426bc883e9fa6abf7f6498e48025edafbc29015e2f5c028f1cca3b9d",
+    "zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9",
-    "zh:1b4d7dafefd6c61764b2f9ed6943ceb9a200dee3590d18747e3a5f6b20ce85e0",
+    "zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6",
-    "zh:1d23a712984866d29f7b07028a4e99c783c71f1a5dddf08bc3d4e7da9d91a1fa",
+    "zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab",
-    "zh:257d23d58c3bb024b6bc8eb88736eaf912e934ad47c639d0c3c742bddda849a1",
+    "zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b",
-    "zh:479860e1a5468f5e04013b9364c9496d7ed0804bf9a1acd8e07558d57609993d",
+    "zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c",
-    "zh:4cb5e681bf599b411b27c4a2c4066a5fb2ed79aaa3a1a3cb5a30002fec062ce9",
+    "zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4",
-    "zh:4fb35c3f643dae9f3670d719397a415f815a0b95f8ed7bd8a72f27a94ba78092",
+    "zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502",
-    "zh:59ba40825ab38db5b4a0989a2db0df35cc15d8984f898176011ba352f27d77b7",
+    "zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69",
-    "zh:61fc1252eb88088638f4c69ea4e2171cde2e5089fa632ac1e943b13787348f73",
+    "zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6",
-    "zh:7c5d6dd5f7cbc460e95d368be35c29b4e0402069b8912dbd5d1cd7fa9acef216",
+    "zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279",
-    "zh:7f76d756240d4284642f359ad470226e5378670239aadc366ef54d9d914d4d2e",
+    "zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6",
-    "zh:8133ad0814098177e0d067c816ccf1bf48bbadacd18f6f2c808c90447505723b",
+    "zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7",
-    "zh:c93be06269bb728f1968f8c50506de56c887017ac1d6e4be1f925651d8437eb6",
+    "zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df",
-    "zh:ef47b78a10a82e6cf53344a6a85a94041c28286c10a70541c564d762f1cfede0",
+    "zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621",
-    "zh:f5796a53a74999135bd9087aff50fddda59129d09b2f9b1902ff8c0c1e047e48",
+    "zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424",
   ]
 }
 
 provider "registry.terraform.io/bunnyway/bunnynet" {
-  version     = "0.17.0"
+  version     = "0.18.2"
-  constraints = "~> 0.17"
+  constraints = "~> 0.18"
   hashes = [
-    "h1:+qDt35lVSK7acw6a1xHuPYrqmZEcHSmtd+6n1TxNuYw=",
+    "h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=",
-    "h1:1dCu2l4DhPBjizVAH/WwAjT1Xbo52K4PMvHoD5zUhuU=",
+    "h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=",
-    "h1:Dvn46Auwuel4jqrqZXs2D7kdujNhs17LEmqhuY0k4/4=",
+    "h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=",
-    "h1:M5eDL3m2uSEr1XATJW0foHzKl8pFhCtgKuOM24bJRwU=",
+    "h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=",
-    "h1:PddaC7nM/gY4x9i3xy6TxOs9MAu2/6g58Xs/gv4DRV8=",
+    "h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=",
-    "h1:QVIKiZluI+NQAKu8NpFBl3Nvyx+d81vW9btEUdIQREc=",
+    "h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=",
-    "h1:S6TnzXHsRoGYvC1vJBkDiVEc0spceksY4n6x5WN5iYw=",
+    "h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=",
-    "h1:VcxZDWqCWMSjcUsC1K4sB6uYEoeoou+BC0ePoJXmf3A=",
+    "h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=",
-    "h1:W0y/agBVqls1cJlFGFYMu2VnqoPXFzxVHPIYe3OqfYQ=",
+    "h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=",
-    "h1:XmNd5fP9a0O77ve5BMQP2vARExgIa7rYl6KvyUYXPSs=",
+    "h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=",
-    "h1:e0EFKrWSQwaa/kGhnha4DXk4T68Av8QxP84mRSdWC9M=",
+    "h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=",
-    "h1:eM+/lUiU0pNSgQKoqKPgE3xJrJ0MHIpKG+yhaGB/P0M=",
+    "h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=",
-    "h1:fPWWA4T0/y7GX+tCGN23l1jODhZ3uCdR/MKgZDXYpAE=",
+    "h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=",
-    "h1:g+r2GVi4gVC4DuQg3PL70gW9BDskgWUzCBIMXTUq63A=",
+    "h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=",
-    "h1:gaZ8eALDtVHqykVDHav8004gHiMGaYR/3KwET0FUgao=",
+    "h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=",
-    "h1:kbqW25eaiv4N/N/z+sxLdJZ15yh5cgnRD/q6RclPMLc=",
+    "h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=",
-    "h1:rGjxue3mXRyQQqpywTXC4zK//JAtf0Cz7RP+uPMMJjw=",
+    "h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=",
-    "zh:05943fef14c2028f4722bf078aa1889229e94302f7678cc6f63adb669d8ea612",
+    "zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c",
-    "zh:26a163930a92a7408f7bbd0130064b84df8a232b500d8c6c3989952986308539",
+    "zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178",
-    "zh:41305feaaade55391447521ec309f3c038b631ca542907ad95132fab71a7e116",
+    "zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd",
-    "zh:606919a930f0299948504adbdcd0f239a8af5c418f85741c48f8add370a3d038",
+    "zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef",
-    "zh:66963d5b445639511939fc508513fd31da3ee1d4ee1a565ee396c9532897a349",
+    "zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738",
-    "zh:6c981ec0c8545556395c43e2511861ab65ee9ecf2a960480e7889c3af0d23af3",
+    "zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc",
-    "zh:7334a1bdb726ce1f1bf0a3155f30f84f65206980c229c832ff5f0b0718c44e0b",
+    "zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c",
-    "zh:75f6c86bf74511e605423332d113711c76c8028361a32282fb3359d6c7ecae9e",
-    "zh:7aebb1a01cfe8be54903853202ae06eba14ad99c37d230ed93ce7d6633e05e9b",
     "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f",
-    "zh:9041d0e20c9ceea532de6eebf5cb3a27dad0bb49d3f5b5154be2a08d68fbbf1f",
+    "zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef",
-    "zh:a6bbf65431a02be4df0ebb1cbe01185ad357ff6e33c01bd0558f59bed90c8f36",
+    "zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3",
-    "zh:c6d075a31096f080c388dfe46036f451c0cc114c3311a4f46ab8dbe1938a202f",
+    "zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb",
-    "zh:dd8703f7b55b8bc8e10f8718bea889781100b18e932b04898995b63178c3d36e",
+    "zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e",
-    "zh:dd92a5cd4e133a4000e7e5bc8cce876ae0ed803543cedd2f3d590661ba244d04",
+    "zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11",
-    "zh:e024fdf121bebc48c1e6debea344c6d4f174117f3ae605fca6e13b9705d92d22",
+    "zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7",
-    "zh:ee0e80c31b438e35fa1608f6a2f5824d2806db1e5e8b9f7a90986585c7bcb895",
+    "zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9",
-    "zh:fc2d4b705411b48f8c045981f9368a3ea2f74969dd6302008c31ff0bedd51f0a",
+    "zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1",
+    "zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff",
+    "zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339",
   ]
 }

cdn.tf

@@ -32,7 +32,7 @@
 #   cache_stale           = ["offline", "updating"]
 #   use_background_update = true
 
-  # block_ips = var.cdn_block_ips
+# block_ips = var.cdn_block_ips
 
 #   # 50 TB
 #   limit_bandwidth = 50000000000000
@@ -82,7 +82,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" {
 
   cache_expiration_time = 31919000
   websockets_enabled    = false
-  errorpage_whitelabel = true
+  errorpage_whitelabel  = true
 
   origin {
     type              = "OriginUrl"
@@ -147,6 +147,65 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" {
   tls_enabled = true
 }
 
+# Alliance SwissPass historically used a separate, manually configured pull
+# zone. Adopt it so both public repositories use the same B2 origin, middleware
+# release and cache behavior.
+import {
+  to = bunnynet_pullzone.cran_allianceswisspass
+  id = "3265648"
+}
+
+resource "bunnynet_pullzone" "cran_allianceswisspass" {
+  name = "cran-allianceswisspass"
+
+  cache_errors          = false
+  cache_expiration_time = 31919000
+  websockets_enabled    = false
+  errorpage_whitelabel  = true
+
+  origin {
+    type              = "OriginUrl"
+    url               = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com"
+    middleware_script = bunnynet_compute_script.rpkgs_router.id
+  }
+
+  routing {
+    filters = [
+      "scripting",
+    ]
+  }
+
+  s3_auth_enabled = true
+  s3_auth_key     = var.B2_S3_ACCESS_KEY
+  s3_auth_secret  = var.B2_S3_SECRET_KEY
+  s3_auth_region  = "eu-central-003"
+
+  cache_enabled              = true
+  request_coalescing_enabled = true
+  block_post_requests        = true
+  cache_vary_headers         = ["User-Agent"]
+
+  limit_requests    = 5000
+  limit_connections = 1000
+
+  safehop_enabled      = true
+  add_canonical_header = true
+  cache_stale          = ["offline", "updating"]
+  block_ips            = var.cdn_block_ips
+
+  # 50 TB
+  limit_bandwidth = 50000000000000
+
+  block_root_path = true
+}
+
+resource "bunnynet_pullzone_hostname" "cran_allianceswisspass" {
+  pullzone    = bunnynet_pullzone.cran_allianceswisspass.id
+  name        = "cran.allianceswisspass.devxy.io"
+  force_ssl   = true
+  tls_enabled = true
+}
+
 # resource "bunnynet_storage_zone" "devxy-r-binaries" {
 #   name      = "devxy-r-binaries-storage"
 #   region    = "DE"

edge/rpkgs-router.test.ts

@@ -117,6 +117,13 @@ Deno.test('rpkgs-router', async (t) => {
       assertEquals(res.status, 200);
     });
 
+    await t.step('serves an archived binary when it exists', async () => {
+      const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`;
+      const res = await probe(path, UA_R45_MUSL);
+      assertEquals(res.status, 200);
+      assertEquals(res.location, null);
+    });
+
     await t.step('does not redirect a path already under a minor', async () => {
       const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL);
       assertEquals(res.location, null);

edge/rpkgs-router.ts

@@ -27,6 +27,7 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12';
 
 const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com';
 const CRAN_ORIGIN = 'https://cran.r-project.org';
+const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']);
 
 /** Slots ("<arch>/<os>", comma separated) whose per-minor index is a union. */
 const UNION_SLOTS = new Set(
@@ -47,6 +48,10 @@ const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/;
 
 const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/;
 
+/** A binary archive URL whose upstream source counterpart CRAN can serve. */
+const ARCHIVE_TARBALL_REGEX =
+  /^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/;
+
 const MACOS_BIN_REGEX =
   /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/;
 
@@ -85,6 +90,10 @@ function redirectTo(location: string, status = 302): Response {
   });
 }
 
+function publicCdnOrigin(url: URL): string {
+  return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN;
+}
+
 function extractRMinor(userAgent: string): string | null {
   for (const regex of R_MINOR_REGEXES) {
     const match = userAgent.match(regex);
@@ -181,15 +190,14 @@ BunnySDK.net.http
     const url = new URL(ctx.request.url);
     const path = normalizePathname(url.pathname);
     const userAgent = ctx.request.headers.get('User-Agent') || '';
+    const publicOrigin = publicCdnOrigin(url);
 
     // macOS clients are served from CRAN's own binary tree.
     const srcContrib = path.match(SRC_CONTRIB_REGEX);
     if (srcContrib && /darwin/.test(userAgent)) {
       const mac = parseMacUserAgent(userAgent);
       if (mac) {
-        return Promise.resolve(
+        return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`));
-          redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`),
-        );
       }
     }
 
@@ -213,7 +221,7 @@ BunnySDK.net.http
       if (target === path) {
         return Promise.resolve(ctx.request);
       }
-      return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`));
+      return Promise.resolve(redirectTo(`${publicOrigin}${target}`));
     }
 
     // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent.
@@ -224,12 +232,27 @@ BunnySDK.net.http
       }
 
       const rest = srcContrib ? srcContrib[1] : '';
-      return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`));
+      return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`));
     }
 
     return Promise.resolve(ctx.request);
   })
-  .onOriginResponse((ctx) => {
+  .onOriginResponse(async (ctx) => {
+    const path = normalizePathname(new URL(ctx.request.url).pathname);
+    const archive = path.match(ARCHIVE_TARBALL_REGEX);
+
+    // Binary archives can be incomplete when an older build never succeeded.
+    // Preserve renv/remotes version restores by falling back to CRAN's source
+    // package only for an absent archived tarball. A requested version can be
+    // either archived upstream or still current, so probe the archive first.
+    // Other 404s remain visible.
+    if (ctx.response.status === 404 && archive) {
+      const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`;
+      const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' });
+      const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`;
+      return redirectTo(sourceUrl);
+    }
+
     ctx.response.headers.append('X-Via', 'MyMiddleware');
     return Promise.resolve(ctx.response);
   });

local/rebuild-missing-helpers.R

@@ -0,0 +1,87 @@
+# Pure helpers for local/rebuild-missing.R, kept separate so local/tests can
+# source them without executing a rebuild.
+
+# Interleaved slice of the rebuild list.
+#
+# The list is alphabetical and build cost clusters by name (Rcpp*, Bioc*,
+# rstan*), so contiguous thirds would be badly unbalanced. Interleaving also
+# makes each shard's progress counter representative of the slot as a whole.
+shard_slice <- function(pkgs, split_into, split_index) {
+  split_into <- as.integer(split_into)
+  split_index <- as.integer(split_index)
+  if (is.na(split_into) || is.na(split_index)) {
+    stop("shard_slice(): split_into and split_index must be integers")
+  }
+  if (split_into < 1L || split_index < 1L || split_index > split_into) {
+    stop(sprintf(
+      "shard_slice(): need 1 <= split_index <= split_into, got %s of %s",
+      split_index,
+      split_into
+    ))
+  }
+  # seq() errors on a descending range, which is what an empty list or a shard
+  # index past the end would produce.
+  if (length(pkgs) < split_index) {
+    return(pkgs[0L])
+  }
+  pkgs[seq.int(split_index, length(pkgs), by = split_into)]
+}
+
+# Packages that still need building, decided from the bucket rather than from
+# remembered progress.
+#
+# This is bincraft's `check_s3_root_package()` evaluated in bulk: an object
+# whose ETag equals CRAN's published MD5sum is byte-identical to CRAN's source,
+# so the build that was supposed to replace it has not happened yet.
+#
+# `etag_by_file`  named by `<pkg>_<ver>.tar.gz`, values are unquoted ETags
+# `cran_version`  named by package
+# `cran_md5`      named by `<pkg>_<ver>`
+#
+# Unknown always means "already a binary", never "rebuild it", so an unreadable
+# CRAN index or a multipart ETag can never mass-schedule work.
+outstanding_packages <- function(pkgs, etag_by_file, cran_version, cran_md5) {
+  if (length(pkgs) == 0L) {
+    return(pkgs)
+  }
+
+  # An empty table indexes to zero length rather than to NA, which would
+  # recycle the whole result away and silently report "nothing to build".
+  lookup <- function(table, key) {
+    if (length(table) == 0L) {
+      return(rep(NA_character_, length(key)))
+    }
+    unname(as.character(table[key]))
+  }
+
+  version <- lookup(cran_version, pkgs)
+  file <- sprintf("%s_%s.tar.gz", pkgs, version)
+  etag <- lookup(etag_by_file, file)
+  md5 <- lookup(cran_md5, paste(pkgs, version, sep = "_"))
+
+  # No CRAN version means the package cannot be resolved to a tarball at all;
+  # leave it in and let bincraft report why.
+  unresolved <- is.na(version)
+  # No object at the key: never built, so it is outstanding by definition.
+  absent <- !unresolved & is.na(etag)
+  # A multipart upload carries a compound ETag rather than an MD5.
+  unknown <- !is.na(etag) & grepl("-", etag, fixed = TRUE)
+
+  is_source <- !unresolved &
+    !is.na(etag) &
+    !unknown &
+    !is.na(md5) &
+    etag == md5
+
+  pkgs[unresolved | absent | is_source]
+}
+
+parse_rebuild_args <- function(args) {
+  pos <- args[!startsWith(args, "--")]
+  budget <- as.numeric(pos[3L])
+  list(
+    split_into = as.integer(pos[1L]),
+    split_index = as.integer(pos[2L]),
+    budget_hours = if (is.na(budget)) 20 else budget
+  )
+}

local/rebuild-missing.R

@@ -0,0 +1,194 @@
+### Rebuild one shard of a slot's missing-binary list.
+#
+# Usage: Rscript local/rebuild-missing.R <split_into> <split_index> [budget_hours]
+#
+# The list itself comes from local/fetch-rebuild-packages-from-issue.R, which
+# writes $REBUILD_PKG_LIST (default /tmp/rebuild_pkgs.txt).
+#
+# Two properties matter here and are the reason this is a script rather than an
+# `R -q -e` argument in the pipeline:
+#
+#   * it is restartable. The outstanding set is re-derived from the bucket on
+#     every start, so a shard that died resumes where it stopped without any
+#     progress file, and without replaying thousands of per-package HEADs.
+#   * it terminates. A wall-clock budget stops the loop cleanly instead of the
+#     run having to be killed, which is what previously skipped the re-index and
+#     CDN purge and left rebuilt binaries hidden behind stale edge copies.
+
+options(error = function() {
+  cat("ERROR:", geterrmessage(), "\n", file = stdout())
+  traceback(2)
+  q(status = 1)
+})
+
+library(bincraft, quietly = TRUE)
+
+source(file.path("local", "rebuild-missing-helpers.R"))
+
+args <- parse_rebuild_args(commandArgs(trailingOnly = TRUE))
+if (is.na(args$split_into) || is.na(args$split_index)) {
+  stop("usage: rebuild-missing.R <split_into> <split_index> [budget_hours]")
+}
+
+list_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt")
+pkgs <- if (file.exists(list_file)) readLines(list_file) else character(0)
+pkgs <- pkgs[nzchar(pkgs)]
+if (length(pkgs) == 0L) {
+  cat("Nothing to rebuild\n")
+  q("no")
+}
+
+excluded <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]]
+pkgs <- setdiff(pkgs, excluded)
+
+mine <- shard_slice(pkgs, args$split_into, args$split_index)
+cat(sprintf(
+  "Shard %s/%s: %s of %s listed packages\n",
+  args$split_index,
+  args$split_into,
+  length(mine),
+  length(pkgs)
+))
+
+### Resume: ask the bucket what is still outstanding
+
+codename <- bincraft::set_codename(NULL)
+local_machine <- Sys.info()[["machine"]]
+arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64"
+slot_dir <- sprintf(
+  "devxy-rpkgs-binaries/%s/%s/latest/src/contrib",
+  arch,
+  codename
+)
+
+s3fs::s3_file_system(
+  aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
+  aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
+  endpoint = "https://s3.eu-central-003.backblazeb2.com",
+  region_name = "eu-central-003",
+  refresh = TRUE
+)
+
+# One paginated listing instead of a HEAD per package. Not recursed: the
+# rebuild passes no `is_r_minor_sensitive`, so it only ever targets the flat
+# path, and the resume filter matches that scope deliberately.
+info <- tryCatch(s3fs::s3_dir_info(slot_dir), error = function(e) NULL)
+etag_by_file <- if (is.null(info) || nrow(info) == 0L) {
+  cat(sprintf(
+    "WARNING: could not list %s; building the whole shard\n",
+    slot_dir
+  ))
+  stats::setNames(character(), character())
+} else {
+  stats::setNames(
+    gsub('^"|"$', "", as.character(info$etag)),
+    basename(as.character(info$uri))
+  )
+}
+
+cran <- tryCatch(
+  {
+    con <- gzcon(url(
+      "https://cloud.r-project.org/src/contrib/PACKAGES.gz",
+      open = "rb"
+    ))
+    on.exit(close(con), add = TRUE)
+    read.dcf(con, fields = c("Package", "Version", "MD5sum"))
+  },
+  error = function(e) {
+    cat(sprintf(
+      "WARNING: could not read CRAN's index (%s)\n",
+      conditionMessage(e)
+    ))
+    NULL
+  }
+)
+cran_version <- stats::setNames(character(), character())
+cran_md5 <- stats::setNames(character(), character())
+if (!is.null(cran)) {
+  cran_version <- stats::setNames(
+    as.character(cran[, "Version"]),
+    as.character(cran[, "Package"])
+  )
+  keep <- !is.na(cran[, "MD5sum"])
+  cran_md5 <- stats::setNames(
+    as.character(cran[keep, "MD5sum"]),
+    paste(cran[keep, "Package"], cran[keep, "Version"], sep = "_")
+  )
+}
+
+before <- length(mine)
+mine <- outstanding_packages(mine, etag_by_file, cran_version, cran_md5)
+cat(sprintf(
+  "Resume: %s of %s already carry a binary; %s outstanding\n",
+  before - length(mine),
+  before,
+  length(mine)
+))
+
+if (length(mine) == 0L) {
+  cat("Nothing outstanding for this shard\n")
+  q("no")
+}
+
+### Build
+
+options(
+  crayon.enabled = TRUE,
+  Ncpus = as.integer(Sys.getenv("NCPUS", "2")),
+  future.globals.onReference = NULL
+)
+
+started <- Sys.time()
+n <- length(mine)
+completed <- 0L
+for (i in seq_along(mine)) {
+  elapsed <- as.numeric(difftime(Sys.time(), started, units = "hours"))
+  if (elapsed > args$budget_hours) {
+    cat(sprintf(
+      "Budget of %sh reached after %d/%d packages; stopping cleanly. The next run resumes from the bucket.\n",
+      args$budget_hours,
+      completed,
+      n
+    ))
+    break
+  }
+
+  x <- mine[i]
+  cat(sprintf("[%d/%d] %s\n", i, n, x))
+  tryCatch(
+    bincraft::build_binary_package(
+      x,
+      tag_limit = 1L,
+      patches = "local/patches",
+      s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
+      s3_region = "eu-central-003",
+      s3_bucket = "devxy-rpkgs-binaries",
+      s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
+      s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
+      metadata_db_host = "r-binaries.devxy.io",
+      metadata_db_name = "build_metadata",
+      metadata_db_table = "single_builds",
+      metadata_db_user = "rpkgs",
+      metadata_db_password = Sys.getenv("PGPASS"),
+      metadata_db_sslmode = "require",
+      metadata_db_port = 15432,
+      archive = TRUE,
+      upload = TRUE,
+      store_build_metadata = TRUE
+    ),
+    error = function(e) {
+      cat(sprintf("ERROR building %s - %s\n", x, conditionMessage(e)))
+    }
+  )
+  completed <- completed + 1L
+}
+
+cat(sprintf(
+  "Shard %s/%s finished: %d/%d packages processed in %.1fh\n",
+  args$split_index,
+  args$split_into,
+  completed,
+  n,
+  as.numeric(difftime(Sys.time(), started, units = "hours"))
+))

local/tests/test-rebuild-missing.R

@@ -0,0 +1,94 @@
+source(file.path("..", "rebuild-missing-helpers.R"))
+
+test_that("shard_slice partitions the list without gaps or overlap", {
+  pkgs <- letters[1:10]
+  parts <- lapply(1:3, function(i) shard_slice(pkgs, 3, i))
+
+  expect_identical(parts[[1]], c("a", "d", "g", "j"))
+  expect_identical(parts[[2]], c("b", "e", "h"))
+  expect_identical(parts[[3]], c("c", "f", "i"))
+
+  expect_identical(sort(unlist(parts)), sort(pkgs))
+  expect_identical(anyDuplicated(unlist(parts)), 0L)
+})
+
+test_that("shard_slice is deterministic and survives short lists", {
+  expect_identical(
+    shard_slice(letters[1:10], 3, 2),
+    shard_slice(letters[1:10], 3, 2)
+  )
+  expect_identical(shard_slice(character(0), 3, 1), character(0))
+  # more shards than packages: the tail shards get nothing rather than erroring
+  expect_identical(shard_slice(c("a"), 3, 1), "a")
+  expect_identical(shard_slice(c("a"), 3, 2), character(0))
+})
+
+test_that("shard_slice rejects an out-of-range index", {
+  expect_error(shard_slice(letters, 3, 4), "split_index")
+  expect_error(shard_slice(letters, 3, 0), "split_index")
+})
+
+test_that("outstanding_packages keeps source fallbacks and drops real binaries", {
+  cran_version <- c(httr = "1.4.8", R6 = "2.6.1", curl = "7.1.0")
+  cran_md5 <- c(
+    httr_1.4.8 = "8756015b94a9cff6f410ca4de8557f12",
+    R6_2.6.1 = "f01b1787f12797c29194d63c9afd5d70",
+    curl_7.1.0 = "8af2ccbf5d85dc18866f45f1f26f348d"
+  )
+  etag <- c(
+    # byte-identical to CRAN: the build never happened
+    "httr_1.4.8.tar.gz" = "8756015b94a9cff6f410ca4de8557f12",
+    # a real binary was published
+    "R6_2.6.1.tar.gz" = "9d6087ee9adda3f0a3b8067cfc652c05"
+    # curl has no object at all
+  )
+
+  out <- outstanding_packages(
+    c("httr", "R6", "curl"),
+    etag,
+    cran_version,
+    cran_md5
+  )
+  expect_identical(out, c("httr", "curl"))
+})
+
+test_that("outstanding_packages treats unknowns as already built", {
+  cran_version <- c(a = "1.0", b = "1.0")
+  cran_md5 <- c(a_1.0 = "aaaa")
+
+  # a multipart ETag carries no MD5, and `b` is missing from CRAN's index:
+  # neither may schedule a rebuild
+  etag <- c("a_1.0.tar.gz" = "abc-3", "b_1.0.tar.gz" = "bbbb")
+
+  expect_identical(
+    outstanding_packages(c("a", "b"), etag, cran_version, cran_md5),
+    character(0)
+  )
+})
+
+test_that("outstanding_packages keeps a package CRAN has no version for", {
+  out <- outstanding_packages(
+    "ghost",
+    c(),
+    c(other = "1.0"),
+    c(other_1.0 = "aaaa")
+  )
+  expect_identical(out, "ghost")
+})
+
+test_that("outstanding_packages handles an empty list", {
+  expect_identical(
+    outstanding_packages(character(0), c(), c(), c()),
+    character(0)
+  )
+})
+
+test_that("parse_rebuild_args defaults the budget", {
+  a <- parse_rebuild_args(c("3", "2"))
+  expect_identical(a$split_into, 3L)
+  expect_identical(a$split_index, 2L)
+  expect_identical(a$budget_hours, 20)
+
+  b <- parse_rebuild_args(c("3", "2", "1.5"))
+  expect_identical(b$budget_hours, 1.5)
+})

local/uvr-install.sh

@@ -84,9 +84,20 @@ trap 'rm -rf "$project_dir"' EXIT
 cd "$project_dir"
 
 "$uvr_bin" init --here --r-version "$r_full"
-# --no-install: resolve and lock only. The install happens in the sync below,
+# --no-install resolves and locks only; retry because concurrent shards can
-# which is the only command that honours --library.
+# expose short-lived DNS or CRAN-index failures and uvr rolls the manifest back
-"$uvr_bin" add --no-install "$@"
+# cleanly after an unsuccessful resolution.
+add_attempt=1
+while ! "$uvr_bin" add --no-install "$@"; do
+  if [ "$add_attempt" -ge 4 ]; then
+    echo "error: uvr add failed after ${add_attempt} attempts" >&2
+    exit 1
+  fi
+  add_delay=$((add_attempt * 10))
+  echo "warning: uvr add attempt ${add_attempt} failed; retrying in ${add_delay}s" >&2
+  sleep "$add_delay"
+  add_attempt=$((add_attempt + 1))
+done
 
 # TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs
 # `apt-get install` for every resolved system dependency without refreshing the

provider.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     bunnynet = {
       source = "registry.terraform.io/BunnyWay/bunnynet"
-      version = "~> 0.17"
+      version = "~> 0.18"
     }
   }
 }

scripts/purge_cdn_zone.sh

@@ -18,35 +18,70 @@
 # objects were replaced. The cost is a cold cache for everything else, which is
 # why this is not used by the daily update path.
 #
-# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io,
+# The public hostnames currently use separate pull zones, so callers must pass
-# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them.
+# every zone that serves the repository. A zone can be identified by its
+# numeric ID or by one of its hostnames; hostname lookup avoids persisting IDs
+# that change when a zone is recreated.
 #
 # Usage:
-#   purge_cdn_zone.sh <BUNNYNET_API_KEY> <pull_zone_id>
+#   purge_cdn_zone.sh <BUNNYNET_API_KEY> <pull_zone> [<pull_zone>...]
 #
 set -euo pipefail
 
 if (($# < 2)); then
-  echo "usage: $0 <api_key> <pull_zone_id>" >&2
+  echo "usage: $0 <api_key> <pull_zone> [<pull_zone>...]" >&2
   exit 2
 fi
 
 api_key="$1"
-zone_id="$2"
+shift
 
-echo "Purging BunnyCDN pull zone ${zone_id}"
+resolve_zone_id() {
+  local zone="$1"
+  local response_file
+  local zone_id
 
-status=$(
+  if [[ "${zone}" =~ ^[0-9]+$ ]]; then
-  curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \
+    echo "${zone}"
+    return
+  fi
+
+  response_file=$(mktemp)
+  curl -sS -o "${response_file}" \
     -H "AccessKey: ${api_key}" \
-    -H "Content-Length: 0" \
+    "https://api.bunny.net/pullzone"
-    "https://api.bunny.net/pullzone/${zone_id}/purgeCache"
+  zone_id=$(
-)
+    jq -r --arg hostname "${zone}" \
+      '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \
+      "${response_file}"
+  )
+  rm -f "${response_file}"
 
-if [[ "${status}" != "200" && "${status}" != "204" ]]; then
+  if [[ -z "${zone_id}" ]]; then
-  echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2
+    echo "Could not find BunnyCDN pull zone for hostname ${zone}" >&2
-  cat /tmp/purge_zone_response.txt >&2
+    exit 1
-  exit 1
+  fi
-fi
 
-echo "Purged pull zone ${zone_id} (HTTP ${status})"
+  echo "${zone_id}"
+}
+
+for zone in "$@"; do
+  zone_id=$(resolve_zone_id "${zone}")
+  echo "Purging BunnyCDN pull zone ${zone_id}"
+
+  response_file="/tmp/purge_zone_response_${zone_id}.txt"
+  status=$(
+    curl -sS -o "${response_file}" -w '%{http_code}' -X POST \
+      -H "AccessKey: ${api_key}" \
+      -H "Content-Length: 0" \
+      "https://api.bunny.net/pullzone/${zone_id}/purgeCache"
+  )
+
+  if [[ "${status}" != "200" && "${status}" != "204" ]]; then
+    echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2
+    cat "${response_file}" >&2
+    exit 1
+  fi
+
+  echo "Purged pull zone ${zone_id} (HTTP ${status})"
+done

specs/2026-08-12-shard-weekly-rebuild-design.md

@@ -0,0 +1,167 @@
+# Design: Sharding and resuming the weekly rebuild
+
+Date: 2026-08-12
+Status: Approved (pending spec review)
+
+## Problem
+
+`weekly-rebuild-missing` runs one job per `<os>-<arch>` and walks that slot's rebuild list serially in a single `R -q -e` invocation (`.crow/weekly-rebuild-missing.yaml:165`).
+Until 2026-08-09 that was cheap, because every source fallback was skipped as "already built" and the list was effectively empty.
+Since bincraft #105/#106/#107 and build-cran-binaries #159 the gate works, and the lists are now large.
+
+Share of records whose object is byte-identical to CRAN's source, measured against `cran.r-project.org` MD5s on 2026-08-12:
+
+| slot               | records | source-served |        share |
+| ------------------ | ------: | ------------: | -----------: |
+| `amd64/resolute`   |  24 212 |        15 023 |        62.1% |
+| `arm64/resolute`   |  24 291 |        13 670 |        56.3% |
+| `arm64/alpine324`  |  24 328 |         9 514 |        39.2% |
+| `amd64/alpine324`  |  24 343 |         8 917 |        36.7% |
+| `arm64/rhel10`     |  24 695 |         5 384 |        21.9% |
+| `amd64/rhel10`     |  24 881 |         4 712 |        19.2% |
+| 12 remaining slots | ~24 700 |  850 to 2 130 | 3.5% to 8.7% |
+
+A single serial job cannot absorb that.
+Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) started on 2026-08-09, ran for roughly two days, reached `[8692/23885] cholera`, and was killed there.
+
+Two distinct failures follow from that shape.
+
+**No parallelism.** The work is embarrassingly parallel across packages, but one job does all of it.
+
+**No resumability, and no clean stopping point.** The loop has no terminating condition other than exhausting the list, so the only way to stop it is a kill.
+A restarted run re-reads the same list and walks it from the first entry.
+It skips completed packages via `check_s3_root_package()`, but that costs a CRAN version resolution and an S3 `HEAD` per package, thousands of times, before it reaches new work.
+Worse, a kill is not a pipeline failure: the `Purge CDN cache` step is guarded by `when: status: [success, failure]` (`.crow/weekly-rebuild-missing.yaml:206-207`), and on 10910 it produced no output at all.
+So the ~4 600 binaries that run did publish stayed hidden behind stale edge copies.
+
+## Goal
+
+Turn each slot's rebuild into bounded, parallel, restartable units, without introducing state that can disagree with the bucket.
+
+## Design
+
+### 1. Shard the matrix three ways
+
+Each of the 18 `OS`/`ARCH` rows in `.crow/weekly-rebuild-missing.yaml` gains `SPLIT_INTO: 3` and `SPLIT_INDEX: 1|2|3`, giving 54 rows.
+This mirrors `.crow/build-all-versions.yaml:57-98`, which already shards its matrix four ways per arch.
+
+Routing needs no change.
+The cron filter `cron: weekly-rebuild-missing-${OS}-${ARCH}` and the manual `evaluate: weekly_rebuild_missing == "${OS}-${ARCH}"` both match all three shards of a slot.
+Placement stays on the `rpkgs-${ARCH}` group label, so shards queue against available capacity rather than oversubscribing it.
+
+### 2. Extract the loop into `local/rebuild-missing.R`
+
+The build is currently a single ~1 500-character `R -q -e` argument.
+Shard arithmetic and resume logic do not belong in a YAML string, and none of it is testable there.
+The loop moves to `local/rebuild-missing.R`, invoked as `Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX`, mirroring `local/build-all.R`.
+Its body is unchanged in substance: read `/tmp/rebuild_pkgs.txt`, subtract `local/excluded-packages.json`, loop with `tryCatch` around `bincraft::build_binary_package()`.
+
+The slice is **interleaved**, not contiguous:
+
+```r
+# the list is alphabetical and build cost clusters by name (Rcpp*, Bioc*,
+# rstan*), so contiguous thirds would be badly unbalanced
+mine <- pkgs[seq(split_index, length(pkgs), by = split_into)]
+```
+
+`local/build-all.R:64` uses contiguous chunks via `cut()`.
+That is fine there because its list is every CRAN package and version, so the chunks average out.
+Here the list is a filtered backlog in which expensive families sit adjacent, so interleaving is the better default.
+Interleaving also makes each shard's `[i/n]` progress representative of the slot as a whole.
+
+### 3. Resume by re-deriving state from the bucket
+
+Before the loop, the shard performs one `s3fs::s3_dir_info()` on `devxy-rpkgs-binaries/<arch>/<codename>/latest/src/contrib` and reads the `etag` column.
+It fetches CRAN's `PACKAGES` once for the latest version and published `MD5sum` of every package.
+A package is still outstanding if and only if the object at `<pkg>_<version>.tar.gz` has an ETag equal to CRAN's `MD5sum` for that version, which is the definition `check_s3_root_package()` already applies one package at a time.
+
+```r
+# one paginated listing instead of ~2900 sequential HEAD requests per shard
+info <- s3fs::s3_dir_info(slot_dir)
+etag <- setNames(gsub('^"|"$', "", info$etag), basename(info$uri))
+
+key <- sprintf("%s_%s.tar.gz", mine, cran_version[mine])
+# keep a package when no object exists yet, or when the object is still
+# byte-identical to CRAN's source; drop it once a real binary is published
+mine <- mine[is.na(etag[key]) | etag[key] == cran_md5[key]]
+```
+
+This is the whole resume mechanism.
+There is no progress file, no volume, and no database cursor.
+A restarted shard recomputes ground truth and continues where it stopped, and it is correct even when a sibling shard, a `process-updates` cron, or a manual `just rebuild` completed something in the meantime.
+
+Three properties make this the right source of truth:
+
+- **It is what the build itself checks.** Any other store can disagree with the bucket; this one cannot.
+- **It is agent-independent.** `.crow/weekly-rebuild-missing.yaml` mounts no `volumes:`, unlike `.crow/build-all-versions.yaml:132-133`, so `/mnt/cache` is per-job and cannot carry progress anyway.
+- **It costs one listing.** `cranlike`'s `s3` fork already does exactly this call against this bucket at ~24 000 objects, so the approach is proven at the required scale.
+
+It must read ETags rather than the slot index's `Built` field, which is how `local/packages-to-build.R:104-130` answers the same question.
+Under this design the index is not rewritten until the dependent re-index pipeline runs (section 5), so mid-run it cannot reflect the current run's progress.
+
+Packages that genuinely fail to build re-publish their CRAN source, so they stay outstanding and would be retried on every restart.
+That is already handled upstream: `bincraft::filter_packages_with_errors()` (`R/build_binaries.R:1018`, `:1143`) drops anything with `error_occurred = TRUE`, and `store_build_metadata = TRUE` is passed on every call.
+No additional poison-pill filter is needed here.
+
+Only the flat `src/contrib` path is considered.
+The rebuild call passes no `is_r_minor_sensitive`, so it defaults to `FALSE` and only ever targets the flat path; the resume filter matches that scope deliberately.
+
+### 4. Give each shard a wall-clock budget
+
+`local/rebuild-missing.R` takes a budget, defaulting to 20 hours, and breaks out of the loop once it is exceeded:
+
+```r
+# exit cleanly rather than being killed, so the dependent re-index still runs
+if (difftime(Sys.time(), started, units = "hours") > budget_hours) {
+  cat(sprintf("Budget of %sh reached after %d/%d packages; stopping cleanly\n", budget_hours, i, n))
+  break
+}
+```
+
+It exits 0 and reports how much of the slice it covered.
+Every run then has a terminating condition, the re-index and purge always fire, and the remainder is picked up by the next run with no bookkeeping, because section 3 recomputes the outstanding set from scratch.
+
+### 5. Move the re-index and purge into `.crow/weekly-rebuild-reindex.yaml`
+
+Three shards per slot means three concurrent `upload_package_index()` calls on the same S3 prefix.
+`cranlike::update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work.
+The re-index steps (`.crow/weekly-rebuild-missing.yaml:171-176`) and the purge step (`:187-207`) therefore leave that file entirely.
+
+The new file carries:
+
+```yaml
+depends_on:
+  - weekly-rebuild-missing
+runs_on: [success, failure]
+```
+
+`runs_on: [success, failure]` validates as a workflow-level key under `crow lint`, so a failing shard no longer withholds the re-index.
+The file uses the same 18-row matrix and the same `when:` gating as `weekly-rebuild-missing`, so it only re-indexes slots that actually ran.
+Each row re-indexes the flat slot and every per-minor slot.
+`scripts/purge_cdn_zone.sh` runs once on a single row, because all hostnames share pull zone `3857050` and 18 identical zone purges would be waste.
+
+## Failure behaviour
+
+| case                       | today                               | after                                                 |
+| -------------------------- | ----------------------------------- | ----------------------------------------------------- |
+| one package errors         | `tryCatch` logs, loop continues     | unchanged                                             |
+| a shard fails outright     | purge runs, re-index does not       | re-index and purge run via `runs_on`                  |
+| a shard exceeds its budget | cannot happen, runs until killed    | exits 0, re-index and purge run                       |
+| a shard is killed          | nothing runs                        | still nothing; trigger the re-index pipeline alone    |
+| a shard restarts           | re-walks the list, HEAD per package | one listing, resumes at the first outstanding package |
+
+The known cost of `depends_on` being file-level rather than row-level: on the weekly cron no slot is re-indexed until the slowest of all 54 jobs finishes.
+The 20-hour budget bounds that at roughly one day.
+
+## Out of scope
+
+- `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row for the platform and arch, which is precisely the source-fallback set. That is a separate change.
+- Bunny Perma-Cache eviction. `scripts/purge_cdn_zone.sh` purges the regular edge cache only; see the note in `CLAUDE.md` and issue history.
+- The audit that produces the rebuild list is unchanged.
+
+## Verification
+
+- `crow lint .crow/` passes for both pipeline files.
+- `local/rebuild-missing.R` gets unit coverage in `local/tests/` for the two pure pieces: the interleaved slice (disjoint, covering, deterministic) and the outstanding-set filter (source-served ETag kept, binary ETag dropped, absent object kept).
+- A single-slot manual run of `alpine-324-amd64` shard 1 confirms the listing shortcut against the live bucket, and that the reported outstanding count is close to the 8 917 measured above divided by three.
+- Restarting that shard mid-run confirms it resumes rather than replaying, by comparing the outstanding count it reports on the second start.