docs(rebuild): record the matrix size against the Crow permutation cap

feat(rebuild): shard the weekly rebuild and make each shard resumable
Split every OS/arch row of weekly-rebuild-missing into three shards and move the build loop out of the inline R one-liner into local/rebuild-missing.R. A shard re-derives its outstanding set from the bucket on every start: an object whose ETag equals CRAN's published MD5sum is still a source fallback and needs building. That is bincraft's check_s3_root_package() evaluated in bulk, so a restart resumes rather than replaying thousands of per-package HEADs, and it stays correct when a sibling shard or a process-updates run finishes work in the meantime. No progress file, no volume, no database cursor. Give each shard a 20h wall-clock budget so it exits cleanly instead of having to be killed. A kill matches neither success nor failure, which is how pipeline 10910 skipped its CDN purge and left ~4600 rebuilt binaries behind stale edge copies. Move re-indexing and the purge into weekly-rebuild-reindex.yaml, which depends on the rebuild and runs on success or failure. Three shards writing one slot's PACKAGES concurrently would race: update_PACKAGES lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work.
2026-08-12 09:18:48 +00:00 · 2026-08-12 08:27:45 +00:00 · 2026-08-12 08:17:49 +00:00
45 changed files with 310 additions and 4551 deletions
--- a/.crow/archive-missed-packages.yaml
+++ b/.crow/archive-missed-packages.yaml
@ -28,6 +28,10 @@ matrix:
      ARCH: amd64
    - CODENAME: redhat-10
      ARCH: arm64
    - CODENAME: alpine322
      ARCH: amd64
    - CODENAME: alpine322
      ARCH: arm64
    - CODENAME: alpine323
      ARCH: amd64
    - CODENAME: alpine323
--- a/.crow/auto-apply-patches.yaml
+++ b/.crow/auto-apply-patches.yaml
@ -58,7 +58,7 @@ steps:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - mkdir -p /mnt/cache/R-pkgs
      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
-      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite
+      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
      - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT
    backend_options:
      kubernetes:
--- a/.crow/backfill-abi-classification.yaml
+++ b/.crow/backfill-abi-classification.yaml
@ -1,90 +0,0 @@
 # Populate the ABI classifier cache for packages that predate the per-minor scheme.
 #
 # bincraft decides whether a flat binary may be carried into a per-minor index
 # from its risky-package set. That set is now the union of the per-minor tarball
 # scan and the `abi_classification` cache, but the cache only holds verdicts for
 # packages that passed through the build pipeline after the scheme existed.
 # Everything older has no verdict, and no verdict means "carry it" - which is how
 # a base64enc built under R 4.4.3 reached R 4.6 clients and died at dyn.load()
 # on SETLENGTH.
 #
 # The verdict is computed from the package source, so it does not depend on OS,
 # arch or R minor: one run populates the cache for every slot. The arch/codename
 # below only choose which flat index to enumerate.
 #
 # Re-running is cheap. Cached verdicts are returned without touching the network,
 # so a second run costs one query per package and downloads only what is still
 # unclassified.
 #
 #   crow pipeline create devxy/build-cran-binaries --var backfill_abi=run
 variables:
  # A manual pipeline creation instantiates every file in .crow/, so the default
  # must be a value the `when` clause rejects.
  backfill_abi:
    description: "Set to 'run' to classify the flat index into the ABI cache."
    options:
      - none
      - run
    default: none
  BACKFILL_ARCH:
    description: 'Arch of the flat index to enumerate.'
    options:
      - amd64
      - arm64
    default: amd64
  BACKFILL_CODENAME:
    description: 'Codename of the flat index to enumerate.'
    default: resolute
  BACKFILL_LIMIT:
    description: 'Classify at most this many packages (0 = no limit).'
    default: '0'
 when:
  - event: manual
    evaluate: 'backfill_abi == "run"'
 skip_clone: true
 labels:
  group: rpkgs-amd64
 steps:
  - name: 'Classify the flat index into the ABI cache'
    image: reg.devxy.io/rpkgs/build-env-ubuntu:resolute
    pull: true
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      PGPASS:
        from_secret: PGPASS
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
      GITHUB_PAT:
        from_secret: GITHUB_PAT
      GIT_USER: pat-s
      # Pipeline variables are substituted into the YAML, not exported, so they
      # have to be bound here to be readable as env vars in the script below.
      BACKFILL_ARCH: ${BACKFILL_ARCH}
      BACKFILL_CODENAME: ${BACKFILL_CODENAME}
      BACKFILL_LIMIT: ${BACKFILL_LIMIT}
      R_LIBS_USER: /tmp/R-pkgs
    commands:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - mkdir -p /tmp/R-pkgs
      - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R
      - |
        set -eu
        LIMIT=""
        if [ "$$BACKFILL_LIMIT" != "0" ]; then
          LIMIT="$$BACKFILL_LIMIT"
        fi
        /opt/R/4.5.3/bin/Rscript local/backfill-abi-classification.R \
          "$$BACKFILL_ARCH" "$$BACKFILL_CODENAME" $$LIMIT
    backend_options:
      docker:
        resources:
          requests:
            memory: 4Gi
            cpu: 1000m
--- a/.crow/backfill-flat-safety.yaml
+++ b/.crow/backfill-flat-safety.yaml
@ -1,176 +0,0 @@
 # Record, per flat-slot object, whether it loads under every supported R minor.
 #
 # `union_index_records()` keeps a carried flat record when that object is
 # recorded safe and drops it otherwise. Objects uploaded before that record
 # existed have no verdict, so they are dropped: a full re-index removed roughly
 # 2000 records per per-minor index (16398 across the 4.4 indexes, 20076 across
 # 4.5, 15532 across 4.6), each one resolving to CRAN source for clients the
 # routing serves correctly.
 #
 # This inspects what is already in the bucket and writes the verdict, which is
 # what restores that coverage. Decided by symbols rather than heuristics: a
 # binary fails at dyn.load() under minor M exactly when it references a symbol M
 # does not export, and the volatile set is derived by diffing the R builds in
 # the image.
 #
 # Re-running is cheap; recorded objects are skipped.
 #
 #   crow pipeline create devxy/build-cran-binaries --var flat_safety=all
 variables:
  # A manual pipeline creation instantiates every file in .crow/, so the default
  # must match no matrix row.
  flat_safety:
    description: "Inspect target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
    options:
      - none
      - all
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
      - alpine-324-arm64
      - redhat-8-amd64
      - redhat-8-arm64
      - redhat-9-amd64
      - redhat-9-arm64
      - redhat-10-amd64
      - redhat-10-arm64
      - ubuntu-2204-amd64
      - ubuntu-2204-arm64
      - ubuntu-2404-amd64
      - ubuntu-2404-arm64
      - ubuntu-2604-amd64
      - ubuntu-2604-arm64
    default: none
  FLAT_SAFETY_LIMIT:
    description: 'Inspect at most this many objects per slot (0 = no limit).'
    default: '0'
 when:
  - event: manual
    evaluate: 'flat_safety == "all" || flat_safety == "${OS}-${ARCH}"'
 skip_clone: true
 labels:
  group: rpkgs-${ARCH}
 matrix:
  include:
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-323
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-324
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: alpine-324
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: redhat-8
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-8
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-9
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-9
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-10
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: redhat-10
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: ubuntu-2204
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2204
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2404
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2404
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2604
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
    - OS: ubuntu-2604
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
 steps:
  - name: 'Inspect flat objects and record safety'
    image: reg.devxy.io/rpkgs/build-env-${IMG}
    pull: true
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      RED_HAT_DEV_PW:
        from_secret: RED_HAT_DEV_PW
      B2_S3_ACCESS_KEY:
        from_secret: B2_S3_ACCESS_KEY
      B2_S3_SECRET_KEY:
        from_secret: B2_S3_SECRET_KEY
      PGPASS:
        from_secret: PGPASS
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
      # uvr resolves bincraft's dependencies through the GitHub API, including
      # the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the
      # failure is not loud: uvr warns, falls back to the registry copy of
      # cranlike, and the index write then dies on `unused argument (built =
      # built)` because upstream has no such parameter. Intermittent by nature,
      # since it depends on the rate-limit window.
      GITHUB_PAT:
        from_secret: GITHUB_PAT
      GIT_USER: pat-s
      R_LIBS_USER: /tmp/R-pkgs
      FLAT_SAFETY_ARCH: ${ARCH}
      FLAT_SAFETY_LIMIT: ${FLAT_SAFETY_LIMIT}
    commands:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - mkdir -p /tmp/R-pkgs
      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
      - |
        set -eu
        LIMIT=""
        if [ "$$FLAT_SAFETY_LIMIT" != "0" ]; then
          LIMIT="$$FLAT_SAFETY_LIMIT"
        fi
        # The codename comes from the image's /etc/os-release, exactly as the
        # re-index step derives it, so it cannot drift from the matrix.
        CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))')
        echo "slot: $$FLAT_SAFETY_ARCH/$$CODENAME"
        /opt/R/$R_VERSION/bin/Rscript local/backfill-flat-safety.R \
          "$$FLAT_SAFETY_ARCH" "$$CODENAME" $$LIMIT
    backend_options:
      docker:
        resources:
          requests:
            memory: 4Gi
            cpu: 1000m
--- a/.crow/build-all-versions.yaml
+++ b/.crow/build-all-versions.yaml
@ -24,6 +24,7 @@ variables:
  OS_VERSION:
    description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).'
    options:
      - '3.22'
      - '3.23'
      - '3.24'
      - '8'
@ -34,14 +35,6 @@ variables:
      - 'resolute'
    default: '3.24'
  R_VERSION:
    # The slot's *primary* R minor: what `local/build-all.R` builds into the
    # generic slot. The loop below already runs `--sensitive-only` for every
    # other installed minor, so filling a non-primary minor's gap needs this
    # left alone, not changed.
    #
    # 4.6.0 was briefly offered here (#183) and removed: selecting it for a
    # slot whose generic binaries are 4.5-built would publish 4.6 binaries
    # into the generic slot and break every 4.5 client.
    description: 'Primary R version under /opt/R.'
    options:
      - 4.5.3
@ -149,11 +142,7 @@ steps:
      # install). Pin bincraft here, exactly like the R-minor pass below.
      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
-      # The dependency-prep job may run on another agent whose cache volume is
+      - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
      # not visible here. Install the packages needed to recompute the snapshot
      # and query build metadata in every build job.
      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite
      - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
      - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1
      - |
        PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2)
@ -165,10 +154,7 @@ steps:
          LIB="/mnt/cache/R-pkgs-$RMINOR"
          mkdir -p "$LIB"
          R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
          R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
          R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true
          # Also build current packages whose required dependency tree needs a newer R.
          R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION" $SPLIT_INTO $SPLIT_INDEX
        done
      # archive missed packages; first arg is the codename (e.g. "alpine324"),
      # derived via bincraft like the upload step, not paste(OS, OS_VERSION).
--- a/.crow/process-updates.yaml
+++ b/.crow/process-updates.yaml
@ -3,18 +3,20 @@
 # Routing is preserved 1:1:
 #   - cron:   each existing `process-cran-updates-<os>-<arch>` cron fires only
 #             its matching matrix row (via the per-row `cron:` name filter).
-#   - manual: pick a target from the `process_cran_updates` dropdown;
+#   - manual: pick a target from the `process_cran_updates` dropdown
-#             "all" fans out every os/arch as parallel matrix workflows.
+#             ("all" = every os/arch).
 # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64).
 variables:
  # Gates this pipeline. A manual pipeline creation instantiates every file in
  # .crow/, and a declared default is applied even when the run never passed
  # this variable, so the default must be a value that matches no matrix row.
  process_cran_updates:
-    description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
+    description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
    options:
      - none
      - all
      - alpine-322-amd64
      - alpine-322-arm64
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
@ -46,6 +48,18 @@ labels:
 matrix:
  include:
    - OS: alpine-322
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
      OS_ID: alpine322
      PROCESS_NEW: 'FALSE'
    - OS: alpine-322
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
      OS_ID: alpine322
      PROCESS_NEW: 'FALSE'
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
@ -189,10 +203,9 @@ steps:
      - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
      - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres
      - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
      # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
-      - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
+      - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
      # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
      - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"
      - |
@ -205,10 +218,7 @@ steps:
          LIB="/mnt/cache/R-pkgs-$RMINOR"
          mkdir -p "$LIB"
          R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
          R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
          R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true
          # Also build current packages whose required dependency tree needs a newer R.
          R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION"
        done
      - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
      - |
--- a/.crow/prune-stale-flat-binaries.yaml
+++ b/.crow/prune-stale-flat-binaries.yaml
@ -1,242 +0,0 @@
 # Remove flat-slot binaries recorded unable to load under some R minor, then
 # re-index.
 #
 # The list comes from `abi_flat_safety`, so it names the objects the binaries
 # themselves report as broken, not the ones a source heuristic suspects. That
 # distinction is the whole cost of the operation: the heuristic calls 93% of
 # compiled packages risky and would delete roughly 3000 objects per slot to fix
 # about 230.
 #
 # `upload_single_binary()` has written a non-portable package only to the
 # per-minor slot since v5.3.0, so this is a one-time cleanup of what was
 # uploaded before that. The flat slot is what every client that cannot be
 # routed by R minor receives, and such a binary was built under whichever R was
 # primary at the time, so it dies at load under any minor that dropped a symbol
 # it uses:
 #
 #   base64enc.so: undefined symbol: SETLENGTH
 #
 # Deleting the flat object does not create a hole. The source fallback
 # republishes the CRAN source under the same key, so the package stays
 # installable everywhere and compiles against the client's own R, while routed
 # clients keep getting the per-minor binary.
 #
 # Dry run unless PRUNE_APPLY is 'yes'. Start with an explicit package list:
 #
 #   crow pipeline create devxy/build-cran-binaries \
 #     --var prune_flat=ubuntu-2604-amd64 \
 #     --var PRUNE_PACKAGES="base64enc lobstr" \
 #     --var PRUNE_APPLY=yes
 variables:
  # A manual pipeline creation instantiates every file in .crow/, so the default
  # must match no matrix row.
  prune_flat:
    description: "Prune target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
    options:
      - none
      - all
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
      - alpine-324-arm64
      - redhat-8-amd64
      - redhat-8-arm64
      - redhat-9-amd64
      - redhat-9-arm64
      - redhat-10-amd64
      - redhat-10-arm64
      - ubuntu-2204-amd64
      - ubuntu-2204-arm64
      - ubuntu-2404-amd64
      - ubuntu-2404-arm64
      - ubuntu-2604-amd64
      - ubuntu-2604-arm64
    default: none
  PRUNE_PACKAGES:
    description: 'Space-separated package names. Empty means every package the classifier cache calls risky.'
    default: ''
  PRUNE_APPLY:
    description: "Set to 'yes' to actually delete. Anything else is a dry run."
    options:
      - 'no'
      - 'yes'
    default: 'no'
 when:
  - event: manual
    evaluate: 'prune_flat == "all" || prune_flat == "${OS}-${ARCH}"'
 skip_clone: true
 labels:
  group: rpkgs-${ARCH}
 matrix:
  include:
    - OS: alpine-323
      ARCH: amd64
      CODENAME: alpine323
      IMG: alpine:3.23
    - OS: alpine-323
      ARCH: arm64
      CODENAME: alpine323
      IMG: alpine:3.23
    - OS: alpine-324
      ARCH: amd64
      CODENAME: alpine324
      IMG: alpine:3.24
    - OS: alpine-324
      ARCH: arm64
      CODENAME: alpine324
      IMG: alpine:3.24
    - OS: redhat-8
      ARCH: amd64
      CODENAME: rhel8
      IMG: redhat:8
    - OS: redhat-8
      ARCH: arm64
      CODENAME: rhel8
      IMG: redhat:8
    - OS: redhat-9
      ARCH: amd64
      CODENAME: rhel9
      IMG: redhat:9
    - OS: redhat-9
      ARCH: arm64
      CODENAME: rhel9
      IMG: redhat:9
    - OS: redhat-10
      ARCH: amd64
      CODENAME: rhel10
      IMG: redhat:10
    - OS: redhat-10
      ARCH: arm64
      CODENAME: rhel10
      IMG: redhat:10
    - OS: ubuntu-2204
      ARCH: amd64
      CODENAME: jammy
      IMG: ubuntu:jammy
    - OS: ubuntu-2204
      ARCH: arm64
      CODENAME: jammy
      IMG: ubuntu:jammy
    - OS: ubuntu-2404
      ARCH: amd64
      CODENAME: noble
      IMG: ubuntu:noble
    - OS: ubuntu-2404
      ARCH: arm64
      CODENAME: noble
      IMG: ubuntu:noble
    - OS: ubuntu-2604
      ARCH: amd64
      CODENAME: resolute
      IMG: ubuntu:resolute
    - OS: ubuntu-2604
      ARCH: arm64
      CODENAME: resolute
      IMG: ubuntu:resolute
 steps:
  - name: 'Prune stale flat binaries'
    image: reg.devxy.io/rpkgs/build-env-${IMG}
    pull: true
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      B2_S3_ACCESS_KEY:
        from_secret: B2_S3_ACCESS_KEY
      B2_S3_SECRET_KEY:
        from_secret: B2_S3_SECRET_KEY
      PGPASS:
        from_secret: PGPASS
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
      GIT_USER: pat-s
      R_LIBS_USER: /tmp/R-pkgs
      # Pipeline variables are substituted into the YAML, not exported, so they
      # have to be bound here to be readable as env vars in the script below.
      PRUNE_ARCH: ${ARCH}
      PRUNE_CODENAME: ${CODENAME}
      PRUNE_PACKAGES: ${PRUNE_PACKAGES}
      PRUNE_APPLY: ${PRUNE_APPLY}
    commands:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - mkdir -p /tmp/R-pkgs
      - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R
      - |
        set -eu
        APPLY=""
        if [ "$$PRUNE_APPLY" = "yes" ]; then
          APPLY="--apply"
        fi
        /opt/R/4.5.3/bin/Rscript local/prune-stale-flat-binaries.R \
          "$$PRUNE_ARCH" "$$PRUNE_CODENAME" $$APPLY $$PRUNE_PACKAGES
    backend_options:
      docker:
        resources:
          requests:
            memory: 3Gi
            cpu: 1000m
  # Deleting objects without re-indexing is worse than leaving them: the
  # published index keeps naming files that no longer exist, so every client
  # gets a 404 instead of a working source fallback. The re-index is what
  # republishes them as CRAN source, so it belongs in the same run.
  - name: 'Re-index the slot'
    image: reg.devxy.io/rpkgs/build-env-${IMG}
    pull: true
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      B2_S3_ACCESS_KEY:
        from_secret: B2_S3_ACCESS_KEY
      B2_S3_SECRET_KEY:
        from_secret: B2_S3_SECRET_KEY
      PGPASS:
        from_secret: PGPASS
      R_LIBS_USER: /tmp/R-pkgs
      PRUNE_APPLY: ${PRUNE_APPLY}
    commands:
      - |
        set -eu
        if [ "$$PRUNE_APPLY" != "yes" ]; then
          echo "Dry run: nothing was deleted, so there is nothing to re-index."
          exit 0
        fi
        /opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
        for RBIN in /opt/R/[0-9]*/bin/R; do
          RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
          /opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
        done
    backend_options:
      docker:
        resources:
          requests:
            memory: 3Gi
            cpu: 1000m
  # The edge caches indexes for ~370 days, so without this the deletion is
  # invisible to clients and they keep being handed the removed binary.
  - name: 'Purge CDN cache'
    image: reg.devxy.io/docker.io/library/alpine:3.24
    environment:
      BUNNYNET_API_KEY:
        from_secret: BUNNYNET_API_KEY
      # Same zones as reindex.yaml: the production pull zone id, the alliance
      # hostname and the staging zone, which shares the B2 origin.
      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
      PRUNE_APPLY: ${PRUNE_APPLY}
    commands:
      - |
        set -eu
        if [ "$$PRUNE_APPLY" != "yes" ]; then
          echo "Dry run: nothing changed at the origin, so no purge is needed."
          exit 0
        fi
        apk add --no-cache -q bash curl jq
        bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
--- a/.crow/rebuild-abi-gap.yaml
+++ b/.crow/rebuild-abi-gap.yaml
@ -1,214 +0,0 @@
 # Rebuild specific packages into a per-R-minor slot.
 #
 # The per-minor scheme only ever covered packages that passed through the build
 # pipeline after it existed. A package built into the flat slot before that, and
 # never updated on CRAN since, is never reclassified and never gets a per-minor
 # copy - so the union index serves its flat binary to every minor. When that
 # binary references a symbol the client's R no longer exports, the install fails
 # at dyn.load():
 #
 #   base64enc.so: undefined symbol: SETLENGTH
 #
 # base64enc calls SETLENGTH, its flat binary was built under R 4.4.3, and R 4.6
 # does not export SETLENGTH. There was no way to rebuild one package into one
 # minor from CI, only `just rebuild` from a workstation with the S3 and Postgres
 # credentials to hand. This adds that path.
 #
 # `--sensitive-only` is deliberately NOT passed. That flag gates on the source
 # classifier, which misses Rcpp: the repair therefore skipped Rcpp entirely,
 # leaving its 4.5-built flat binary as the only copy and breaking every R 4.4
 # client on the slot.
 #
 # Since bincraft v5.3.0 the destination is decided by the built binary's symbols
 # in `upload_single_binary()`, so building unconditionally is safe: a package
 # that needs a per-minor slot lands there, and one that does not lands in the
 # flat slot. Skipping before the build is what removed that decision.
 #
 # Trigger with the slot and an explicit package list:
 #
 #   crow pipeline create devxy/build-cran-binaries \
 #     --var rebuild_abi_gap=ubuntu-2604-amd64 \
 #     --var ABI_PACKAGES="base64enc@0.1-6 lobstr@1.2.1" \
 #     --var ABI_R_MINOR=4.6
 variables:
  # A manual pipeline creation instantiates every file in .crow/, so the
  # default must match no matrix row.
  rebuild_abi_gap:
    description: "Rebuild target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
    options:
      - none
      - all
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
      - alpine-324-arm64
      - redhat-8-amd64
      - redhat-8-arm64
      - redhat-9-amd64
      - redhat-9-arm64
      - redhat-10-amd64
      - redhat-10-arm64
      - ubuntu-2204-amd64
      - ubuntu-2204-arm64
      - ubuntu-2404-amd64
      - ubuntu-2404-arm64
      - ubuntu-2604-amd64
      - ubuntu-2604-arm64
    default: none
  ABI_PACKAGES:
    description: 'Space-separated package@version list, e.g. "base64enc@0.1-6 lobstr@1.2.1".'
    default: ''
  ABI_R_MINOR:
    description: 'The R minor to build into.'
    options:
      - '4.4'
      - '4.5'
      - '4.6'
    default: '4.6'
 when:
  - event: manual
    evaluate: 'rebuild_abi_gap == "all" || rebuild_abi_gap == "${OS}-${ARCH}"'
 skip_clone: true
 labels:
  group: rpkgs-${ARCH}
 matrix:
  include:
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-323
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-324
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: alpine-324
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: redhat-8
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-8
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-9
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-9
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-10
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: redhat-10
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: ubuntu-2204
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2204
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2404
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2404
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2604
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
    - OS: ubuntu-2604
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
 steps:
  - name: 'Rebuild packages into the per-minor slot'
    image: reg.devxy.io/rpkgs/build-env-${IMG}
    pull: true
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      RED_HAT_DEV_PW:
        from_secret: RED_HAT_DEV_PW
      B2_S3_ACCESS_KEY:
        from_secret: B2_S3_ACCESS_KEY
      B2_S3_SECRET_KEY:
        from_secret: B2_S3_SECRET_KEY
      PGPASS:
        from_secret: PGPASS
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
      GITHUB_PAT:
        from_secret: GITHUB_PAT
      GIT_USER: pat-s
      # Pipeline variables are substituted into the YAML, not exported, so they
      # have to be bound here to be readable as env vars in the script below.
      ABI_PACKAGES: ${ABI_PACKAGES}
      ABI_R_MINOR: ${ABI_R_MINOR}
      UVR_CACHE_DIR: /tmp/uvr/cache
      UVR_PACKAGES_DIR: /tmp/uvr/packages
      CCACHE_DIR: /tmp/ccache
      NCPUS: 2
    commands:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - mkdir -p $UVR_CACHE_DIR $UVR_PACKAGES_DIR $CCACHE_DIR
      - |
        set -eu
        if [ -z "$$ABI_PACKAGES" ]; then
          echo "ABI_PACKAGES is empty; nothing to rebuild." >&2
          exit 1
        fi
        RBIN=$(ls -d /opt/R/$${ABI_R_MINOR}*/bin 2>/dev/null | head -1)
        if [ -z "$$RBIN" ]; then
          echo "R $$ABI_R_MINOR is not installed in this image." >&2
          exit 1
        fi
        # Container-local library: this workflow mounts no shared cache volume,
        # so it cannot disturb the build pipeline's per-agent caches.
        LIB="/tmp/R-pkgs-$$ABI_R_MINOR"
        mkdir -p "$$LIB"
        export R_LIBS_USER="$$LIB"
        rm -rf "$$LIB"/00LOCK-*
        "$$RBIN/Rscript" local/install-bincraft.R
        rc=0
        for spec in $$ABI_PACKAGES; do
          pkg=$${spec%@*}
          ver=$${spec#*@}
          if [ "$$pkg" = "$$ver" ]; then
            echo "malformed spec '$$spec'; expected package@version" >&2
            rc=1
            continue
          fi
          echo "=== rebuilding $$pkg $$ver into $$ABI_R_MINOR ==="
          "$$RBIN/Rscript" local/build-one.R "$$pkg" "$$ver" || rc=1
        done
        exit $$rc
    backend_options:
      docker:
        resources:
          requests:
            memory: 5Gi
            cpu: 1000m
--- a/.crow/reindex.yaml
+++ b/.crow/reindex.yaml
@ -1,198 +0,0 @@
 # Re-index every slot without rebuilding anything.
 #
 # `weekly-rebuild-reindex` exists to run after `weekly-rebuild-missing`, so it
 # depends on that workflow and shares its gate: triggering it manually also
 # starts hours of package rebuilds. That is the wrong tool when only the index
 # needs regenerating - after a bincraft release that changes how the index is
 # written, for instance, where the objects in the bucket are already correct
 # and only `PACKAGES*` is stale.
 #
 # This workflow does the index half on its own. It installs the latest bincraft
 # release, republishes the generic and per-R-minor indexes for each slot, and
 # purges the edge. No package is built.
 #
 # Trigger with the `reindex` variable set to `all` or to a single
 # `<os>-<arch>`, e.g.
 #
 #   crow pipeline create devxy/build-cran-binaries --var reindex=all
 variables:
  # A manual pipeline creation instantiates every file in .crow/, so the
  # default must match no matrix row.
  reindex:
    description: "Re-index target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
    options:
      - none
      - all
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
      - alpine-324-arm64
      - redhat-8-amd64
      - redhat-8-arm64
      - redhat-9-amd64
      - redhat-9-arm64
      - redhat-10-amd64
      - redhat-10-arm64
      - ubuntu-2204-amd64
      - ubuntu-2204-arm64
      - ubuntu-2404-amd64
      - ubuntu-2404-arm64
      - ubuntu-2604-amd64
      - ubuntu-2604-arm64
    default: none
 when:
  - event: manual
    evaluate: 'reindex == "all" || reindex == "${OS}-${ARCH}"'
 skip_clone: true
 labels:
  group: rpkgs-${ARCH}
 matrix:
  include:
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-323
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-324
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: alpine-324
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: redhat-8
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-8
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-9
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-9
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-10
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: redhat-10
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: ubuntu-2204
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2204
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2404
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2404
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2604
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
    - OS: ubuntu-2604
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
 steps:
  - name: 'Re-index the slot'
    image: reg.devxy.io/rpkgs/build-env-${IMG}
    pull: true
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      RED_HAT_DEV_PW:
        from_secret: RED_HAT_DEV_PW
      B2_S3_ACCESS_KEY:
        from_secret: B2_S3_ACCESS_KEY
      B2_S3_SECRET_KEY:
        from_secret: B2_S3_SECRET_KEY
      # The index build reads the ABI classifier cache to decide which packages
      # may be carried from the flat slot into a per-minor one. Without the
      # password that query fails, and it fails soft, so the run still succeeds
      # while quietly using an empty risky set: exactly the state the cache was
      # added to fix. It logs one connection error per slot.
      PGPASS:
        from_secret: PGPASS
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
      # uvr resolves bincraft's dependencies through the GitHub API, including
      # the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the
      # failure is not loud: uvr warns, falls back to the registry copy of
      # cranlike, and the index write then dies on `unused argument (built =
      # built)` because upstream has no such parameter. Intermittent by nature,
      # since it depends on the rate-limit window.
      GITHUB_PAT:
        from_secret: GITHUB_PAT
      GIT_USER: pat-s
      R_LIBS_USER: /mnt/cache/R-pkgs
      R_VERSION: ${R_VERSION}
      PLATFORM: ${OS}
      ARCH: ${ARCH}
    commands:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - mkdir -p /mnt/cache/R-pkgs
      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
      # The codename is detected from the image's /etc/os-release.
      - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
      - |
        for RBIN in /opt/R/[0-9]*/bin/R; do
          RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
        done
  - name: Purge CDN cache
    image: reg.devxy.io/docker.io/library/alpine:3.24
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      BUNNYNET_API_KEY:
        from_secret: BUNNYNET_API_KEY
      # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
      # Bunny pull zones, so both must be purged after the shared origin changes.
      # The staging zone is listed too. It shares the B2 origin, so an index
      # it still holds is a stale copy of the same object, and its
      # cache_expiration_time is the same ~370 days: without a purge here it
      # serves pre-reindex indexes indefinitely and any verification run
      # against it measures the past.
      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
    commands:
      - apk add --no-cache -q bash curl jq
      # Crow carries the checkout from the re-index step into this step.
      - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
    # Runs on every row rather than on one designated slot: a cron fires only
    # its own slot's row, so gating on a named slot would leave every other
    # slot unpurged. A manual "all" run therefore purges the zone 18 times,
    # which is a cheap API call and rare.
    #
    # Run it even when the re-index above failed: the objects were still
    # replaced, and a stale edge is exactly what keeps them hidden.
    when:
      - status: [success, failure]
--- a/.crow/repair-built-stamp.yaml
+++ b/.crow/repair-built-stamp.yaml
@ -16,7 +16,7 @@
 # pipeline here gates on a variable named after itself for exactly that reason.
 #
 #   crow pipeline create --branch main \
-#     --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.23 \
+#     --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \
 #     --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries
 variables:
  # Gates this pipeline. A manual pipeline creation instantiates every file in
@ -37,8 +37,9 @@ variables:
      - ubuntu
    default: alpine
  OS_VERSION:
-    description: 'OS image tag. Must match OS (alpine: 3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).'
+    description: 'OS image tag. Must match OS (alpine: 3.22/3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).'
    options:
      - '3.22'
      - '3.23'
      - '3.24'
      - '8'
@ -47,7 +48,7 @@ variables:
      - 'jammy'
      - 'noble'
      - 'resolute'
-    default: '3.23'
+    default: '3.22'
  R_VERSION:
    description: 'R version whose stamp the slot should advertise.'
    options:
--- a/.crow/repair-unsafe-flat-objects.yaml
+++ b/.crow/repair-unsafe-flat-objects.yaml
@ -1,173 +0,0 @@
 # Rebuild every flat object recorded as unable to load under some R minor.
 #
 # The list comes from `abi_flat_safety`, written by the flat-safety backfill
 # from the binaries themselves, so it names exactly the objects that fail at
 # dyn.load() somewhere rather than everything a source heuristic suspects. The
 # heuristic calls 93% of compiled packages risky; the binaries say 4.3%.
 #
 # Each object is rebuilt under every R minor installed in the image, with no
 # `--sensitive-only` gate. Since bincraft v5.3.0 the destination is decided by
 # the built binary's symbols in `upload_single_binary()`, so a package needing a
 # per-minor slot lands there and a portable one lands in the flat slot. Gating
 # on the source classifier is what let Rcpp slip through and break every R 4.4
 # client on the slot.
 #
 #   crow pipeline create devxy/build-cran-binaries --var repair_unsafe=all
 variables:
  # A manual pipeline creation instantiates every file in .crow/, so the default
  # must match no matrix row.
  repair_unsafe:
    description: "Repair target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
    options:
      - none
      - all
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
      - alpine-324-arm64
      - redhat-8-amd64
      - redhat-8-arm64
      - redhat-9-amd64
      - redhat-9-arm64
      - redhat-10-amd64
      - redhat-10-arm64
      - ubuntu-2204-amd64
      - ubuntu-2204-arm64
      - ubuntu-2404-amd64
      - ubuntu-2404-arm64
      - ubuntu-2604-amd64
      - ubuntu-2604-arm64
    default: none
 when:
  - event: manual
    evaluate: 'repair_unsafe == "all" || repair_unsafe == "${OS}-${ARCH}"'
 skip_clone: true
 labels:
  group: rpkgs-${ARCH}
 matrix:
  include:
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-323
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-324
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: alpine-324
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: redhat-8
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-8
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-9
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-9
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-10
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: redhat-10
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: ubuntu-2204
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2204
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2404
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2404
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2604
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
    - OS: ubuntu-2604
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
 steps:
  - name: 'Rebuild objects recorded unsafe'
    image: reg.devxy.io/rpkgs/build-env-${IMG}
    pull: true
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      RED_HAT_DEV_PW:
        from_secret: RED_HAT_DEV_PW
      B2_S3_ACCESS_KEY:
        from_secret: B2_S3_ACCESS_KEY
      B2_S3_SECRET_KEY:
        from_secret: B2_S3_SECRET_KEY
      PGPASS:
        from_secret: PGPASS
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
      GITHUB_PAT:
        from_secret: GITHUB_PAT
      GIT_USER: pat-s
      R_LIBS_USER: /tmp/R-pkgs
      NCPUS: 2
      REPAIR_ARCH: ${ARCH}
    commands:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - mkdir -p /tmp/R-pkgs
      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
      - |
        set -eu
        CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))')
        /opt/R/$R_VERSION/bin/Rscript local/list-unsafe-flat-objects.R \
          "$$REPAIR_ARCH" "$$CODENAME" > /tmp/unsafe.txt
        echo "objects to rebuild: $(wc -l < /tmp/unsafe.txt)"
        rc=0
        for RBIN in /opt/R/[0-9]*/bin/R; do
          RV=$(basename "$(dirname "$(dirname "$$RBIN")")")
          RMINOR=$(echo "$$RV" | cut -d. -f1-2)
          echo "=== rebuilding under R $$RV ==="
          LIB="/tmp/R-pkgs-$$RMINOR"
          mkdir -p "$$LIB"
          R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" local/install-bincraft.R || true
          while read -r pkg ver; do
            [ -z "$$pkg" ] && continue
            R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" \
              local/build-one.R "$$pkg" "$$ver" || rc=1
          done < /tmp/unsafe.txt
        done
        exit $$rc
    backend_options:
      docker:
        resources:
          requests:
            memory: 5Gi
            cpu: 1000m
--- a/.crow/retire-slot.yaml
+++ b/.crow/retire-slot.yaml
@ -1,72 +0,0 @@
 # Delete a retired slot's objects from the bucket.
 #
 # The config half is a commit, so it runs on a workstation:
 #   just retire-slot <codename> --apply
 #
 # This half needs the B2 credentials, which only exist in CI.
 # `local/retire-slot.R` refuses any slot still listed in `local.rpkgs_slots`,
 # so this cannot delete a live slot. Both architectures go together.
 #
 # Dry run unless RETIRE_APPLY is 'yes':
 #
 #   crow pipeline create devxy/build-cran-binaries --branch main \
 #     --var retire_slot=alpine322 --var RETIRE_APPLY=yes
 variables:
  # A manual pipeline creation instantiates every file in .crow/, so the
  # default must be a value the script never accepts.
  retire_slot:
    description: "Codename of the slot to retire (e.g. alpine322), or 'none'."
    default: none
  RETIRE_APPLY:
    description: "Set to 'yes' to delete. Anything else reports and changes nothing."
    options:
      - 'no'
      - 'yes'
    default: 'no'
 when:
  - event: manual
    evaluate: 'retire_slot != "none"'
 skip_clone: true
 labels:
  group: rpkgs-amd64
 steps:
  - name: 'Delete the retired slot from the bucket'
    image: reg.devxy.io/rpkgs/build-env-alpine:3.24
    pull: true
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      B2_S3_ACCESS_KEY:
        from_secret: B2_S3_ACCESS_KEY
      B2_S3_SECRET_KEY:
        from_secret: B2_S3_SECRET_KEY
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
      R_LIBS_USER: /tmp/R-pkgs
      # Pipeline variables are substituted into the YAML, not exported, so they
      # have to be bound here to be readable as env vars in the script below.
      RETIRE_SLOT: ${retire_slot}
      RETIRE_APPLY: ${RETIRE_APPLY}
    commands:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - mkdir -p /tmp/R-pkgs
      - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R
      - |
        set -eu
        APPLY=""
        if [ "$$RETIRE_APPLY" = "yes" ]; then
          APPLY="--apply"
        fi
        /opt/R/4.5.3/bin/Rscript local/retire-slot.R "$$RETIRE_SLOT" --assets-only $$APPLY
    backend_options:
      docker:
        resources:
          requests:
            memory: 2Gi
            cpu: 1000m
--- a/.crow/trial-build-registry.yaml
+++ b/.crow/trial-build-registry.yaml
@ -38,6 +38,14 @@ labels:
 matrix:
  include:
    - OS: alpine-322
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
    - OS: alpine-322
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
--- a/.crow/verify-repo-routing.yaml
+++ b/.crow/verify-repo-routing.yaml
@ -1,146 +0,0 @@
 # Install a fixed set of packages from a published slot under every R minor the
 # image carries, through each URL form the website tells people to configure,
 # and check that each one loads.
 #
 # This is the end-to-end counterpart to the ABI bookkeeping: `abi_flat_safety`
 # records what the binaries say about themselves, this proves what a client
 # actually receives once the CDN, the per-minor indexes and the `Path:` field
 # have had their say. It runs on the build agents rather than a workstation so
 # arm64 is covered on real hardware.
 #
 #   crow pipeline create devxy/build-cran-binaries --var verify_routing=all
 variables:
  # A manual pipeline creation instantiates every file in .crow/, so the default
  # must match no matrix row.
  verify_routing:
    description: "Verify target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
    options:
      - none
      - all
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
      - alpine-324-arm64
      - redhat-8-amd64
      - redhat-8-arm64
      - redhat-9-amd64
      - redhat-9-arm64
      - redhat-10-amd64
      - redhat-10-arm64
      - ubuntu-2204-amd64
      - ubuntu-2204-arm64
      - ubuntu-2404-amd64
      - ubuntu-2404-arm64
      - ubuntu-2604-amd64
      - ubuntu-2604-arm64
    default: none
 when:
  - event: manual
    evaluate: 'verify_routing == "all" || verify_routing == "${OS}-${ARCH}"'
 skip_clone: true
 labels:
  group: rpkgs-${ARCH}
 matrix:
  include:
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-323
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-324
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: alpine-324
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: redhat-8
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-8
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-9
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-9
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-10
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: redhat-10
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: ubuntu-2204
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2204
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2404
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2404
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2604
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
    - OS: ubuntu-2604
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
 steps:
  - name: 'Verify routing serves loadable binaries'
    image: reg.devxy.io/rpkgs/build-env-${IMG}
    pull: true
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
      VERIFY_ARCH: ${ARCH}
    commands:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - |
        set -eu
        CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))')
        rc=0
        for RBIN in /opt/R/[0-9]*/bin/R; do
          RV=$(basename "$(dirname "$(dirname "$$RBIN")")")
          for FORM in arch generic flat; do
            echo "=== R $$RV via $$FORM ==="
            "$(dirname "$$RBIN")/Rscript" local/verify-repo-routing.R \
              "$$VERIFY_ARCH" "$$CODENAME" "$$FORM" || rc=1
          done
        done
        exit $$rc
    backend_options:
      docker:
        resources:
          requests:
            memory: 4Gi
            cpu: 1000m
--- a/.crow/weekly-audit-missing.yaml
+++ b/.crow/weekly-audit-missing.yaml
@ -3,18 +3,20 @@
 # Routing is preserved 1:1:
 #   - cron:   each existing `weekly-audit-missing-<os>-<arch>` cron fires only
 #             its matching matrix row (via the per-row `cron:` name filter).
-#   - manual: pick a target from the `weekly_audit_missing` dropdown;
+#   - manual: pick a target from the `weekly_audit_missing` dropdown
-#             "all" fans out every os/arch as parallel matrix workflows.
+#             ("all" = every os/arch).
 # Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64).
 variables:
  # Gates this pipeline. A manual pipeline creation instantiates every file in
  # .crow/, and a declared default is applied even when the run never passed
  # this variable, so the default must be a value that matches no matrix row.
  weekly_audit_missing:
-    description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
+    description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
    options:
      - none
      - all
      - alpine-322-amd64
      - alpine-322-arm64
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
@ -46,6 +48,14 @@ labels:
 matrix:
  include:
    - OS: alpine-322
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
    - OS: alpine-322
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
@ -137,7 +147,7 @@ steps:
      - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs
      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
-      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite
+      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
      - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")'
    backend_options:
      docker:
--- a/.crow/weekly-patch-proposals.yaml
+++ b/.crow/weekly-patch-proposals.yaml
@ -53,7 +53,7 @@ steps:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - mkdir -p /mnt/cache/R-pkgs
      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
-      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite
+      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
      - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue
      - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue
    backend_options:
--- a/.crow/weekly-rebuild-missing.yaml
+++ b/.crow/weekly-rebuild-missing.yaml
@ -4,11 +4,17 @@
 #   - cron:   each existing `weekly-rebuild-missing-<os>-<arch>` cron fires only
 #             its matching matrix rows (via the per-row `cron:` name filter),
 #             which is now all three shards of that slot.
-#   - manual: pick a target from the `weekly_rebuild_missing` dropdown;
+#   - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the
-#             "all" fans out every os/arch and shard as parallel matrix
+#             previous bare manual trigger that ran every os/arch); pick a
-#             workflows, while a single <os>-<arch> runs its three shards.
+#             single <os>-<arch> to run just one.
 # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64).
 #
 # 9 OS versions x 2 arches x 3 shards = 54 rows. Crow counts the *declared*
 # matrix against CROW_MAX_MATRIX_SIZE before any `when:` gate is applied, so a
 # single-slot manual run expands all 54 too. The server default is 50 and was
 # raised for this; `crow lint` does not check the limit, so adding an OS
 # version here is only caught when a pipeline is triggered.
 #
 # The shard picks up its own slice and re-derives what is still outstanding
 # from the bucket, so a restart resumes rather than replaying; see
 # local/rebuild-missing.R.
@ -21,10 +27,12 @@ variables:
  # .crow/, and a declared default is applied even when the run never passed
  # this variable, so the default must be a value that matches no matrix row.
  weekly_rebuild_missing:
-    description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
+    description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
    options:
      - none
      - all
      - alpine-322-amd64
      - alpine-322-arm64
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
@ -56,6 +64,42 @@ labels:
 matrix:
  include:
    - OS: alpine-322
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
      SPLIT_INTO: 3
      SPLIT_INDEX: 1
    - OS: alpine-322
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
      SPLIT_INTO: 3
      SPLIT_INDEX: 2
    - OS: alpine-322
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
      SPLIT_INTO: 3
      SPLIT_INDEX: 3
    - OS: alpine-322
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
      SPLIT_INTO: 3
      SPLIT_INDEX: 1
    - OS: alpine-322
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
      SPLIT_INTO: 3
      SPLIT_INDEX: 2
    - OS: alpine-322
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
      SPLIT_INTO: 3
      SPLIT_INDEX: 3
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
@ -388,7 +432,7 @@ steps:
      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
      - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
-      - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
+      - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2
      - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
      - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1
--- a/.crow/weekly-rebuild-reindex.yaml
+++ b/.crow/weekly-rebuild-reindex.yaml
@ -16,10 +16,12 @@ variables:
  # exactly the slots it rebuilt. A manual pipeline creation instantiates every
  # file in .crow/, so the default must match no matrix row.
  weekly_rebuild_missing:
-    description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
+    description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
    options:
      - none
      - all
      - alpine-322-amd64
      - alpine-322-arm64
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
@ -56,6 +58,14 @@ labels:
 matrix:
  include:
    - OS: alpine-322
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
    - OS: alpine-322
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.22
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
@ -135,23 +145,8 @@ steps:
        from_secret: B2_S3_ACCESS_KEY
      B2_S3_SECRET_KEY:
        from_secret: B2_S3_SECRET_KEY
      # The index build reads the ABI classifier cache to decide which packages
      # may be carried from the flat slot into a per-minor one. Without the
      # password that query fails, and it fails soft, so the run still succeeds
      # while quietly using an empty risky set: exactly the state the cache was
      # added to fix. It logs one connection error per slot.
      PGPASS:
        from_secret: PGPASS
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
      # uvr resolves bincraft's dependencies through the GitHub API, including
      # the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the
      # failure is not loud: uvr warns, falls back to the registry copy of
      # cranlike, and the index write then dies on `unused argument (built =
      # built)` because upstream has no such parameter. Intermittent by nature,
      # since it depends on the rate-limit window.
      GITHUB_PAT:
        from_secret: GITHUB_PAT
      GIT_USER: pat-s
      R_LIBS_USER: /mnt/cache/R-pkgs
      R_VERSION: ${R_VERSION}
@ -178,18 +173,15 @@ steps:
      OTEL_R_METRICS_EXPORTER: none
      BUNNYNET_API_KEY:
        from_secret: BUNNYNET_API_KEY
-      # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
+      REPO_RO_TOKEN:
-      # Bunny pull zones, so both must be purged after the shared origin changes.
+        from_secret: REPO_RO_TOKEN
-      # The staging zone is listed too. It shares the B2 origin, so an index
+      # All hostnames on the zone share this id, so one purge covers
-      # it still holds is a stale copy of the same object, and its
+      # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com.
-      # cache_expiration_time is the same ~370 days: without a purge here it
+      BUNNY_PULLZONE: '3857050'
      # serves pre-reindex indexes indefinitely and any verification run
      # against it measures the past.
      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
    commands:
-      - apk add --no-cache -q bash curl jq
+      - apk add --no-cache -q bash curl git
-      # Crow carries the checkout from the re-index step into this step.
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
-      - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
+      - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE"
    # Runs on every row rather than on one designated slot: a cron fires only
    # its own slot's row, so gating on a named slot would leave every other
    # slot unpurged. A manual "all" run therefore purges the zone 18 times,
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -24,19 +24,19 @@ repos:
        args:
          - --markdown-linebreak-ext=md
  - repo: https://github.com/DavidAnson/markdownlint-cli2
-    rev: v0.23.3
+    rev: v0.23.2
    hooks:
      - id: markdownlint-cli2
  - repo: https://github.com/rbubley/mirrors-prettier
-    rev: v3.9.10
+    rev: v3.9.6
    hooks:
      - id: prettier
  - repo: https://github.com/posit-dev/air-pre-commit
-    rev: 0.12.0
+    rev: 0.11.0
    hooks:
      - id: air-format
  - repo: https://github.com/editorconfig-checker/editorconfig-checker
-    rev: v4.0.2
+    rev: v3.11.1
    hooks:
      - id: editorconfig-checker
        exclude: ^local/patches/.*\.patch$
--- a/.terraform.lock.hcl
+++ b/.terraform.lock.hcl
@ -2,79 +2,79 @@
 # Manual edits may be lost in future updates.
 provider "registry.opentofu.org/hashicorp/http" {
-  version = "3.6.2"
+  version = "3.6.0"
  hashes = [
-    "h1:7+Ewr07oLewLfoe5K7Fhy4T5JUU4y7I7iRGTG2hrzy4=",
+    "h1:0n4RBz9zNw6TTddh5+x7E8L2+qzPXNwKhK4uoZ/DUwE=",
-    "h1:8AvJ6H30J1tgxwEoVKxe0B0Ac9XFn1gwOzidrIc+F5o=",
+    "h1:22Ob7lpzMBSqdrCvoFN5EgmhGPHPBovV/9qo0c/Cd+A=",
-    "h1:AllxtaK2m88aY+QitHui6kik/qJmBakrH5o5W7vOKI0=",
+    "h1:2IRBvmWOYrq/ooaYYn2i86jZb7iIUvlg0KlmOMfDHoQ=",
-    "h1:OimQGm1/9sIkdDCuUb8D5sxDorhektHeCQ337DHDfGk=",
+    "h1:5mucXikk4OcW3un3u94QnMx4AB4Wfih+sXeMd5QxSNk=",
-    "h1:T5zxkFXQ9To+ZysjCZu6gFqE8G1aTGh58Ol20bCk1Fc=",
+    "h1:5oU7Zm+2gAVGmxqtJ9E8uTudUkYy/DEn/y3IWphdv4k=",
-    "h1:TZlCCHsmKXSI3FyzmSgCU+TarKOHgiOft0wiYh85Y+E=",
+    "h1:5w0R4b1/VSzpqQF1tXXPr/qmaQLPVRXamOmPKWFcTk4=",
-    "h1:VgyzPdGw1U/KU8gKhefKlIhy5kQYdpoZkpNcGiTgvVs=",
+    "h1:AEVeJr8xGmwad+JUUQ833C3x5d4W+W2szF5DfwxYppw=",
-    "h1:aF+Ztfy9m0Ll6nFOtC90NxN6WIEKAmfmVFhXAtFF8CI=",
+    "h1:CPHJ+0zQbS/cX1m55Y90jIOgf1jV3ocUUnqsXAh+9Eg=",
-    "h1:bgViIoXF7kMXA7xgNikFYfv1D15rregO8I8Sq95sLGQ=",
+    "h1:JPewnGDOJudNer5+ghqwXoaJkfot3QRq9uiEYvo+JHU=",
-    "h1:d7tCAc1wr1seWpGMBx7tpRyGKOCicvozaHvn5X8ZFfM=",
+    "h1:QzbluV2vQLxsJYxjpziQCmPndIoJ/UGS4/UHH/GpwUM=",
-    "h1:dRCrpHH2UrkKs/lCqewyA00L0VfOCETVM14NBhyxHaM=",
+    "h1:TjUNbUdqweRBq/ycQ4ixpNkx5qaYwpXEOn9QCpqNZP8=",
-    "h1:eIqaNIixQTlhmr2GU1fAFH+eVYLQIfeGw7Ec+JT/bEo=",
+    "h1:XNbcODP60ajj21N/OO7af8bBg1ltIsYkq9egn7BYbiY=",
-    "h1:hrK+Jzpgivi4Jo356K3LYDoeWBV1bMqKPBEqrEHwgpA=",
+    "h1:tgrbgmX7WYQz9G9ncgu7TkpVB+RlLjJA/Rvp9KPlZH8=",
-    "h1:mrpIAC+TK3rQQz2jllAF8o2WLMKM14/WoclLgYKc4WA=",
+    "h1:vLxthX/ZWsOZ+aHKbAMqmNKqD0K5f4nJ8ppy0Ioyup0=",
-    "h1:zwE/uNBapBDpkKa5I24moawVPTanbG+eKumHRm/qINw=",
+    "h1:wZOdGBAZkY8OKEPjKz82j1HloAKOmmvtjWyTxM+I110=",
-    "zh:0cf6173a426f65a85e8baf4f9a7808dc94d0e02a82b34bc903555fda1eb45d7a",
+    "zh:0f719fa5426bc883e9fa6abf7f6498e48025edafbc29015e2f5c028f1cca3b9d",
-    "zh:2b5fa2b0487e692014fa86745a2a98d43f34f7aaea044acd3e2cee139627c9f1",
+    "zh:1b4d7dafefd6c61764b2f9ed6943ceb9a200dee3590d18747e3a5f6b20ce85e0",
-    "zh:2c9c18893ce0f34e077559632c88237d172ac6646853bf00c84a5a4ec3b6c241",
+    "zh:1d23a712984866d29f7b07028a4e99c783c71f1a5dddf08bc3d4e7da9d91a1fa",
-    "zh:316c848e892764aa10a890cf2455f52b592c6051a92ac8451955ca1d47ca1d2f",
+    "zh:257d23d58c3bb024b6bc8eb88736eaf912e934ad47c639d0c3c742bddda849a1",
-    "zh:4300e32ecaa967ca0ed56ca48f8dcd10796e70203bf6b8311e71dc43cd0d2aa1",
+    "zh:479860e1a5468f5e04013b9364c9496d7ed0804bf9a1acd8e07558d57609993d",
-    "zh:4a0a555d85a31bd257399253b502ad03816ccdba7b8fbbbd1e07a313d04d83fb",
+    "zh:4cb5e681bf599b411b27c4a2c4066a5fb2ed79aaa3a1a3cb5a30002fec062ce9",
-    "zh:6278773c31c1605c6d780fc982eacb83944e2807c8d8afe11534253b225c5aa3",
+    "zh:4fb35c3f643dae9f3670d719397a415f815a0b95f8ed7bd8a72f27a94ba78092",
-    "zh:81b73da8dbcedf54604f131e0acd7ebd36cf673276174b4b4f10af0b0f10de11",
+    "zh:59ba40825ab38db5b4a0989a2db0df35cc15d8984f898176011ba352f27d77b7",
-    "zh:887beaa6ef6521d3f5e2b1193d458575fb91855f36bc66dd26a10988127a39fb",
+    "zh:61fc1252eb88088638f4c69ea4e2171cde2e5089fa632ac1e943b13787348f73",
-    "zh:91b2260424c5f8e5968d9e6d5bdbbdb60d61ccc4ff001aa0d637427629349c54",
+    "zh:7c5d6dd5f7cbc460e95d368be35c29b4e0402069b8912dbd5d1cd7fa9acef216",
-    "zh:a5d9d7032d00a6dd844e8d0d374b5ae89e25f9a6635a90e01c5f65e9dd14a93e",
+    "zh:7f76d756240d4284642f359ad470226e5378670239aadc366ef54d9d914d4d2e",
-    "zh:bcf8b2aca448774e4701f221b74d762847be55dc312022b4aad66da3b389bf5d",
+    "zh:8133ad0814098177e0d067c816ccf1bf48bbadacd18f6f2c808c90447505723b",
-    "zh:d62a3ad37ee1fb943ad0590978f4e7ee0ac064d19554c1c05ec98e9d2bb658f7",
+    "zh:c93be06269bb728f1968f8c50506de56c887017ac1d6e4be1f925651d8437eb6",
-    "zh:e517b8b81fe88bf49868923f076515b9157771d1d3a80849cc8a73b9a19039a1",
+    "zh:ef47b78a10a82e6cf53344a6a85a94041c28286c10a70541c564d762f1cfede0",
-    "zh:fa9e0e39ef3e5ed2e6cf5886a0b569415ca3b89b9289fd6d01824ff11e470618",
+    "zh:f5796a53a74999135bd9087aff50fddda59129d09b2f9b1902ff8c0c1e047e48",
  ]
 }
 provider "registry.terraform.io/bunnyway/bunnynet" {
-  version     = "0.19.1"
+  version     = "0.17.0"
-  constraints = "~> 0.19"
+  constraints = "~> 0.17"
  hashes = [
-    "h1:02CmekB5A4NtAeptTR7waaECJl66Gf3ww7Wtyg1cy+g=",
+    "h1:+qDt35lVSK7acw6a1xHuPYrqmZEcHSmtd+6n1TxNuYw=",
-    "h1:0VWwMHkHQXusR2vdyYWN1pNvfJUPt9OWh6glgZshCck=",
+    "h1:1dCu2l4DhPBjizVAH/WwAjT1Xbo52K4PMvHoD5zUhuU=",
-    "h1:2NGfMzSWPCcyO2P2T9/lDS5MExYnaOfLuDEMiYvFBg4=",
+    "h1:Dvn46Auwuel4jqrqZXs2D7kdujNhs17LEmqhuY0k4/4=",
-    "h1:2dXsvcYEGXsIf1IHD2Rq9HzgwrNNEA4+qOzBKpbFEpw=",
+    "h1:M5eDL3m2uSEr1XATJW0foHzKl8pFhCtgKuOM24bJRwU=",
-    "h1:8R0H9DumQe0DjXTHRJXNiZUyTBDHT2vjfu/vB+WdRIw=",
+    "h1:PddaC7nM/gY4x9i3xy6TxOs9MAu2/6g58Xs/gv4DRV8=",
-    "h1:9Gi5M2iKY+Oz8wAYoJh7i0u3mySBk5pretuyzcwgJOw=",
+    "h1:QVIKiZluI+NQAKu8NpFBl3Nvyx+d81vW9btEUdIQREc=",
-    "h1:BZwoP8Ubi1iDueWXhSWb3Tlt/II4bJI0/p6/tqw4i1M=",
+    "h1:S6TnzXHsRoGYvC1vJBkDiVEc0spceksY4n6x5WN5iYw=",
-    "h1:F4yWXkyv55F+97IQi/jYBLhBLyhz4hkokVCyEEQzbM4=",
+    "h1:VcxZDWqCWMSjcUsC1K4sB6uYEoeoou+BC0ePoJXmf3A=",
-    "h1:MGHXEapStuXemfw3SikP2aLQCT+b7Ee/RrCHevSAAWA=",
+    "h1:W0y/agBVqls1cJlFGFYMu2VnqoPXFzxVHPIYe3OqfYQ=",
-    "h1:Q10nENYoX8jIsBMz1kP8OBrkMeSNmFI2CfB6/MsWKAc=",
+    "h1:XmNd5fP9a0O77ve5BMQP2vARExgIa7rYl6KvyUYXPSs=",
-    "h1:R9z1oacW8gD9hPTGTO37ngDAhAvlP33/PRhzkgXDStY=",
+    "h1:e0EFKrWSQwaa/kGhnha4DXk4T68Av8QxP84mRSdWC9M=",
-    "h1:UYLBa5/V+giaZ2YYjkbPDKMRO9eQRNcWEnpZqU2I/yU=",
+    "h1:eM+/lUiU0pNSgQKoqKPgE3xJrJ0MHIpKG+yhaGB/P0M=",
-    "h1:Wz/QDxt1Kved07fCwqMh+6jtTtwQ/xKi9mXE2StV7vw=",
+    "h1:fPWWA4T0/y7GX+tCGN23l1jODhZ3uCdR/MKgZDXYpAE=",
-    "h1:Yq0JSIW4Jir3EVO2w7C/iM76yzYKG5eYw51woX6CXJo=",
+    "h1:g+r2GVi4gVC4DuQg3PL70gW9BDskgWUzCBIMXTUq63A=",
-    "h1:e/PEmAJszFbjdfsAMJosDbkxtHwHNnByd/3nKJmE3Sg=",
+    "h1:gaZ8eALDtVHqykVDHav8004gHiMGaYR/3KwET0FUgao=",
-    "h1:lMVwei5IGLQCrPMqAmcpbXs/BjaG7k6HbjhuWtr0kIA=",
+    "h1:kbqW25eaiv4N/N/z+sxLdJZ15yh5cgnRD/q6RclPMLc=",
-    "h1:m8dyuq1sXJ9sxjZkU3ZUsjC77BWY87GiraNvxHluSYs=",
+    "h1:rGjxue3mXRyQQqpywTXC4zK//JAtf0Cz7RP+uPMMJjw=",
-    "zh:15e84e2fca2bbaec9c9f4d14d1a037d02f4d8cc1d1bdff04709d51085f950774",
+    "zh:05943fef14c2028f4722bf078aa1889229e94302f7678cc6f63adb669d8ea612",
-    "zh:1b9d0cbb7f08e7a554afb8c1e733942a77aa0c8112e8b2b7092c040794f0b113",
+    "zh:26a163930a92a7408f7bbd0130064b84df8a232b500d8c6c3989952986308539",
-    "zh:28768988a8ec70e5392ca93375717b83a9aa2eeaa21be2f53c28550273b97065",
+    "zh:41305feaaade55391447521ec309f3c038b631ca542907ad95132fab71a7e116",
-    "zh:2f357c1d2c013627e4e785c350fc72ccc0bfbebaa6062e536ec7eb9cfecfc913",
+    "zh:606919a930f0299948504adbdcd0f239a8af5c418f85741c48f8add370a3d038",
-    "zh:4a41bab7fba5415b8c422f5c03d6fb054ae210e1eb180fab4081d4fa976080b5",
+    "zh:66963d5b445639511939fc508513fd31da3ee1d4ee1a565ee396c9532897a349",
-    "zh:6c10fad0bd03bbf053b97ad51f0a24e9a0c43a7fc10727033a39eec90ed0a51e",
+    "zh:6c981ec0c8545556395c43e2511861ab65ee9ecf2a960480e7889c3af0d23af3",
-    "zh:795980ac64b0f9bda521f829187eb7b296baa682b7e8d409126f85e6a6f9bcc0",
+    "zh:7334a1bdb726ce1f1bf0a3155f30f84f65206980c229c832ff5f0b0718c44e0b",
-    "zh:841a7e234852ecd516e2dc52c7d31025a704242aeafee15eb36bf392e62f918f",
+    "zh:75f6c86bf74511e605423332d113711c76c8028361a32282fb3359d6c7ecae9e",
    "zh:7aebb1a01cfe8be54903853202ae06eba14ad99c37d230ed93ce7d6633e05e9b",
    "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f",
-    "zh:8c876a0a26b249151f0918ee836e24660513c2345c23a06e0f54a28c7c6966b7",
+    "zh:9041d0e20c9ceea532de6eebf5cb3a27dad0bb49d3f5b5154be2a08d68fbbf1f",
-    "zh:906a791496cf11dcfdfe43a4167414cb473e548f0bd86d4b8a6a95450b523133",
+    "zh:a6bbf65431a02be4df0ebb1cbe01185ad357ff6e33c01bd0558f59bed90c8f36",
-    "zh:9431757bf5b7d055ecc1d3c8465216d5e7a648b75e0e35b136ac8fc25382e97d",
+    "zh:c6d075a31096f080c388dfe46036f451c0cc114c3311a4f46ab8dbe1938a202f",
-    "zh:967fd248df86fc246536727e996a7334711f4c6fe9e3d01ec9f03abe621a9783",
+    "zh:dd8703f7b55b8bc8e10f8718bea889781100b18e932b04898995b63178c3d36e",
-    "zh:a6710fbd30f8a4844eb025d616057e78bbb255a7d05cc73876dae55a5d6b0d06",
+    "zh:dd92a5cd4e133a4000e7e5bc8cce876ae0ed803543cedd2f3d590661ba244d04",
-    "zh:b5f260899555430c109da35d2c483999d61b127beda6c2a7b39eb67f27d754c8",
+    "zh:e024fdf121bebc48c1e6debea344c6d4f174117f3ae605fca6e13b9705d92d22",
-    "zh:d24882eb63c6452ed27666e896224b1d866ad00645d6ecc53de84d1687d27fc4",
+    "zh:ee0e80c31b438e35fa1608f6a2f5824d2806db1e5e8b9f7a90986585c7bcb895",
-    "zh:e573eb0c99ea7195ec1ed3da728caddacc68a126d82679fb490438c781382f4d",
+    "zh:fc2d4b705411b48f8c045981f9368a3ea2f74969dd6302008c31ff0bedd51f0a",
    "zh:fcf0a66c0c0b62af505c5f0f8fe59bca77d6085a0b07763be33420101183702e",
  ]
 }
--- a/Show more
+++ b/Show more