diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 7600876..73bfff7 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -3,15 +3,15 @@ # Routing is preserved 1:1: # - cron: each existing `process-cran-updates--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `process_cran_updates` dropdown -# ("all" = every os/arch). +# - manual: pick a target from the `process_cran_updates` dropdown; +# "all" fans out every os/arch as parallel matrix workflows. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. process_cran_updates: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all @@ -203,6 +203,7 @@ steps: - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi @@ -218,6 +219,7 @@ steps: LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true + R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 3a9d98f..428aa04 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -3,15 +3,15 @@ # Routing is preserved 1:1: # - cron: each existing `weekly-audit-missing--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `weekly_audit_missing` dropdown -# ("all" = every os/arch). +# - manual: pick a target from the `weekly_audit_missing` dropdown; +# "all" fans out every os/arch as parallel matrix workflows. # Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64). variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. weekly_audit_missing: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index c86fe5a..9639792 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -1,18 +1,27 @@ # Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). -# One matrix row per OS/arch replaces the former per-platform files. +# Three matrix rows per OS/arch, one per shard of that slot's rebuild list. # Routing is preserved 1:1: # - cron: each existing `weekly-rebuild-missing--` cron fires only -# its matching matrix row (via the per-row `cron:` name filter). -# - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the -# previous bare manual trigger that ran every os/arch); pick a -# single - to run just one. +# its matching matrix rows (via the per-row `cron:` name filter), +# which is now all three shards of that slot. +# - manual: pick a target from the `weekly_rebuild_missing` dropdown; +# "all" fans out every os/arch and shard as parallel matrix +# workflows, while a single - runs its three shards. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). +# +# The shard picks up its own slice and re-derives what is still outstanding +# from the bucket, so a restart resumes rather than replaying; see +# local/rebuild-missing.R. +# +# Re-indexing and the CDN purge deliberately do NOT live here. Three shards +# writing one slot's PACKAGES concurrently would race, so they moved to +# .crow/weekly-rebuild-reindex.yaml, which depends on this pipeline. variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all @@ -53,74 +62,326 @@ matrix: ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-8 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-9 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-9 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-10 ARCH: amd64 R_VERSION: 4.5.3 IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-10 ARCH: arm64 R_VERSION: 4.5.3 IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2204 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2204 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2404 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2404 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2604 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2604 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 steps: - name: 'Rebuild missing binaries' @@ -153,6 +414,12 @@ steps: PLATFORM: ${OS} ARCH: ${ARCH} NCPUS: 2 + SPLIT_INTO: ${SPLIT_INTO} + SPLIT_INDEX: ${SPLIT_INDEX} + # Wall clock after which the shard stops cleanly instead of having to be + # killed. A kill matches neither `success` nor `failure`, so it would skip + # the dependent re-index and leave rebuilt binaries behind a stale edge. + REBUILD_BUDGET_HOURS: 20 commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -162,18 +429,7 @@ steps: - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - # A rebuild replaces objects in place, so the slot's index still advertises - # the old MD5 and, for anything that had been served from source, no Built - # stamp. Re-index here rather than waiting for the next process-updates - # run, or the rebuilt binaries stay invisible to clients until then. - # The codename is detected from the image's /etc/os-release. - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 backend_options: docker: resources: @@ -183,25 +439,3 @@ steps: limits: memory: 18Gi cpu: 3000m - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - # All hostnames on the zone share this id, so one purge covers - # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. - BUNNY_PULLZONE: '3857050' - commands: - - apk add --no-cache -q bash curl git - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" - # A rebuild that died part-way still replaced objects, and those are exactly - # the ones a stale edge would keep hiding, so purge either way. - when: - - status: [success, failure] diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml new file mode 100644 index 0000000..2e0f2a0 --- /dev/null +++ b/.crow/weekly-rebuild-reindex.yaml @@ -0,0 +1,191 @@ +# Re-index and purge after weekly-rebuild-missing. +# +# weekly-rebuild-missing runs three shards per slot. Each of them replaces +# objects in place, so the slot's index still advertises the old MD5 and, for +# anything that had been served from source, no Built stamp. Re-indexing from +# inside a shard would mean three concurrent `upload_package_index()` calls on +# one prefix: `cranlike::update_PACKAGES()` lists the live bucket, so an early +# lister that uploads last publishes an index missing its siblings' work. +# +# So it happens exactly once per slot, here, after every shard has finished. +# `runs_on: [success, failure]` keeps that true when a shard fails; only an +# explicit cancel skips it, and this pipeline can then be triggered on its own. + +variables: + # Mirrors the gate on weekly-rebuild-missing so a manual run re-indexes + # exactly the slots it rebuilt. A manual pipeline creation instantiates every + # file in .crow/, so the default must match no matrix row. + weekly_rebuild_missing: + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-rebuild-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' + +depends_on: + - weekly-rebuild-missing + +runs_on: [success, failure] + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + PLATFORM: ${OS} + ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + commands: + - apk add --no-cache -q bash curl jq + # Crow carries the checkout from the re-index step into this step. + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES + # Runs on every row rather than on one designated slot: a cron fires only + # its own slot's row, so gating on a named slot would leave every other + # slot unpurged. A manual "all" run therefore purges the zone 18 times, + # which is a cheap API call and rare. + # + # Run it even when the re-index above failed: the objects were still + # replaced, and a stale edge is exactly what keeps them hidden. + when: + - status: [success, failure] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82461cb..2bf38b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.11.1 + rev: v3.11.2 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index aca8ce1..9591b83 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,79 +2,79 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/http" { - version = "3.6.0" + version = "3.6.1" hashes = [ - "h1:0n4RBz9zNw6TTddh5+x7E8L2+qzPXNwKhK4uoZ/DUwE=", - "h1:22Ob7lpzMBSqdrCvoFN5EgmhGPHPBovV/9qo0c/Cd+A=", - "h1:2IRBvmWOYrq/ooaYYn2i86jZb7iIUvlg0KlmOMfDHoQ=", - "h1:5mucXikk4OcW3un3u94QnMx4AB4Wfih+sXeMd5QxSNk=", - "h1:5oU7Zm+2gAVGmxqtJ9E8uTudUkYy/DEn/y3IWphdv4k=", - "h1:5w0R4b1/VSzpqQF1tXXPr/qmaQLPVRXamOmPKWFcTk4=", - "h1:AEVeJr8xGmwad+JUUQ833C3x5d4W+W2szF5DfwxYppw=", - "h1:CPHJ+0zQbS/cX1m55Y90jIOgf1jV3ocUUnqsXAh+9Eg=", - "h1:JPewnGDOJudNer5+ghqwXoaJkfot3QRq9uiEYvo+JHU=", - "h1:QzbluV2vQLxsJYxjpziQCmPndIoJ/UGS4/UHH/GpwUM=", - "h1:TjUNbUdqweRBq/ycQ4ixpNkx5qaYwpXEOn9QCpqNZP8=", - "h1:XNbcODP60ajj21N/OO7af8bBg1ltIsYkq9egn7BYbiY=", - "h1:tgrbgmX7WYQz9G9ncgu7TkpVB+RlLjJA/Rvp9KPlZH8=", - "h1:vLxthX/ZWsOZ+aHKbAMqmNKqD0K5f4nJ8ppy0Ioyup0=", - "h1:wZOdGBAZkY8OKEPjKz82j1HloAKOmmvtjWyTxM+I110=", - "zh:0f719fa5426bc883e9fa6abf7f6498e48025edafbc29015e2f5c028f1cca3b9d", - "zh:1b4d7dafefd6c61764b2f9ed6943ceb9a200dee3590d18747e3a5f6b20ce85e0", - "zh:1d23a712984866d29f7b07028a4e99c783c71f1a5dddf08bc3d4e7da9d91a1fa", - "zh:257d23d58c3bb024b6bc8eb88736eaf912e934ad47c639d0c3c742bddda849a1", - "zh:479860e1a5468f5e04013b9364c9496d7ed0804bf9a1acd8e07558d57609993d", - "zh:4cb5e681bf599b411b27c4a2c4066a5fb2ed79aaa3a1a3cb5a30002fec062ce9", - "zh:4fb35c3f643dae9f3670d719397a415f815a0b95f8ed7bd8a72f27a94ba78092", - "zh:59ba40825ab38db5b4a0989a2db0df35cc15d8984f898176011ba352f27d77b7", - "zh:61fc1252eb88088638f4c69ea4e2171cde2e5089fa632ac1e943b13787348f73", - "zh:7c5d6dd5f7cbc460e95d368be35c29b4e0402069b8912dbd5d1cd7fa9acef216", - "zh:7f76d756240d4284642f359ad470226e5378670239aadc366ef54d9d914d4d2e", - "zh:8133ad0814098177e0d067c816ccf1bf48bbadacd18f6f2c808c90447505723b", - "zh:c93be06269bb728f1968f8c50506de56c887017ac1d6e4be1f925651d8437eb6", - "zh:ef47b78a10a82e6cf53344a6a85a94041c28286c10a70541c564d762f1cfede0", - "zh:f5796a53a74999135bd9087aff50fddda59129d09b2f9b1902ff8c0c1e047e48", + "h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=", + "h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=", + "h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=", + "h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=", + "h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=", + "h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=", + "h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=", + "h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=", + "h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=", + "h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=", + "h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=", + "h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=", + "h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=", + "h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=", + "h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=", + "zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9", + "zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6", + "zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab", + "zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b", + "zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c", + "zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4", + "zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502", + "zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69", + "zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6", + "zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279", + "zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6", + "zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7", + "zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df", + "zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621", + "zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424", ] } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.17.0" - constraints = "~> 0.17" + version = "0.18.2" + constraints = "~> 0.18" hashes = [ - "h1:+qDt35lVSK7acw6a1xHuPYrqmZEcHSmtd+6n1TxNuYw=", - "h1:1dCu2l4DhPBjizVAH/WwAjT1Xbo52K4PMvHoD5zUhuU=", - "h1:Dvn46Auwuel4jqrqZXs2D7kdujNhs17LEmqhuY0k4/4=", - "h1:M5eDL3m2uSEr1XATJW0foHzKl8pFhCtgKuOM24bJRwU=", - "h1:PddaC7nM/gY4x9i3xy6TxOs9MAu2/6g58Xs/gv4DRV8=", - "h1:QVIKiZluI+NQAKu8NpFBl3Nvyx+d81vW9btEUdIQREc=", - "h1:S6TnzXHsRoGYvC1vJBkDiVEc0spceksY4n6x5WN5iYw=", - "h1:VcxZDWqCWMSjcUsC1K4sB6uYEoeoou+BC0ePoJXmf3A=", - "h1:W0y/agBVqls1cJlFGFYMu2VnqoPXFzxVHPIYe3OqfYQ=", - "h1:XmNd5fP9a0O77ve5BMQP2vARExgIa7rYl6KvyUYXPSs=", - "h1:e0EFKrWSQwaa/kGhnha4DXk4T68Av8QxP84mRSdWC9M=", - "h1:eM+/lUiU0pNSgQKoqKPgE3xJrJ0MHIpKG+yhaGB/P0M=", - "h1:fPWWA4T0/y7GX+tCGN23l1jODhZ3uCdR/MKgZDXYpAE=", - "h1:g+r2GVi4gVC4DuQg3PL70gW9BDskgWUzCBIMXTUq63A=", - "h1:gaZ8eALDtVHqykVDHav8004gHiMGaYR/3KwET0FUgao=", - "h1:kbqW25eaiv4N/N/z+sxLdJZ15yh5cgnRD/q6RclPMLc=", - "h1:rGjxue3mXRyQQqpywTXC4zK//JAtf0Cz7RP+uPMMJjw=", - "zh:05943fef14c2028f4722bf078aa1889229e94302f7678cc6f63adb669d8ea612", - "zh:26a163930a92a7408f7bbd0130064b84df8a232b500d8c6c3989952986308539", - "zh:41305feaaade55391447521ec309f3c038b631ca542907ad95132fab71a7e116", - "zh:606919a930f0299948504adbdcd0f239a8af5c418f85741c48f8add370a3d038", - "zh:66963d5b445639511939fc508513fd31da3ee1d4ee1a565ee396c9532897a349", - "zh:6c981ec0c8545556395c43e2511861ab65ee9ecf2a960480e7889c3af0d23af3", - "zh:7334a1bdb726ce1f1bf0a3155f30f84f65206980c229c832ff5f0b0718c44e0b", - "zh:75f6c86bf74511e605423332d113711c76c8028361a32282fb3359d6c7ecae9e", - "zh:7aebb1a01cfe8be54903853202ae06eba14ad99c37d230ed93ce7d6633e05e9b", + "h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=", + "h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=", + "h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=", + "h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=", + "h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=", + "h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=", + "h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=", + "h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=", + "h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=", + "h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=", + "h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=", + "h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=", + "h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=", + "h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=", + "h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=", + "h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=", + "h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=", + "zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c", + "zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178", + "zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd", + "zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef", + "zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738", + "zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc", + "zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9041d0e20c9ceea532de6eebf5cb3a27dad0bb49d3f5b5154be2a08d68fbbf1f", - "zh:a6bbf65431a02be4df0ebb1cbe01185ad357ff6e33c01bd0558f59bed90c8f36", - "zh:c6d075a31096f080c388dfe46036f451c0cc114c3311a4f46ab8dbe1938a202f", - "zh:dd8703f7b55b8bc8e10f8718bea889781100b18e932b04898995b63178c3d36e", - "zh:dd92a5cd4e133a4000e7e5bc8cce876ae0ed803543cedd2f3d590661ba244d04", - "zh:e024fdf121bebc48c1e6debea344c6d4f174117f3ae605fca6e13b9705d92d22", - "zh:ee0e80c31b438e35fa1608f6a2f5824d2806db1e5e8b9f7a90986585c7bcb895", - "zh:fc2d4b705411b48f8c045981f9368a3ea2f74969dd6302008c31ff0bedd51f0a", + "zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef", + "zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3", + "zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb", + "zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e", + "zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11", + "zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7", + "zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9", + "zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1", + "zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff", + "zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339", ] } diff --git a/cdn.tf b/cdn.tf index 5e8899d..96877b2 100644 --- a/cdn.tf +++ b/cdn.tf @@ -32,7 +32,7 @@ # cache_stale = ["offline", "updating"] # use_background_update = true - # block_ips = var.cdn_block_ips +# block_ips = var.cdn_block_ips # # 50 TB # limit_bandwidth = 50000000000000 @@ -82,7 +82,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { cache_expiration_time = 31919000 websockets_enabled = false - errorpage_whitelabel = true + errorpage_whitelabel = true origin { type = "OriginUrl" @@ -147,6 +147,65 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } +# Alliance SwissPass historically used a separate, manually configured pull +# zone. Adopt it so both public repositories use the same B2 origin, middleware +# release and cache behavior. +import { + to = bunnynet_pullzone.cran_allianceswisspass + id = "3265648" +} + +resource "bunnynet_pullzone" "cran_allianceswisspass" { + name = "cran-allianceswisspass" + + cache_errors = false + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + + origin { + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router.id + } + + routing { + filters = [ + "scripting", + ] + } + + s3_auth_enabled = true + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" + + cache_enabled = true + request_coalescing_enabled = true + block_post_requests = true + cache_vary_headers = ["User-Agent"] + + limit_requests = 5000 + limit_connections = 1000 + + safehop_enabled = true + add_canonical_header = true + cache_stale = ["offline", "updating"] + block_ips = var.cdn_block_ips + + # 50 TB + limit_bandwidth = 50000000000000 + + block_root_path = true +} + +resource "bunnynet_pullzone_hostname" "cran_allianceswisspass" { + pullzone = bunnynet_pullzone.cran_allianceswisspass.id + name = "cran.allianceswisspass.devxy.io" + force_ssl = true + tls_enabled = true +} + # resource "bunnynet_storage_zone" "devxy-r-binaries" { # name = "devxy-r-binaries-storage" # region = "DE" diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 553185d..9493f8a 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -117,6 +117,13 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.status, 200); }); + await t.step('serves an archived binary when it exists', async () => { + const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`; + const res = await probe(path, UA_R45_MUSL); + assertEquals(res.status, 200); + assertEquals(res.location, null); + }); + await t.step('does not redirect a path already under a minor', async () => { const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); assertEquals(res.location, null); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index 9cd410b..9278d4a 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -27,6 +27,7 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; const CRAN_ORIGIN = 'https://cran.r-project.org'; +const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']); /** Slots ("/", comma separated) whose per-minor index is a union. */ const UNION_SLOTS = new Set( @@ -47,6 +48,10 @@ const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; +/** A binary archive URL whose upstream source counterpart CRAN can serve. */ +const ARCHIVE_TARBALL_REGEX = + /^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/; + const MACOS_BIN_REGEX = /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; @@ -85,6 +90,10 @@ function redirectTo(location: string, status = 302): Response { }); } +function publicCdnOrigin(url: URL): string { + return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; +} + function extractRMinor(userAgent: string): string | null { for (const regex of R_MINOR_REGEXES) { const match = userAgent.match(regex); @@ -181,15 +190,14 @@ BunnySDK.net.http const url = new URL(ctx.request.url); const path = normalizePathname(url.pathname); const userAgent = ctx.request.headers.get('User-Agent') || ''; + const publicOrigin = publicCdnOrigin(url); // macOS clients are served from CRAN's own binary tree. const srcContrib = path.match(SRC_CONTRIB_REGEX); if (srcContrib && /darwin/.test(userAgent)) { const mac = parseMacUserAgent(userAgent); if (mac) { - return Promise.resolve( - redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`), - ); + return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`)); } } @@ -213,7 +221,7 @@ BunnySDK.net.http if (target === path) { return Promise.resolve(ctx.request); } - return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`)); + return Promise.resolve(redirectTo(`${publicOrigin}${target}`)); } // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. @@ -224,12 +232,27 @@ BunnySDK.net.http } const rest = srcContrib ? srcContrib[1] : ''; - return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`)); + return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); } return Promise.resolve(ctx.request); }) - .onOriginResponse((ctx) => { + .onOriginResponse(async (ctx) => { + const path = normalizePathname(new URL(ctx.request.url).pathname); + const archive = path.match(ARCHIVE_TARBALL_REGEX); + + // Binary archives can be incomplete when an older build never succeeded. + // Preserve renv/remotes version restores by falling back to CRAN's source + // package only for an absent archived tarball. A requested version can be + // either archived upstream or still current, so probe the archive first. + // Other 404s remain visible. + if (ctx.response.status === 404 && archive) { + const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`; + const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' }); + const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`; + return redirectTo(sourceUrl); + } + ctx.response.headers.append('X-Via', 'MyMiddleware'); return Promise.resolve(ctx.response); }); diff --git a/local/rebuild-missing-helpers.R b/local/rebuild-missing-helpers.R new file mode 100644 index 0000000..16a588e --- /dev/null +++ b/local/rebuild-missing-helpers.R @@ -0,0 +1,87 @@ +# Pure helpers for local/rebuild-missing.R, kept separate so local/tests can +# source them without executing a rebuild. + +# Interleaved slice of the rebuild list. +# +# The list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, +# rstan*), so contiguous thirds would be badly unbalanced. Interleaving also +# makes each shard's progress counter representative of the slot as a whole. +shard_slice <- function(pkgs, split_into, split_index) { + split_into <- as.integer(split_into) + split_index <- as.integer(split_index) + if (is.na(split_into) || is.na(split_index)) { + stop("shard_slice(): split_into and split_index must be integers") + } + if (split_into < 1L || split_index < 1L || split_index > split_into) { + stop(sprintf( + "shard_slice(): need 1 <= split_index <= split_into, got %s of %s", + split_index, + split_into + )) + } + # seq() errors on a descending range, which is what an empty list or a shard + # index past the end would produce. + if (length(pkgs) < split_index) { + return(pkgs[0L]) + } + pkgs[seq.int(split_index, length(pkgs), by = split_into)] +} + +# Packages that still need building, decided from the bucket rather than from +# remembered progress. +# +# This is bincraft's `check_s3_root_package()` evaluated in bulk: an object +# whose ETag equals CRAN's published MD5sum is byte-identical to CRAN's source, +# so the build that was supposed to replace it has not happened yet. +# +# `etag_by_file` named by `_.tar.gz`, values are unquoted ETags +# `cran_version` named by package +# `cran_md5` named by `_` +# +# Unknown always means "already a binary", never "rebuild it", so an unreadable +# CRAN index or a multipart ETag can never mass-schedule work. +outstanding_packages <- function(pkgs, etag_by_file, cran_version, cran_md5) { + if (length(pkgs) == 0L) { + return(pkgs) + } + + # An empty table indexes to zero length rather than to NA, which would + # recycle the whole result away and silently report "nothing to build". + lookup <- function(table, key) { + if (length(table) == 0L) { + return(rep(NA_character_, length(key))) + } + unname(as.character(table[key])) + } + + version <- lookup(cran_version, pkgs) + file <- sprintf("%s_%s.tar.gz", pkgs, version) + etag <- lookup(etag_by_file, file) + md5 <- lookup(cran_md5, paste(pkgs, version, sep = "_")) + + # No CRAN version means the package cannot be resolved to a tarball at all; + # leave it in and let bincraft report why. + unresolved <- is.na(version) + # No object at the key: never built, so it is outstanding by definition. + absent <- !unresolved & is.na(etag) + # A multipart upload carries a compound ETag rather than an MD5. + unknown <- !is.na(etag) & grepl("-", etag, fixed = TRUE) + + is_source <- !unresolved & + !is.na(etag) & + !unknown & + !is.na(md5) & + etag == md5 + + pkgs[unresolved | absent | is_source] +} + +parse_rebuild_args <- function(args) { + pos <- args[!startsWith(args, "--")] + budget <- as.numeric(pos[3L]) + list( + split_into = as.integer(pos[1L]), + split_index = as.integer(pos[2L]), + budget_hours = if (is.na(budget)) 20 else budget + ) +} diff --git a/local/rebuild-missing.R b/local/rebuild-missing.R new file mode 100644 index 0000000..369024e --- /dev/null +++ b/local/rebuild-missing.R @@ -0,0 +1,194 @@ +### Rebuild one shard of a slot's missing-binary list. +# +# Usage: Rscript local/rebuild-missing.R [budget_hours] +# +# The list itself comes from local/fetch-rebuild-packages-from-issue.R, which +# writes $REBUILD_PKG_LIST (default /tmp/rebuild_pkgs.txt). +# +# Two properties matter here and are the reason this is a script rather than an +# `R -q -e` argument in the pipeline: +# +# * it is restartable. The outstanding set is re-derived from the bucket on +# every start, so a shard that died resumes where it stopped without any +# progress file, and without replaying thousands of per-package HEADs. +# * it terminates. A wall-clock budget stops the loop cleanly instead of the +# run having to be killed, which is what previously skipped the re-index and +# CDN purge and left rebuilt binaries hidden behind stale edge copies. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +source(file.path("local", "rebuild-missing-helpers.R")) + +args <- parse_rebuild_args(commandArgs(trailingOnly = TRUE)) +if (is.na(args$split_into) || is.na(args$split_index)) { + stop("usage: rebuild-missing.R [budget_hours]") +} + +list_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") +pkgs <- if (file.exists(list_file)) readLines(list_file) else character(0) +pkgs <- pkgs[nzchar(pkgs)] +if (length(pkgs) == 0L) { + cat("Nothing to rebuild\n") + q("no") +} + +excluded <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] +pkgs <- setdiff(pkgs, excluded) + +mine <- shard_slice(pkgs, args$split_into, args$split_index) +cat(sprintf( + "Shard %s/%s: %s of %s listed packages\n", + args$split_index, + args$split_into, + length(mine), + length(pkgs) +)) + +### Resume: ask the bucket what is still outstanding + +codename <- bincraft::set_codename(NULL) +local_machine <- Sys.info()[["machine"]] +arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" +slot_dir <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", + arch, + codename +) + +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) + +# One paginated listing instead of a HEAD per package. Not recursed: the +# rebuild passes no `is_r_minor_sensitive`, so it only ever targets the flat +# path, and the resume filter matches that scope deliberately. +info <- tryCatch(s3fs::s3_dir_info(slot_dir), error = function(e) NULL) +etag_by_file <- if (is.null(info) || nrow(info) == 0L) { + cat(sprintf( + "WARNING: could not list %s; building the whole shard\n", + slot_dir + )) + stats::setNames(character(), character()) +} else { + stats::setNames( + gsub('^"|"$', "", as.character(info$etag)), + basename(as.character(info$uri)) + ) +} + +cran <- tryCatch( + { + con <- gzcon(url( + "https://cloud.r-project.org/src/contrib/PACKAGES.gz", + open = "rb" + )) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "MD5sum")) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read CRAN's index (%s)\n", + conditionMessage(e) + )) + NULL + } +) +cran_version <- stats::setNames(character(), character()) +cran_md5 <- stats::setNames(character(), character()) +if (!is.null(cran)) { + cran_version <- stats::setNames( + as.character(cran[, "Version"]), + as.character(cran[, "Package"]) + ) + keep <- !is.na(cran[, "MD5sum"]) + cran_md5 <- stats::setNames( + as.character(cran[keep, "MD5sum"]), + paste(cran[keep, "Package"], cran[keep, "Version"], sep = "_") + ) +} + +before <- length(mine) +mine <- outstanding_packages(mine, etag_by_file, cran_version, cran_md5) +cat(sprintf( + "Resume: %s of %s already carry a binary; %s outstanding\n", + before - length(mine), + before, + length(mine) +)) + +if (length(mine) == 0L) { + cat("Nothing outstanding for this shard\n") + q("no") +} + +### Build + +options( + crayon.enabled = TRUE, + Ncpus = as.integer(Sys.getenv("NCPUS", "2")), + future.globals.onReference = NULL +) + +started <- Sys.time() +n <- length(mine) +completed <- 0L +for (i in seq_along(mine)) { + elapsed <- as.numeric(difftime(Sys.time(), started, units = "hours")) + if (elapsed > args$budget_hours) { + cat(sprintf( + "Budget of %sh reached after %d/%d packages; stopping cleanly. The next run resumes from the bucket.\n", + args$budget_hours, + completed, + n + )) + break + } + + x <- mine[i] + cat(sprintf("[%d/%d] %s\n", i, n, x)) + tryCatch( + bincraft::build_binary_package( + x, + tag_limit = 1L, + patches = "local/patches", + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + upload = TRUE, + store_build_metadata = TRUE + ), + error = function(e) { + cat(sprintf("ERROR building %s - %s\n", x, conditionMessage(e))) + } + ) + completed <- completed + 1L +} + +cat(sprintf( + "Shard %s/%s finished: %d/%d packages processed in %.1fh\n", + args$split_index, + args$split_into, + completed, + n, + as.numeric(difftime(Sys.time(), started, units = "hours")) +)) diff --git a/local/tests/test-rebuild-missing.R b/local/tests/test-rebuild-missing.R new file mode 100644 index 0000000..a5e70b9 --- /dev/null +++ b/local/tests/test-rebuild-missing.R @@ -0,0 +1,94 @@ +source(file.path("..", "rebuild-missing-helpers.R")) + +test_that("shard_slice partitions the list without gaps or overlap", { + pkgs <- letters[1:10] + parts <- lapply(1:3, function(i) shard_slice(pkgs, 3, i)) + + expect_identical(parts[[1]], c("a", "d", "g", "j")) + expect_identical(parts[[2]], c("b", "e", "h")) + expect_identical(parts[[3]], c("c", "f", "i")) + + expect_identical(sort(unlist(parts)), sort(pkgs)) + expect_identical(anyDuplicated(unlist(parts)), 0L) +}) + +test_that("shard_slice is deterministic and survives short lists", { + expect_identical( + shard_slice(letters[1:10], 3, 2), + shard_slice(letters[1:10], 3, 2) + ) + expect_identical(shard_slice(character(0), 3, 1), character(0)) + # more shards than packages: the tail shards get nothing rather than erroring + expect_identical(shard_slice(c("a"), 3, 1), "a") + expect_identical(shard_slice(c("a"), 3, 2), character(0)) +}) + +test_that("shard_slice rejects an out-of-range index", { + expect_error(shard_slice(letters, 3, 4), "split_index") + expect_error(shard_slice(letters, 3, 0), "split_index") +}) + +test_that("outstanding_packages keeps source fallbacks and drops real binaries", { + cran_version <- c(httr = "1.4.8", R6 = "2.6.1", curl = "7.1.0") + cran_md5 <- c( + httr_1.4.8 = "8756015b94a9cff6f410ca4de8557f12", + R6_2.6.1 = "f01b1787f12797c29194d63c9afd5d70", + curl_7.1.0 = "8af2ccbf5d85dc18866f45f1f26f348d" + ) + etag <- c( + # byte-identical to CRAN: the build never happened + "httr_1.4.8.tar.gz" = "8756015b94a9cff6f410ca4de8557f12", + # a real binary was published + "R6_2.6.1.tar.gz" = "9d6087ee9adda3f0a3b8067cfc652c05" + # curl has no object at all + ) + + out <- outstanding_packages( + c("httr", "R6", "curl"), + etag, + cran_version, + cran_md5 + ) + expect_identical(out, c("httr", "curl")) +}) + +test_that("outstanding_packages treats unknowns as already built", { + cran_version <- c(a = "1.0", b = "1.0") + cran_md5 <- c(a_1.0 = "aaaa") + + # a multipart ETag carries no MD5, and `b` is missing from CRAN's index: + # neither may schedule a rebuild + etag <- c("a_1.0.tar.gz" = "abc-3", "b_1.0.tar.gz" = "bbbb") + + expect_identical( + outstanding_packages(c("a", "b"), etag, cran_version, cran_md5), + character(0) + ) +}) + +test_that("outstanding_packages keeps a package CRAN has no version for", { + out <- outstanding_packages( + "ghost", + c(), + c(other = "1.0"), + c(other_1.0 = "aaaa") + ) + expect_identical(out, "ghost") +}) + +test_that("outstanding_packages handles an empty list", { + expect_identical( + outstanding_packages(character(0), c(), c(), c()), + character(0) + ) +}) + +test_that("parse_rebuild_args defaults the budget", { + a <- parse_rebuild_args(c("3", "2")) + expect_identical(a$split_into, 3L) + expect_identical(a$split_index, 2L) + expect_identical(a$budget_hours, 20) + + b <- parse_rebuild_args(c("3", "2", "1.5")) + expect_identical(b$budget_hours, 1.5) +}) diff --git a/local/uvr-install.sh b/local/uvr-install.sh index 1e25e47..3966e85 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -84,9 +84,20 @@ trap 'rm -rf "$project_dir"' EXIT cd "$project_dir" "$uvr_bin" init --here --r-version "$r_full" -# --no-install: resolve and lock only. The install happens in the sync below, -# which is the only command that honours --library. -"$uvr_bin" add --no-install "$@" +# --no-install resolves and locks only; retry because concurrent shards can +# expose short-lived DNS or CRAN-index failures and uvr rolls the manifest back +# cleanly after an unsuccessful resolution. +add_attempt=1 +while ! "$uvr_bin" add --no-install "$@"; do + if [ "$add_attempt" -ge 4 ]; then + echo "error: uvr add failed after ${add_attempt} attempts" >&2 + exit 1 + fi + add_delay=$((add_attempt * 10)) + echo "warning: uvr add attempt ${add_attempt} failed; retrying in ${add_delay}s" >&2 + sleep "$add_delay" + add_attempt=$((add_attempt + 1)) +done # TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs # `apt-get install` for every resolved system dependency without refreshing the diff --git a/provider.tf b/provider.tf index badbbc1..d4f2564 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.17" + version = "~> 0.18" } } } diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 391a814..648dfc3 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -18,35 +18,70 @@ # objects were replaced. The cost is a cold cache for everything else, which is # why this is not used by the daily update path. # -# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io, -# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them. +# The public hostnames currently use separate pull zones, so callers must pass +# every zone that serves the repository. A zone can be identified by its +# numeric ID or by one of its hostnames; hostname lookup avoids persisting IDs +# that change when a zone is recreated. # # Usage: -# purge_cdn_zone.sh +# purge_cdn_zone.sh [...] # set -euo pipefail if (($# < 2)); then - echo "usage: $0 " >&2 + echo "usage: $0 [...]" >&2 exit 2 fi api_key="$1" -zone_id="$2" +shift -echo "Purging BunnyCDN pull zone ${zone_id}" +resolve_zone_id() { + local zone="$1" + local response_file + local zone_id -status=$( - curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \ + if [[ "${zone}" =~ ^[0-9]+$ ]]; then + echo "${zone}" + return + fi + + response_file=$(mktemp) + curl -sS -o "${response_file}" \ -H "AccessKey: ${api_key}" \ - -H "Content-Length: 0" \ - "https://api.bunny.net/pullzone/${zone_id}/purgeCache" -) + "https://api.bunny.net/pullzone" + zone_id=$( + jq -r --arg hostname "${zone}" \ + '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \ + "${response_file}" + ) + rm -f "${response_file}" -if [[ "${status}" != "200" && "${status}" != "204" ]]; then - echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 - cat /tmp/purge_zone_response.txt >&2 - exit 1 -fi + if [[ -z "${zone_id}" ]]; then + echo "Could not find BunnyCDN pull zone for hostname ${zone}" >&2 + exit 1 + fi -echo "Purged pull zone ${zone_id} (HTTP ${status})" + echo "${zone_id}" +} + +for zone in "$@"; do + zone_id=$(resolve_zone_id "${zone}") + echo "Purging BunnyCDN pull zone ${zone_id}" + + response_file="/tmp/purge_zone_response_${zone_id}.txt" + status=$( + curl -sS -o "${response_file}" -w '%{http_code}' -X POST \ + -H "AccessKey: ${api_key}" \ + -H "Content-Length: 0" \ + "https://api.bunny.net/pullzone/${zone_id}/purgeCache" + ) + + if [[ "${status}" != "200" && "${status}" != "204" ]]; then + echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 + cat "${response_file}" >&2 + exit 1 + fi + + echo "Purged pull zone ${zone_id} (HTTP ${status})" +done diff --git a/specs/2026-08-12-shard-weekly-rebuild-design.md b/specs/2026-08-12-shard-weekly-rebuild-design.md new file mode 100644 index 0000000..f4ccbaa --- /dev/null +++ b/specs/2026-08-12-shard-weekly-rebuild-design.md @@ -0,0 +1,167 @@ +# Design: Sharding and resuming the weekly rebuild + +Date: 2026-08-12 +Status: Approved (pending spec review) + +## Problem + +`weekly-rebuild-missing` runs one job per `-` and walks that slot's rebuild list serially in a single `R -q -e` invocation (`.crow/weekly-rebuild-missing.yaml:165`). +Until 2026-08-09 that was cheap, because every source fallback was skipped as "already built" and the list was effectively empty. +Since bincraft #105/#106/#107 and build-cran-binaries #159 the gate works, and the lists are now large. + +Share of records whose object is byte-identical to CRAN's source, measured against `cran.r-project.org` MD5s on 2026-08-12: + +| slot | records | source-served | share | +| ------------------ | ------: | ------------: | -----------: | +| `amd64/resolute` | 24 212 | 15 023 | 62.1% | +| `arm64/resolute` | 24 291 | 13 670 | 56.3% | +| `arm64/alpine324` | 24 328 | 9 514 | 39.2% | +| `amd64/alpine324` | 24 343 | 8 917 | 36.7% | +| `arm64/rhel10` | 24 695 | 5 384 | 21.9% | +| `amd64/rhel10` | 24 881 | 4 712 | 19.2% | +| 12 remaining slots | ~24 700 | 850 to 2 130 | 3.5% to 8.7% | + +A single serial job cannot absorb that. +Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) started on 2026-08-09, ran for roughly two days, reached `[8692/23885] cholera`, and was killed there. + +Two distinct failures follow from that shape. + +**No parallelism.** The work is embarrassingly parallel across packages, but one job does all of it. + +**No resumability, and no clean stopping point.** The loop has no terminating condition other than exhausting the list, so the only way to stop it is a kill. +A restarted run re-reads the same list and walks it from the first entry. +It skips completed packages via `check_s3_root_package()`, but that costs a CRAN version resolution and an S3 `HEAD` per package, thousands of times, before it reaches new work. +Worse, a kill is not a pipeline failure: the `Purge CDN cache` step is guarded by `when: status: [success, failure]` (`.crow/weekly-rebuild-missing.yaml:206-207`), and on 10910 it produced no output at all. +So the ~4 600 binaries that run did publish stayed hidden behind stale edge copies. + +## Goal + +Turn each slot's rebuild into bounded, parallel, restartable units, without introducing state that can disagree with the bucket. + +## Design + +### 1. Shard the matrix three ways + +Each of the 18 `OS`/`ARCH` rows in `.crow/weekly-rebuild-missing.yaml` gains `SPLIT_INTO: 3` and `SPLIT_INDEX: 1|2|3`, giving 54 rows. +This mirrors `.crow/build-all-versions.yaml:57-98`, which already shards its matrix four ways per arch. + +Routing needs no change. +The cron filter `cron: weekly-rebuild-missing-${OS}-${ARCH}` and the manual `evaluate: weekly_rebuild_missing == "${OS}-${ARCH}"` both match all three shards of a slot. +Placement stays on the `rpkgs-${ARCH}` group label, so shards queue against available capacity rather than oversubscribing it. + +### 2. Extract the loop into `local/rebuild-missing.R` + +The build is currently a single ~1 500-character `R -q -e` argument. +Shard arithmetic and resume logic do not belong in a YAML string, and none of it is testable there. +The loop moves to `local/rebuild-missing.R`, invoked as `Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX`, mirroring `local/build-all.R`. +Its body is unchanged in substance: read `/tmp/rebuild_pkgs.txt`, subtract `local/excluded-packages.json`, loop with `tryCatch` around `bincraft::build_binary_package()`. + +The slice is **interleaved**, not contiguous: + +```r +# the list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, +# rstan*), so contiguous thirds would be badly unbalanced +mine <- pkgs[seq(split_index, length(pkgs), by = split_into)] +``` + +`local/build-all.R:64` uses contiguous chunks via `cut()`. +That is fine there because its list is every CRAN package and version, so the chunks average out. +Here the list is a filtered backlog in which expensive families sit adjacent, so interleaving is the better default. +Interleaving also makes each shard's `[i/n]` progress representative of the slot as a whole. + +### 3. Resume by re-deriving state from the bucket + +Before the loop, the shard performs one `s3fs::s3_dir_info()` on `devxy-rpkgs-binaries///latest/src/contrib` and reads the `etag` column. +It fetches CRAN's `PACKAGES` once for the latest version and published `MD5sum` of every package. +A package is still outstanding if and only if the object at `_.tar.gz` has an ETag equal to CRAN's `MD5sum` for that version, which is the definition `check_s3_root_package()` already applies one package at a time. + +```r +# one paginated listing instead of ~2900 sequential HEAD requests per shard +info <- s3fs::s3_dir_info(slot_dir) +etag <- setNames(gsub('^"|"$', "", info$etag), basename(info$uri)) + +key <- sprintf("%s_%s.tar.gz", mine, cran_version[mine]) +# keep a package when no object exists yet, or when the object is still +# byte-identical to CRAN's source; drop it once a real binary is published +mine <- mine[is.na(etag[key]) | etag[key] == cran_md5[key]] +``` + +This is the whole resume mechanism. +There is no progress file, no volume, and no database cursor. +A restarted shard recomputes ground truth and continues where it stopped, and it is correct even when a sibling shard, a `process-updates` cron, or a manual `just rebuild` completed something in the meantime. + +Three properties make this the right source of truth: + +- **It is what the build itself checks.** Any other store can disagree with the bucket; this one cannot. +- **It is agent-independent.** `.crow/weekly-rebuild-missing.yaml` mounts no `volumes:`, unlike `.crow/build-all-versions.yaml:132-133`, so `/mnt/cache` is per-job and cannot carry progress anyway. +- **It costs one listing.** `cranlike`'s `s3` fork already does exactly this call against this bucket at ~24 000 objects, so the approach is proven at the required scale. + +It must read ETags rather than the slot index's `Built` field, which is how `local/packages-to-build.R:104-130` answers the same question. +Under this design the index is not rewritten until the dependent re-index pipeline runs (section 5), so mid-run it cannot reflect the current run's progress. + +Packages that genuinely fail to build re-publish their CRAN source, so they stay outstanding and would be retried on every restart. +That is already handled upstream: `bincraft::filter_packages_with_errors()` (`R/build_binaries.R:1018`, `:1143`) drops anything with `error_occurred = TRUE`, and `store_build_metadata = TRUE` is passed on every call. +No additional poison-pill filter is needed here. + +Only the flat `src/contrib` path is considered. +The rebuild call passes no `is_r_minor_sensitive`, so it defaults to `FALSE` and only ever targets the flat path; the resume filter matches that scope deliberately. + +### 4. Give each shard a wall-clock budget + +`local/rebuild-missing.R` takes a budget, defaulting to 20 hours, and breaks out of the loop once it is exceeded: + +```r +# exit cleanly rather than being killed, so the dependent re-index still runs +if (difftime(Sys.time(), started, units = "hours") > budget_hours) { + cat(sprintf("Budget of %sh reached after %d/%d packages; stopping cleanly\n", budget_hours, i, n)) + break +} +``` + +It exits 0 and reports how much of the slice it covered. +Every run then has a terminating condition, the re-index and purge always fire, and the remainder is picked up by the next run with no bookkeeping, because section 3 recomputes the outstanding set from scratch. + +### 5. Move the re-index and purge into `.crow/weekly-rebuild-reindex.yaml` + +Three shards per slot means three concurrent `upload_package_index()` calls on the same S3 prefix. +`cranlike::update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work. +The re-index steps (`.crow/weekly-rebuild-missing.yaml:171-176`) and the purge step (`:187-207`) therefore leave that file entirely. + +The new file carries: + +```yaml +depends_on: + - weekly-rebuild-missing +runs_on: [success, failure] +``` + +`runs_on: [success, failure]` validates as a workflow-level key under `crow lint`, so a failing shard no longer withholds the re-index. +The file uses the same 18-row matrix and the same `when:` gating as `weekly-rebuild-missing`, so it only re-indexes slots that actually ran. +Each row re-indexes the flat slot and every per-minor slot. +`scripts/purge_cdn_zone.sh` runs once on a single row, because all hostnames share pull zone `3857050` and 18 identical zone purges would be waste. + +## Failure behaviour + +| case | today | after | +| -------------------------- | ----------------------------------- | ----------------------------------------------------- | +| one package errors | `tryCatch` logs, loop continues | unchanged | +| a shard fails outright | purge runs, re-index does not | re-index and purge run via `runs_on` | +| a shard exceeds its budget | cannot happen, runs until killed | exits 0, re-index and purge run | +| a shard is killed | nothing runs | still nothing; trigger the re-index pipeline alone | +| a shard restarts | re-walks the list, HEAD per package | one listing, resumes at the first outstanding package | + +The known cost of `depends_on` being file-level rather than row-level: on the weekly cron no slot is re-indexed until the slowest of all 54 jobs finishes. +The 20-hour budget bounds that at roughly one day. + +## Out of scope + +- `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row for the platform and arch, which is precisely the source-fallback set. That is a separate change. +- Bunny Perma-Cache eviction. `scripts/purge_cdn_zone.sh` purges the regular edge cache only; see the note in `CLAUDE.md` and issue history. +- The audit that produces the rebuild list is unchanged. + +## Verification + +- `crow lint .crow/` passes for both pipeline files. +- `local/rebuild-missing.R` gets unit coverage in `local/tests/` for the two pure pieces: the interleaved slice (disjoint, covering, deterministic) and the outstanding-set filter (source-served ETag kept, binary ETag dropped, absent object kept). +- A single-slot manual run of `alpine-324-amd64` shard 1 confirms the listing shortcut against the live bucket, and that the reported outstanding count is close to the 8 917 measured above divided by three. +- Restarting that shard mid-run confirms it resumes rather than replaying, by comparing the outstanding count it reports on the second start.