.crow/weekly-rebuild-missing.yaml

@@ -163,6 +163,17 @@ steps:
       - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2
       - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
       - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1
+      # A rebuild replaces objects in place, so the slot's index still advertises
+      # the old MD5 and, for anything that had been served from source, no Built
+      # stamp. Re-index here rather than waiting for the next process-updates
+      # run, or the rebuilt binaries stay invisible to clients until then.
+      # The codename is detected from the image's /etc/os-release.
+      - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
+      - |
+        for RBIN in /opt/R/[0-9]*/bin/R; do
+          RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
+          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
+        done
     backend_options:
       docker:
         resources:
@@ -172,3 +183,25 @@ steps:
           limits:
             memory: 18Gi
             cpu: 3000m
+
+  - name: Purge CDN cache
+    image: reg.devxy.io/docker.io/library/alpine:3.24
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      BUNNYNET_API_KEY:
+        from_secret: BUNNYNET_API_KEY
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      # All hostnames on the zone share this id, so one purge covers
+      # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com.
+      BUNNY_PULLZONE: '3857050'
+    commands:
+      - apk add --no-cache -q bash curl git
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE"
+    # A rebuild that died part-way still replaced objects, and those are exactly
+    # the ones a stale edge would keep hiding, so purge either way.
+    when:
+      - status: [success, failure]

local/packages-to-build.R

@@ -89,14 +89,61 @@ s3_pkgs <- s3fs::s3_dir_ls(
   recurse = TRUE
 )
 
-# Save the raw S3 file listing for the build step to use as s3_package_cache
+file_names <- basename(s3_pkgs)
+
+# An object occupying a key is not proof a binary was built: a package whose
+# build failed has its CRAN source published under exactly that name. Left in
+# the cache, `build_binary_package()` reads it as "already built" and skips the
+# package forever, which is how alpine324 accumulated ~13.5k source tarballs.
+#
+# bincraft stamps `Built` only on records it actually built, so the slot's own
+# index distinguishes them. A slot last indexed by a bincraft that predates that
+# fix stamps `Built` on everything, so the cache is then unchanged from before.
+# Archived objects have no index record and are kept: unknown means binary,
+# never "rebuild it".
+index_url <- sprintf(
+  "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz",
+  arch,
+  codename
+)
+source_served <- tryCatch(
+  {
+    con_idx <- gzcon(url(index_url, open = "rb"))
+    on.exit(close(con_idx), add = TRUE)
+    idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built"))
+    sprintf(
+      "%s_%s.tar.gz",
+      idx[is.na(idx[, "Built"]), "Package"],
+      idx[is.na(idx[, "Built"]), "Version"]
+    )
+  },
+  error = function(e) {
+    cat(sprintf(
+      "WARNING: could not read %s (%s); keeping the full S3 cache\n",
+      index_url,
+      conditionMessage(e)
+    ))
+    character(0)
+  }
+)
+
+binary_cache <- setdiff(file_names, source_served)
+cat(sprintf(
+  "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n",
+  length(file_names),
+  length(file_names) - length(binary_cache),
+  length(binary_cache)
+))
+
+# Save the S3 file listing for the build step to use as s3_package_cache.
 # This avoids loading s3fs/reticulate in the build container, saving memory for
 # the dependency-installer subprocesses
-saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds")
-
-file_names <- basename(s3_pkgs)
-matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names)
-parts <- regmatches(file_names, matches)
+saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds")
+# Built from the filtered listing, not the raw one: `s3_dt` is subtracted from
+# the build list below, so a source fallback left in here would exclude the very
+# package that needs building.
+matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", binary_cache)
+parts <- regmatches(binary_cache, matches)
 parts <- parts[sapply(parts, length) == 3]
 s3_dt <- data.table(
   Package = sapply(parts, `[`, 2),

scripts/purge_cdn_zone.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+#
+# Purge the entire BunnyCDN pull zone.
+#
+# `purge_cdn_cache.sh` purges the five index files by URL, which is right after
+# a normal update: new packages arrive at new URLs, so only the index is stale.
+#
+# A rebuild is different. It replaces an object *in place*: a package whose
+# build failed was published as its CRAN source, and the rebuilt binary takes
+# exactly the same URL. The zone caches tarballs for ~370 days
+# (`cache_expiration_time` in cdn.tf), so without a purge every client keeps
+# receiving the source tarball for up to a year, and nothing about it looks
+# wrong from the outside.
+#
+# Purging per URL would mean one API call per replaced package -- ~13.5k per
+# arch against a rate-limited endpoint, where a single missed call leaves a
+# silently stale package. One zone purge is a single call regardless of how many
+# objects were replaced. The cost is a cold cache for everything else, which is
+# why this is not used by the daily update path.
+#
+# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io,
+# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them.
+#
+# Usage:
+#   purge_cdn_zone.sh <BUNNYNET_API_KEY> <pull_zone_id>
+#
+set -euo pipefail
+
+if (($# < 2)); then
+  echo "usage: $0 <api_key> <pull_zone_id>" >&2
+  exit 2
+fi
+
+api_key="$1"
+zone_id="$2"
+
+echo "Purging BunnyCDN pull zone ${zone_id}"
+
+status=$(
+  curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \
+    -H "AccessKey: ${api_key}" \
+    -H "Content-Length: 0" \
+    "https://api.bunny.net/pullzone/${zone_id}/purgeCache"
+)
+
+if [[ "${status}" != "200" && "${status}" != "204" ]]; then
+  echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2
+  cat /tmp/purge_zone_response.txt >&2
+  exit 1
+fi
+
+echo "Purged pull zone ${zone_id} (HTTP ${status})"