fix(ci): clear stale pkgcache metadata at start of build-all runs

The install-deps step precomputes pkgs_to_build.rds, r_minor_sensitive_pkgs.rds
and s3_cache.rds into /mnt/cache, but that cache volume is per-agent.
A build job scheduled on a fresh agent (or racing install-deps) finds the
snapshot absent and fails at readRDS.

Guard the reads in build-all.R: when any snapshot file is missing, source
packages-to-build.R and save the derived files (atomic temp+rename so a
concurrent job never reads a half-written rds). The first job on an agent
repopulates the shared cache for subsequent jobs; jobs that also miss redo
the work.

.crow/build-all-versions-install-deps.yaml

@@ -4,19 +4,19 @@
 # they are merged with build-all-versions' identical declarations.
 variables:
   target_arch:
-    description: "Architecture to build."
+    description: 'Architecture to build.'
     options:
       - amd64
       - arm64
     default: amd64
   OS:
-    description: "Base OS image name (e.g. alpine, redhat, ubuntu)."
+    description: 'Base OS image name (e.g. alpine, redhat, ubuntu).'
     default: alpine
   OS_VERSION:
-    description: "OS version / image tag (e.g. 3.24, 9, jammy, noble)."
-    default: "3.24"
+    description: 'OS version / image tag (e.g. 3.24, 9, jammy, noble).'
+    default: '3.24'
   R_VERSION:
-    description: "Primary R version under /opt/R."
+    description: 'Primary R version under /opt/R.'
     options:
       - 4.5.3
       - 4.4.3
@@ -60,6 +60,10 @@ steps:
     commands:
       # one-time full wipe to fix corrupted .so files from previous failed builds
       # - rm -rf /mnt/cache/R-pkgs
+      # Clear churny pkgcache metadata left by a prior crashed run (the "patched"
+      # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*).
+      # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs.
+      - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
       - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft

.crow/build-all-versions.yaml

@@ -5,19 +5,19 @@
 # Skip list lives in local/excluded-packages.json (read by local/build-all.R).
 variables:
   target_arch:
-    description: "Architecture to build."
+    description: 'Architecture to build.'
     options:
       - amd64
       - arm64
     default: amd64
   OS:
-    description: "Base OS image name (e.g. alpine, redhat, ubuntu)."
+    description: 'Base OS image name (e.g. alpine, redhat, ubuntu).'
     default: alpine
   OS_VERSION:
-    description: "OS version / image tag (e.g. 3.24, 9, jammy, noble)."
-    default: "3.24"
+    description: 'OS version / image tag (e.g. 3.24, 9, jammy, noble).'
+    default: '3.24'
   R_VERSION:
-    description: "Primary R version under /opt/R."
+    description: 'Primary R version under /opt/R.'
     options:
       - 4.5.3
       - 4.4.3
@@ -99,6 +99,11 @@ steps:
       - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
     commands:
       - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      # Clear churny pkgcache metadata left by a prior crashed run (the "patched"
+      # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*).
+      # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. Within-run
+      # growth is bounded separately by trim_pkgcache_metadata() in build-all.R.
+      - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
       - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1
@@ -152,4 +157,4 @@ steps:
             cpu: 1000m
           limits:
             memory: 20Gi
-            cpu: 2000m
+            cpu: 2000m

local/build-all.R

@@ -15,6 +15,31 @@ library(bincraft, quietly = TRUE)
 library(future)
 plan("sequential")
 
+# The install-deps step precomputes the package snapshot into /mnt/cache, but
+# that volume is per-agent: a job landing on a fresh agent (or racing
+# install-deps) finds it empty. Recompute the snapshot here when any part is
+# missing, so the first job on an agent repopulates the cache for the jobs that
+# follow; concurrent jobs that also miss simply redo the work. Write via a
+# temp file + atomic rename so a concurrent reader never sees a half-written rds.
+package_cache_files <- c(
+  "/mnt/cache/packages/pkgs_to_build.rds",
+  "/mnt/cache/packages/r_minor_sensitive_pkgs.rds",
+  "/mnt/cache/packages/s3_cache.rds"
+)
+if (!all(file.exists(package_cache_files))) {
+  message("Package snapshot missing from cache; recomputing via packages-to-build.R")
+  dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
+  save_rds_atomic <- function(obj, path) {
+    tmp <- paste0(path, ".tmp.", Sys.getpid())
+    saveRDS(obj, tmp)
+    file.rename(tmp, path)
+  }
+  source(file.path("local", "packages-to-build.R"))
+  save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds")
+  save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
+  message("Package snapshot recomputed.")
+}
+
 pkgs <- if (sensitive_only) {
   readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
 } else {
@@ -71,6 +96,10 @@ s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
 sprintf("S3 cache: %s files", length(s3_cache))
 
 n <- nrow(chunk)
+# Every `trim_every` packages, bound the pkgcache _metadata dir so a full-platform
+# run does not accumulate thousands of ~70 MB snapshots and fill the host disk.
+# No-op on amd64 (R_PKG_CACHE_DIR is empty / cache not persisted).
+trim_every <- 25L
 mapply(
   function(pkg, ver, sens, i) {
     cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
@@ -95,6 +124,12 @@ mapply(
       upload = TRUE,
       store_build_metadata = TRUE
     )
+    if (i %% trim_every == 0L) {
+      removed <- trim_pkgcache_metadata()
+      if (removed > 0L) {
+        cat(sprintf("  [pkgcache trim] removed %d stale _metadata entries\n", removed))
+      }
+    }
   },
   chunk$Package,
   chunk$Version,

local/r-minor-helpers.R

@@ -31,3 +31,42 @@ parse_build_args <- function(args) {
     ncpus = as.integer(pos[3L])
   )
 }
+
+# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the
+# "patched" repo mints a new content hash on every PACKAGES change, so each build
+# writes a fresh ~70 MB _metadata/pkgs-<hash>.rds (+ patched-<hash>/) that is
+# never reused. Keep the `keep` newest entries by mtime; only remove entries
+# older than `min_age_secs`, so a concurrent split-job's in-flight files are
+# never deleted (each build uses a unique hash, so aged entries are
+# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are
+# not matched and thus preserved. Returns the number of entries removed.
+trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"),
+                                   keep = 20L,
+                                   min_age_secs = 600) {
+  meta <- file.path(cache_dir, "R", "pkgcache", "_metadata")
+  if (!nzchar(cache_dir) || !dir.exists(meta)) {
+    return(0L)
+  }
+  entries <- c(
+    Sys.glob(file.path(meta, "patched-*")),
+    Sys.glob(file.path(meta, "pkgs-*.rds"))
+  )
+  if (length(entries) == 0L) {
+    return(0L)
+  }
+  info <- file.info(entries)
+  order_new_first <- order(info$mtime, decreasing = TRUE)
+  ranked <- entries[order_new_first]
+  ranked_mtime <- info$mtime[order_new_first]
+  if (length(ranked) <= keep) {
+    return(0L)
+  }
+  candidates <- ranked[(keep + 1L):length(ranked)]
+  candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)])
+  removable <- candidates[candidate_age >= min_age_secs]
+  if (length(removable) == 0L) {
+    return(0L)
+  }
+  unlink(removable, recursive = TRUE, force = TRUE)
+  length(removable)
+}

local/tests/test-trim-pkgcache.R

@@ -0,0 +1,68 @@
+source(file.path("..", "r-minor-helpers.R"))
+
+# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is
+# set to `age_secs` in the past so we can exercise the age gate deterministically.
+make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) {
+  root <- tempfile("pkgcache-")
+  meta <- file.path(root, "R", "pkgcache", "_metadata")
+  dir.create(meta, recursive = TRUE)
+  old <- Sys.time() - age_secs
+  mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p }
+  mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p }
+  for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i)))
+  for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i)))
+  if (keep_repos) {
+    mk_dir(file.path(meta, "CRAN-075c426938"))
+    mk_dir(file.path(meta, "BioCsoft-1ac964ed6c"))
+    mk_file(file.path(meta, "bioc-sysreqs.dcf.gz"))
+    mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive
+  }
+  root
+}
+
+n_churn <- function(root) {
+  meta <- file.path(root, "R", "pkgcache", "_metadata")
+  length(Sys.glob(file.path(meta, "patched-*"))) +
+    length(Sys.glob(file.path(meta, "pkgs-*.rds")))
+}
+
+test_that("empty cache_dir is a no-op", {
+  expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L)
+})
+
+test_that("missing _metadata dir is a no-op", {
+  expect_identical(
+    trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0),
+    0L
+  )
+})
+
+test_that("fewer than keep entries removes nothing", {
+  root <- make_meta(patched = 2L, pkgs = 2L)
+  expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L)
+  expect_identical(n_churn(root), 4L)
+})
+
+test_that("trims down to keep newest, leaving churn == keep", {
+  root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old
+  removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0)
+  expect_identical(removed, 40L)
+  expect_identical(n_churn(root), 20L)
+})
+
+test_that("entries younger than min_age_secs are protected", {
+  root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60)
+  # keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed
+  expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L)
+  expect_identical(n_churn(root), 30L)
+})
+
+test_that("stable repo dirs and pkg downloads are never touched", {
+  root <- make_meta(patched = 30L, pkgs = 30L)
+  trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0)
+  meta <- file.path(root, "R", "pkgcache", "_metadata")
+  expect_true(dir.exists(file.path(meta, "CRAN-075c426938")))
+  expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c")))
+  expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz")))
+  expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg")))
+})