pat-s
a4b274f281
Some checks failed
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was canceled
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps/2 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was canceled
ci/crow/cron/process-updates/8 Pipeline failed
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/17 Pipeline was canceled
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline failed
ci/crow/manual/build-all-versions-install-deps/1 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline failed
ci/crow/cron/process-updates/1 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/manual/build-all-versions/2 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/manual/build-all-versions/3 Pipeline failed
ci/crow/manual/build-all-versions/1 Pipeline failed
ci/crow/cron/process-updates/18 Pipeline was successful
ci/crow/manual/build-all-versions/6 Pipeline failed
ci/crow/manual/build-all-versions/5 Pipeline was canceled
ci/crow/manual/build-all-versions/8 Pipeline was canceled
ci/crow/cron/process-updates/12 Pipeline was canceled
ci/crow/manual/build-all-versions/7 Pipeline was canceled
ci/crow/cron/process-updates/6 Pipeline failed
ci/crow/cron/process-updates/2 Pipeline failed
ci/crow/manual/build-all-versions/4 Pipeline was canceled
## Problem
The arm64 `build-all` pipeline fills the macmini (gaia) host disk despite an 8h prune.
Root cause is not images or job volumes: it is the persistent dep-cache volume, specifically `pkgcache/R/pkgcache/_metadata`, which grew to ~165 GB.
`{pkgcache}` mints a new content hash for the "patched" binaries repo on every PACKAGES change, so each per-package build writes a fresh ~70 MB `pkgs-<hash>.rds` (+ `patched-<hash>/`) that is never evicted (2407 snapshots observed).
When the disk hits 100% OrbStack stops and the on-host prune can no longer connect to the daemon, so it never self-heals.
## Change (Workstream A of the disk-fill fix)
- Add `trim_pkgcache_metadata()` to `local/r-minor-helpers.R`: keeps the newest `keep` (default 20) `patched-*`/`pkgs-*.rds` entries under `_metadata`, deleting only entries older than `min_age_secs` (default 600s) so it never races the up-to-4 concurrent split-jobs sharing the volume.
Preserves `pkg/` downloads and the stable CRAN/BioC/INLA repo dirs.
No-op when `R_PKG_CACHE_DIR` is empty (amd64) or `_metadata` is absent (first run).
- Call it every 25 packages inside the build loop in `local/build-all.R`.
- Add a defensive start-of-run cleanup of `_metadata/patched-*` + `pkgs-*.rds` to the two workflows that mount the persistent volume (`build-all-versions.yaml`, `build-all-versions-install-deps.yaml`).
Only these paths are touched; `process-updates.yaml`/`weekly-rebuild-missing.yaml` (no persistent volume) are unchanged.
Follow-ups (separate workstreams): on-host self-healing prune watcher + OrbStack disk cap (ansible), and Prometheus/Grafana alerting (k8s-talos).
Upstream: bincraft patched-repo hash churn is the true source fix.
New unit tests (6) for the helper; full suite 24/24 green.
Reviewed-on: #110
68 lines
2.8 KiB
R
68 lines
2.8 KiB
R
source(file.path("..", "r-minor-helpers.R"))
|
|
|
|
# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is
|
|
# set to `age_secs` in the past so we can exercise the age gate deterministically.
|
|
make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) {
|
|
root <- tempfile("pkgcache-")
|
|
meta <- file.path(root, "R", "pkgcache", "_metadata")
|
|
dir.create(meta, recursive = TRUE)
|
|
old <- Sys.time() - age_secs
|
|
mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p }
|
|
mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p }
|
|
for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i)))
|
|
for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i)))
|
|
if (keep_repos) {
|
|
mk_dir(file.path(meta, "CRAN-075c426938"))
|
|
mk_dir(file.path(meta, "BioCsoft-1ac964ed6c"))
|
|
mk_file(file.path(meta, "bioc-sysreqs.dcf.gz"))
|
|
mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive
|
|
}
|
|
root
|
|
}
|
|
|
|
n_churn <- function(root) {
|
|
meta <- file.path(root, "R", "pkgcache", "_metadata")
|
|
length(Sys.glob(file.path(meta, "patched-*"))) +
|
|
length(Sys.glob(file.path(meta, "pkgs-*.rds")))
|
|
}
|
|
|
|
test_that("empty cache_dir is a no-op", {
|
|
expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L)
|
|
})
|
|
|
|
test_that("missing _metadata dir is a no-op", {
|
|
expect_identical(
|
|
trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0),
|
|
0L
|
|
)
|
|
})
|
|
|
|
test_that("fewer than keep entries removes nothing", {
|
|
root <- make_meta(patched = 2L, pkgs = 2L)
|
|
expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L)
|
|
expect_identical(n_churn(root), 4L)
|
|
})
|
|
|
|
test_that("trims down to keep newest, leaving churn == keep", {
|
|
root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old
|
|
removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0)
|
|
expect_identical(removed, 40L)
|
|
expect_identical(n_churn(root), 20L)
|
|
})
|
|
|
|
test_that("entries younger than min_age_secs are protected", {
|
|
root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60)
|
|
# keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed
|
|
expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L)
|
|
expect_identical(n_churn(root), 30L)
|
|
})
|
|
|
|
test_that("stable repo dirs and pkg downloads are never touched", {
|
|
root <- make_meta(patched = 30L, pkgs = 30L)
|
|
trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0)
|
|
meta <- file.path(root, "R", "pkgcache", "_metadata")
|
|
expect_true(dir.exists(file.path(meta, "CRAN-075c426938")))
|
|
expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c")))
|
|
expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz")))
|
|
expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg")))
|
|
})
|