pat-s
a4b274f281
Some checks failed
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was canceled
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps/2 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was canceled
ci/crow/cron/process-updates/8 Pipeline failed
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/17 Pipeline was canceled
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline failed
ci/crow/manual/build-all-versions-install-deps/1 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline failed
ci/crow/cron/process-updates/1 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/manual/build-all-versions/2 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/manual/build-all-versions/3 Pipeline failed
ci/crow/manual/build-all-versions/1 Pipeline failed
ci/crow/cron/process-updates/18 Pipeline was successful
ci/crow/manual/build-all-versions/6 Pipeline failed
ci/crow/manual/build-all-versions/5 Pipeline was canceled
ci/crow/manual/build-all-versions/8 Pipeline was canceled
ci/crow/cron/process-updates/12 Pipeline was canceled
ci/crow/manual/build-all-versions/7 Pipeline was canceled
ci/crow/cron/process-updates/6 Pipeline failed
ci/crow/cron/process-updates/2 Pipeline failed
ci/crow/manual/build-all-versions/4 Pipeline was canceled
## Problem
The arm64 `build-all` pipeline fills the macmini (gaia) host disk despite an 8h prune.
Root cause is not images or job volumes: it is the persistent dep-cache volume, specifically `pkgcache/R/pkgcache/_metadata`, which grew to ~165 GB.
`{pkgcache}` mints a new content hash for the "patched" binaries repo on every PACKAGES change, so each per-package build writes a fresh ~70 MB `pkgs-<hash>.rds` (+ `patched-<hash>/`) that is never evicted (2407 snapshots observed).
When the disk hits 100% OrbStack stops and the on-host prune can no longer connect to the daemon, so it never self-heals.
## Change (Workstream A of the disk-fill fix)
- Add `trim_pkgcache_metadata()` to `local/r-minor-helpers.R`: keeps the newest `keep` (default 20) `patched-*`/`pkgs-*.rds` entries under `_metadata`, deleting only entries older than `min_age_secs` (default 600s) so it never races the up-to-4 concurrent split-jobs sharing the volume.
Preserves `pkg/` downloads and the stable CRAN/BioC/INLA repo dirs.
No-op when `R_PKG_CACHE_DIR` is empty (amd64) or `_metadata` is absent (first run).
- Call it every 25 packages inside the build loop in `local/build-all.R`.
- Add a defensive start-of-run cleanup of `_metadata/patched-*` + `pkgs-*.rds` to the two workflows that mount the persistent volume (`build-all-versions.yaml`, `build-all-versions-install-deps.yaml`).
Only these paths are touched; `process-updates.yaml`/`weekly-rebuild-missing.yaml` (no persistent volume) are unchanged.
Follow-ups (separate workstreams): on-host self-healing prune watcher + OrbStack disk cap (ansible), and Prometheus/Grafana alerting (k8s-talos).
Upstream: bincraft patched-repo hash churn is the true source fix.
New unit tests (6) for the helper; full suite 24/24 green.
Reviewed-on: #110
72 lines
2.7 KiB
R
72 lines
2.7 KiB
R
# Metadata-only ABI triage so the full build avoids downloading every source.
|
|
# Mirrors bincraft::abi_classify rules 1-2; "ambiguous" packages still need a
|
|
# source grep via bincraft::needs_per_minor_recompile().
|
|
|
|
classify_from_metadata <- function(needs_compilation, linking_to, risky_deps) {
|
|
nc <- if (length(needs_compilation) == 0L || is.na(needs_compilation)) {
|
|
""
|
|
} else {
|
|
tolower(trimws(needs_compilation))
|
|
}
|
|
if (!identical(nc, "yes")) {
|
|
return("not-sensitive")
|
|
}
|
|
lt <- if (length(linking_to) == 0L || is.na(linking_to)) "" else linking_to
|
|
linked <- trimws(unlist(strsplit(lt, "[,\n]")))
|
|
linked <- sub("\\s*\\(.*\\)$", "", linked) # strip "(>= x)" constraints
|
|
linked <- linked[nzchar(linked)]
|
|
if (any(linked %in% risky_deps)) {
|
|
return("sensitive")
|
|
}
|
|
"ambiguous"
|
|
}
|
|
|
|
parse_build_args <- function(args) {
|
|
sensitive_only <- "--sensitive-only" %in% args
|
|
pos <- args[!startsWith(args, "--")]
|
|
list(
|
|
sensitive_only = sensitive_only,
|
|
split_into = as.integer(pos[1L]),
|
|
split_index = as.integer(pos[2L]),
|
|
ncpus = as.integer(pos[3L])
|
|
)
|
|
}
|
|
|
|
# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the
|
|
# "patched" repo mints a new content hash on every PACKAGES change, so each build
|
|
# writes a fresh ~70 MB _metadata/pkgs-<hash>.rds (+ patched-<hash>/) that is
|
|
# never reused. Keep the `keep` newest entries by mtime; only remove entries
|
|
# older than `min_age_secs`, so a concurrent split-job's in-flight files are
|
|
# never deleted (each build uses a unique hash, so aged entries are
|
|
# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are
|
|
# not matched and thus preserved. Returns the number of entries removed.
|
|
trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"),
|
|
keep = 20L,
|
|
min_age_secs = 600) {
|
|
meta <- file.path(cache_dir, "R", "pkgcache", "_metadata")
|
|
if (!nzchar(cache_dir) || !dir.exists(meta)) {
|
|
return(0L)
|
|
}
|
|
entries <- c(
|
|
Sys.glob(file.path(meta, "patched-*")),
|
|
Sys.glob(file.path(meta, "pkgs-*.rds"))
|
|
)
|
|
if (length(entries) == 0L) {
|
|
return(0L)
|
|
}
|
|
info <- file.info(entries)
|
|
order_new_first <- order(info$mtime, decreasing = TRUE)
|
|
ranked <- entries[order_new_first]
|
|
ranked_mtime <- info$mtime[order_new_first]
|
|
if (length(ranked) <= keep) {
|
|
return(0L)
|
|
}
|
|
candidates <- ranked[(keep + 1L):length(ranked)]
|
|
candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)])
|
|
removable <- candidates[candidate_age >= min_age_secs]
|
|
if (length(removable) == 0L) {
|
|
return(0L)
|
|
}
|
|
unlink(removable, recursive = TRUE, force = TRUE)
|
|
length(removable)
|
|
}
|