fix: recompute package snapshot when missing from cache (#95)
Some checks failed
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline failed
ci/crow/cron/process-updates/1 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps/1 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline failed
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps/2 Pipeline was successful
ci/crow/manual/build-all-versions/5 Pipeline was canceled
ci/crow/manual/build-all-versions/6 Pipeline was canceled
ci/crow/manual/build-all-versions/7 Pipeline was canceled
ci/crow/manual/build-all-versions/8 Pipeline was canceled
ci/crow/manual/build-all-versions/4 Pipeline failed
ci/crow/manual/build-all-versions/1 Pipeline failed
ci/crow/manual/build-all-versions/3 Pipeline failed
ci/crow/manual/build-all-versions/2 Pipeline failed
Some checks failed
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline failed
ci/crow/cron/process-updates/1 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps/1 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline failed
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps/2 Pipeline was successful
ci/crow/manual/build-all-versions/5 Pipeline was canceled
ci/crow/manual/build-all-versions/6 Pipeline was canceled
ci/crow/manual/build-all-versions/7 Pipeline was canceled
ci/crow/manual/build-all-versions/8 Pipeline was canceled
ci/crow/manual/build-all-versions/4 Pipeline failed
ci/crow/manual/build-all-versions/1 Pipeline failed
ci/crow/manual/build-all-versions/3 Pipeline failed
ci/crow/manual/build-all-versions/2 Pipeline failed
## Summary `build-all.R` reads three snapshot files from `/mnt/cache/packages/` that the `build-all-versions-install-deps` step precomputes: `pkgs_to_build.rds`, `r_minor_sensitive_pkgs.rds`, and `s3_cache.rds`. That cache volume is **per-agent**, so a build job scheduled on a different (fresh) agent than the one that ran install-deps finds the snapshot absent and dies at `readRDS` (`cannot open compressed file '/mnt/cache/packages/pkgs_to_build.rds'`). This adds a conditional guard at the top of `build-all.R`: when any of the three files is missing, it sources `local/packages-to-build.R` (which has all needed creds via `PGPASS` / `B2_S3_*` env, already present in the build step) and writes the derived `.rds` files — exactly mirroring the install-deps command. - The first build job on a fresh agent repopulates the shared cache, so subsequent jobs on that agent reuse it. - Concurrent jobs that also miss simply redo the work (accepted tradeoff vs. slow shared storage like NFS). - Saves use a temp-file + atomic `file.rename`, so a concurrent reader never sees a half-written `.rds`. Reviewed-on: #95
This commit is contained in:
parent
4c67703f52
commit
fa955c10bf
1 changed files with 25 additions and 0 deletions
|
|
@ -15,6 +15,31 @@ library(bincraft, quietly = TRUE)
|
|||
library(future)
|
||||
plan("sequential")
|
||||
|
||||
# The install-deps step precomputes the package snapshot into /mnt/cache, but
|
||||
# that volume is per-agent: a job landing on a fresh agent (or racing
|
||||
# install-deps) finds it empty. Recompute the snapshot here when any part is
|
||||
# missing, so the first job on an agent repopulates the cache for the jobs that
|
||||
# follow; concurrent jobs that also miss simply redo the work. Write via a
|
||||
# temp file + atomic rename so a concurrent reader never sees a half-written rds.
|
||||
package_cache_files <- c(
|
||||
"/mnt/cache/packages/pkgs_to_build.rds",
|
||||
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds",
|
||||
"/mnt/cache/packages/s3_cache.rds"
|
||||
)
|
||||
if (!all(file.exists(package_cache_files))) {
|
||||
message("Package snapshot missing from cache; recomputing via packages-to-build.R")
|
||||
dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
|
||||
save_rds_atomic <- function(obj, path) {
|
||||
tmp <- paste0(path, ".tmp.", Sys.getpid())
|
||||
saveRDS(obj, tmp)
|
||||
file.rename(tmp, path)
|
||||
}
|
||||
source(file.path("local", "packages-to-build.R"))
|
||||
save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds")
|
||||
save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
||||
message("Package snapshot recomputed.")
|
||||
}
|
||||
|
||||
pkgs <- if (sensitive_only) {
|
||||
readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
||||
} else {
|
||||
|
|
|
|||
Loading…
Reference in a new issue