fix(build): recompute a stale package snapshot, not just a missing one (#190)
Some checks failed
ci/crow/manual/build-all-versions-install-deps/2 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps/1 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline is pending
ci/crow/cron/process-updates/17 Pipeline is pending
ci/crow/cron/process-updates/6 Pipeline is pending
ci/crow/cron/process-updates/5 Pipeline is pending
ci/crow/cron/process-updates/1 Pipeline is pending
ci/crow/cron/process-updates/7 Pipeline is pending
ci/crow/cron/process-updates/9 Pipeline is pending
ci/crow/cron/process-updates/3 Pipeline is pending
ci/crow/cron/process-updates/13 Pipeline is pending
ci/crow/cron/process-updates/15 Pipeline is pending
ci/crow/manual/build-all-versions/4 Pipeline was canceled
ci/crow/manual/build-all-versions/1 Pipeline was canceled
ci/crow/manual/build-all-versions/5 Pipeline was canceled
ci/crow/manual/build-all-versions/8 Pipeline was canceled
ci/crow/manual/build-all-versions/6 Pipeline was canceled
ci/crow/manual/build-all-versions/7 Pipeline was canceled
ci/crow/cron/process-updates/10 Pipeline is running
ci/crow/manual/build-all-versions/3 Pipeline was canceled
ci/crow/manual/build-all-versions/2 Pipeline was canceled
ci/crow/cron/process-updates/2 Pipeline failed
ci/crow/cron/process-updates/14 Pipeline is running
ci/crow/cron/process-updates/4 Pipeline failed
ci/crow/cron/process-updates/16 Pipeline is running
ci/crow/cron/process-updates/8 Pipeline failed
ci/crow/cron/process-updates/18 Pipeline is pending
ci/crow/cron/process-updates/11 Pipeline was successful
Some checks failed
ci/crow/manual/build-all-versions-install-deps/2 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps/1 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline is pending
ci/crow/cron/process-updates/17 Pipeline is pending
ci/crow/cron/process-updates/6 Pipeline is pending
ci/crow/cron/process-updates/5 Pipeline is pending
ci/crow/cron/process-updates/1 Pipeline is pending
ci/crow/cron/process-updates/7 Pipeline is pending
ci/crow/cron/process-updates/9 Pipeline is pending
ci/crow/cron/process-updates/3 Pipeline is pending
ci/crow/cron/process-updates/13 Pipeline is pending
ci/crow/cron/process-updates/15 Pipeline is pending
ci/crow/manual/build-all-versions/4 Pipeline was canceled
ci/crow/manual/build-all-versions/1 Pipeline was canceled
ci/crow/manual/build-all-versions/5 Pipeline was canceled
ci/crow/manual/build-all-versions/8 Pipeline was canceled
ci/crow/manual/build-all-versions/6 Pipeline was canceled
ci/crow/manual/build-all-versions/7 Pipeline was canceled
ci/crow/cron/process-updates/10 Pipeline is running
ci/crow/manual/build-all-versions/3 Pipeline was canceled
ci/crow/manual/build-all-versions/2 Pipeline was canceled
ci/crow/cron/process-updates/2 Pipeline failed
ci/crow/cron/process-updates/14 Pipeline is running
ci/crow/cron/process-updates/4 Pipeline failed
ci/crow/cron/process-updates/16 Pipeline is running
ci/crow/cron/process-updates/8 Pipeline failed
ci/crow/cron/process-updates/18 Pipeline is pending
ci/crow/cron/process-updates/11 Pipeline was successful
## Motivation
`arm64/alpine324` reported nothing to build while thousands were missing:
```
line 49: Precomputed 7192 package versions (6871 r-minor-sensitive) <- install-deps agent
line 99: Total# of remaining package versions: 43 (sensitive_only=TRUE) <- a build shard
line 101: Skipped 0 package versions already attempted under R 4.4; 0 remaining
```
Both numbers come from the **same pipeline**. The same run's index step dropped 2407 packages as missing for 4.4 and 2436 for 4.6.
## Cause
```r
if (!all(file.exists(package_cache_files))) { ... recompute ... }
```
Existence is not freshness. The snapshot describes S3 and CRAN state when it was written, and the cache volume is per-agent — the file's own comment says so. An agent that ran an earlier pipeline keeps serving that pipeline's answer forever, and no later fix to how the snapshot is computed (#189) can reach it.
## Change
Recompute when the snapshot is stale as well as when it is missing. Keyed on the pipeline when the CI exposes an identifier (`CI_PIPELINE_NUMBER`, `CI_BUILD_NUMBER`, `CI_PIPELINE_ID`), so a new pipeline recomputes once per agent and its shards then share the result. Off CI, or when none is set, an age check with a two hour default (`PACKAGE_SNAPSHOT_TTL_HOURS`).
## Verification
| scenario | decision |
|---|---|
| files missing | RECOMPUTE |
| same pipeline id | reuse |
| **new pipeline id** | **RECOMPUTE** |
| no CI var, recent file | reuse |
| no CI var, aged out | RECOMPUTE |
I could not confirm which identifier Crow actually sets — none is referenced anywhere in this repo — so all three are tried and the age check backs them up. If none is present the behaviour is the age path, which is still correct, just coarser.
Reviewed-on: #190
This commit is contained in:
parent
94e6c697cf
commit
642e07e1d6
1 changed files with 46 additions and 2 deletions
|
|
@ -26,9 +26,50 @@ package_cache_files <- c(
|
||||||
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds",
|
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds",
|
||||||
"/mnt/cache/packages/s3_cache.rds"
|
"/mnt/cache/packages/s3_cache.rds"
|
||||||
)
|
)
|
||||||
if (!all(file.exists(package_cache_files))) {
|
# Existence is not freshness. The snapshot describes S3 and CRAN state at the
|
||||||
|
# moment it was written, and the volume is per-agent, so an agent that ran an
|
||||||
|
# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324
|
||||||
|
# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43
|
||||||
|
# sensitive packages, while the install-deps step in the very same pipeline had
|
||||||
|
# just computed 6871 on another agent.
|
||||||
|
#
|
||||||
|
# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes
|
||||||
|
# once per agent and its shards then share the result. Off CI, or when no such
|
||||||
|
# variable is set, fall back to an age check.
|
||||||
|
snapshot_id_path <- "/mnt/cache/packages/snapshot.id"
|
||||||
|
snapshot_ttl_hours <- as.numeric(
|
||||||
|
Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2")
|
||||||
|
)
|
||||||
|
current_snapshot_id <- ""
|
||||||
|
for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) {
|
||||||
|
val <- Sys.getenv(v, unset = "")
|
||||||
|
if (nzchar(val)) {
|
||||||
|
current_snapshot_id <- paste(v, val, sep = "=")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot_is_stale <- function() {
|
||||||
|
if (!all(file.exists(package_cache_files))) {
|
||||||
|
return(TRUE)
|
||||||
|
}
|
||||||
|
if (nzchar(current_snapshot_id)) {
|
||||||
|
cached <- tryCatch(
|
||||||
|
readLines(snapshot_id_path, warn = FALSE)[1L],
|
||||||
|
error = function(e) NA_character_,
|
||||||
|
warning = function(w) NA_character_
|
||||||
|
)
|
||||||
|
return(!identical(cached, current_snapshot_id))
|
||||||
|
}
|
||||||
|
age_hours <- as.numeric(
|
||||||
|
difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours")
|
||||||
|
)
|
||||||
|
isTRUE(age_hours > snapshot_ttl_hours)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (snapshot_is_stale()) {
|
||||||
message(
|
message(
|
||||||
"Package snapshot missing from cache; recomputing via packages-to-build.R"
|
"Package snapshot missing or stale; recomputing via packages-to-build.R"
|
||||||
)
|
)
|
||||||
dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
|
dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
|
||||||
save_rds_atomic <- function(obj, path) {
|
save_rds_atomic <- function(obj, path) {
|
||||||
|
|
@ -42,6 +83,9 @@ if (!all(file.exists(package_cache_files))) {
|
||||||
pkgs[r_minor_sensitive == TRUE],
|
pkgs[r_minor_sensitive == TRUE],
|
||||||
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds"
|
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds"
|
||||||
)
|
)
|
||||||
|
if (nzchar(current_snapshot_id)) {
|
||||||
|
writeLines(current_snapshot_id, snapshot_id_path)
|
||||||
|
}
|
||||||
message("Package snapshot recomputed.")
|
message("Package snapshot recomputed.")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue