From 642e07e1d6b327e9ecb435c06100786ecf0ce060 Mon Sep 17 00:00:00 2001
From: pat-s <patrick.schratz@gmail.com>
Date: Mon, 31 Aug 2026 22:11:17 +0000
Subject: [PATCH] fix(build): recompute a stale package snapshot, not just a
 missing one (#190)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Motivation

`arm64/alpine324` reported nothing to build while thousands were missing:

```
line  49: Precomputed 7192 package versions (6871 r-minor-sensitive)   <- install-deps agent
line  99: Total# of remaining package versions: 43 (sensitive_only=TRUE) <- a build shard
line 101: Skipped 0 package versions already attempted under R 4.4; 0 remaining
```

Both numbers come from the **same pipeline**. The same run's index step dropped 2407 packages as missing for 4.4 and 2436 for 4.6.

## Cause

```r
if (!all(file.exists(package_cache_files))) { ... recompute ... }
```

Existence is not freshness. The snapshot describes S3 and CRAN state when it was written, and the cache volume is per-agent — the file's own comment says so. An agent that ran an earlier pipeline keeps serving that pipeline's answer forever, and no later fix to how the snapshot is computed (#189) can reach it.

## Change

Recompute when the snapshot is stale as well as when it is missing. Keyed on the pipeline when the CI exposes an identifier (`CI_PIPELINE_NUMBER`, `CI_BUILD_NUMBER`, `CI_PIPELINE_ID`), so a new pipeline recomputes once per agent and its shards then share the result. Off CI, or when none is set, an age check with a two hour default (`PACKAGE_SNAPSHOT_TTL_HOURS`).

## Verification

| scenario | decision |
|---|---|
| files missing | RECOMPUTE |
| same pipeline id | reuse |
| **new pipeline id** | **RECOMPUTE** |
| no CI var, recent file | reuse |
| no CI var, aged out | RECOMPUTE |

I could not confirm which identifier Crow actually sets — none is referenced anywhere in this repo — so all three are tried and the age check backs them up. If none is present the behaviour is the age path, which is still correct, just coarser.

Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/190
---
 local/build-all.R | 48 +++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 46 insertions(+), 2 deletions(-)

diff --git a/local/build-all.R b/local/build-all.R
index 18b0a52..c4d7f97 100644
--- a/local/build-all.R
+++ b/local/build-all.R
@@ -26,9 +26,50 @@ package_cache_files <- c(
   "/mnt/cache/packages/r_minor_sensitive_pkgs.rds",
   "/mnt/cache/packages/s3_cache.rds"
 )
-if (!all(file.exists(package_cache_files))) {
+# Existence is not freshness. The snapshot describes S3 and CRAN state at the
+# moment it was written, and the volume is per-agent, so an agent that ran an
+# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324
+# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43
+# sensitive packages, while the install-deps step in the very same pipeline had
+# just computed 6871 on another agent.
+#
+# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes
+# once per agent and its shards then share the result. Off CI, or when no such
+# variable is set, fall back to an age check.
+snapshot_id_path <- "/mnt/cache/packages/snapshot.id"
+snapshot_ttl_hours <- as.numeric(
+  Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2")
+)
+current_snapshot_id <- ""
+for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) {
+  val <- Sys.getenv(v, unset = "")
+  if (nzchar(val)) {
+    current_snapshot_id <- paste(v, val, sep = "=")
+    break
+  }
+}
+
+snapshot_is_stale <- function() {
+  if (!all(file.exists(package_cache_files))) {
+    return(TRUE)
+  }
+  if (nzchar(current_snapshot_id)) {
+    cached <- tryCatch(
+      readLines(snapshot_id_path, warn = FALSE)[1L],
+      error = function(e) NA_character_,
+      warning = function(w) NA_character_
+    )
+    return(!identical(cached, current_snapshot_id))
+  }
+  age_hours <- as.numeric(
+    difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours")
+  )
+  isTRUE(age_hours > snapshot_ttl_hours)
+}
+
+if (snapshot_is_stale()) {
   message(
-    "Package snapshot missing from cache; recomputing via packages-to-build.R"
+    "Package snapshot missing or stale; recomputing via packages-to-build.R"
   )
   dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
   save_rds_atomic <- function(obj, path) {
@@ -42,6 +83,9 @@ if (!all(file.exists(package_cache_files))) {
     pkgs[r_minor_sensitive == TRUE],
     "/mnt/cache/packages/r_minor_sensitive_pkgs.rds"
   )
+  if (nzchar(current_snapshot_id)) {
+    writeLines(current_snapshot_id, snapshot_id_path)
+  }
   message("Package snapshot recomputed.")
 }