local/packages-to-build.R

@@ -154,26 +154,30 @@ index_url <- sprintf(
   arch,
   codename
 )
-source_served <- tryCatch(
-  {
-    con_idx <- gzcon(url(index_url, open = "rb"))
-    on.exit(close(con_idx), add = TRUE)
-    idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built"))
-    sprintf(
-      "%s_%s.tar.gz",
-      idx[is.na(idx[, "Built"]), "Package"],
-      idx[is.na(idx[, "Built"]), "Version"]
-    )
-  },
-  error = function(e) {
-    cat(sprintf(
-      "WARNING: could not read %s (%s); keeping the full S3 cache\n",
-      index_url,
-      conditionMessage(e)
-    ))
-    character(0)
-  }
-)
+read_source_served <- function(index) {
+  tryCatch(
+    {
+      con_idx <- gzcon(url(index, open = "rb"))
+      on.exit(close(con_idx), add = TRUE)
+      idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built"))
+      sprintf(
+        "%s_%s.tar.gz",
+        idx[is.na(idx[, "Built"]), "Package"],
+        idx[is.na(idx[, "Built"]), "Version"]
+      )
+    },
+    error = function(e) {
+      cat(sprintf(
+        "WARNING: could not read %s (%s); keeping that index's cache in full\n",
+        index,
+        conditionMessage(e)
+      ))
+      character(0)
+    }
+  )
+}
+
+source_served <- read_source_served(index_url)
 
 binary_cache <- setdiff(file_names, source_served)
 
@@ -225,9 +229,65 @@ if (sum(stripped_per_minor) != sum(per_minor_object)) {
   ))
 }
 source_basenames <- source_served
-existence_cache <- relative_paths[
-  !basename(relative_paths) %in% source_basenames
-]
+
+# A source-fallback list describes ONE index, so it may only be applied to that
+# index's objects. Applied to every path it deletes real per-minor binaries from
+# the cache: a package the flat slot serves as CRAN source still has a genuine
+# `4.4/<pkg>.tar.gz`, and dropping that name makes every per-minor pass rebuild
+# it, every run, forever.
+#
+# The overlap is total rather than partial, which is why this pinned the skip
+# rate near 0%: the sensitive candidate list is exactly "sensitive packages with
+# no flat binary", which is the same set this was removing. amd64/resolute
+# listed 21212 per-minor objects but cached only 13572, then recompiled osmdata,
+# osqp and outbreaker2 while their 4.4 binaries sat in the bucket.
+#
+# So judge each minor by its own index, and the flat slot by the flat index.
+is_per_minor_path <- grepl("^[0-9]+\\.[0-9]+/", relative_paths)
+path_minor <- ifelse(
+  is_per_minor_path,
+  sub("^([0-9]+\\.[0-9]+)/.*$", "\\1", relative_paths),
+  ""
+)
+minors_present <- sort(unique(path_minor[is_per_minor_path]))
+per_minor_source <- lapply(minors_present, function(m) {
+  read_source_served(sprintf(
+    "https://cran.rpkgs.com/%s/%s/latest/src/contrib/%s/PACKAGES.gz",
+    arch,
+    codename,
+    m
+  ))
+})
+names(per_minor_source) <- minors_present
+cat(sprintf(
+  "Source fallbacks per index: flat=%d%s\n",
+  length(source_basenames),
+  if (length(minors_present)) {
+    paste0(
+      ", ",
+      paste(
+        sprintf("%s=%d", minors_present, lengths(per_minor_source)),
+        collapse = ", "
+      )
+    )
+  } else {
+    ""
+  }
+))
+
+served_as_source <- vapply(
+  seq_along(relative_paths),
+  function(i) {
+    scope <- if (nzchar(path_minor[i])) {
+      per_minor_source[[path_minor[i]]]
+    } else {
+      source_basenames
+    }
+    basename(relative_paths[i]) %in% scope
+  },
+  logical(1L)
+)
+existence_cache <- relative_paths[!served_as_source]
 cat(sprintf(
   "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n",
   length(file_names),