From d9a652904e8885820138c2488f46ddb60874e503 Mon Sep 17 00:00:00 2001
From: pat-s <patrick.schratz@gmail.com>
Date: Wed, 9 Sep 2026 18:26:38 +0000
Subject: [PATCH] fix(abi): inspect flat objects of packages CRAN has removed

The backfill decided which objects were worth inspecting by looking the package
up in CRAN's current index. CRAN lists only what it currently ships, so a
package it has archived is absent, is never recognised as compiled, and is
never inspected. No verdict is ever recorded for it.

`union_index_records()` meanwhile reads the slot's *own* `NeedsCompilation`,
finds `yes`, and drops the record when the source classifier flags the package
and no safe verdict exists to clear it. The repair cannot rescue it either,
because it works from recorded verdicts and there is no row.

The record is therefore dropped from every per-minor index on every re-index,
permanently. `mixedMem` and `vcpen`, both removed from CRAN, were stuck out of
all three rhel8 per-minor indexes on both architectures that way, surviving a
backfill, two re-indexes and a repair.

Read `NeedsCompilation` from the slot's own flat index instead, which describes
exactly the objects being inspected. It is fetched from the bucket rather than
over HTTPS because this runs inside R, where the edge would route the request
to a per-minor index.

Keyed on `<package>_<version>` rather than the package name: a slot can carry
several versions and only some of them compile.
---
 local/backfill-flat-safety.R | 50 ++++++++++++++++++++++++++++--------
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/local/backfill-flat-safety.R b/local/backfill-flat-safety.R
index a8a8bbe..59faa3c 100644
--- a/local/backfill-flat-safety.R
+++ b/local/backfill-flat-safety.R
@@ -75,19 +75,47 @@ flat_pkg <- sub("_.*$", "", flat_files)
 flat_ver <- sub("\\.tar\\.gz$", "", sub("^[^_]*_", "", flat_files))
 
 # Only compiled packages can fail at dyn.load(), so the rest are not worth
-# downloading. CRAN's own index says which; it is not behind our edge.
-cran <- read.dcf(
-  gzcon(url(
-    "https://cloud.r-project.org/src/contrib/PACKAGES.gz",
-    open = "rb"
-  )),
-  fields = c("Package", "NeedsCompilation")
+# downloading. Ask the slot's own index rather than CRAN's: CRAN lists only the
+# versions it currently ships, so a package it has archived is absent, is never
+# recognised as compiled, and is never inspected. No verdict is ever recorded
+# for it, while `union_index_records()` reads the slot's own `NeedsCompilation`
+# and happily drops it -- with nothing able to clear the flag, because the
+# repair works from recorded verdicts too. `mixedMem` and `vcpen`, both removed
+# from CRAN, were stuck out of every rhel8 per-minor index that way.
+#
+# Read it from the bucket, not over HTTPS: this runs inside R, so the edge would
+# route the request to a per-minor index and the flat objects would not all be
+# there. Same reason the object listing above comes from S3.
+index_file <- file.path(tempdir(), "flat-PACKAGES.gz")
+s3fs::s3_file_download(
+  sprintf("%s/PACKAGES.gz", contrib),
+  index_file,
+  overwrite = TRUE
 )
-compiled_names <- cran[
-  !is.na(cran[, "NeedsCompilation"]) & cran[, "NeedsCompilation"] == "yes",
-  "Package"
+index <- read.dcf(
+  gzfile(index_file),
+  fields = c("Package", "Version", "NeedsCompilation")
+)
+if (nrow(index) == 0L) {
+  stop(
+    sprintf(
+      "Read no records from %s/PACKAGES.gz; refusing to guess what is compiled.",
+      contrib
+    ),
+    call. = FALSE
+  )
+}
+# Key on the object, not the package: a slot can carry several versions and
+# only some of them compile.
+compiled_objects <- paste(
+  index[, "Package"],
+  index[, "Version"],
+  sep = "_"
+)[
+  !is.na(index[, "NeedsCompilation"]) &
+    tolower(index[, "NeedsCompilation"]) == "yes"
 ]
-keep <- flat_pkg %in% compiled_names
+keep <- paste(flat_pkg, flat_ver, sep = "_") %in% compiled_objects
 compiled <- data.frame(
   Package = flat_pkg[keep],
   Version = flat_ver[keep],
-- 
2.54.0