local/backfill-flat-safety.R

@@ -75,19 +75,47 @@ flat_pkg <- sub("_.*$", "", flat_files)
 flat_ver <- sub("\\.tar\\.gz$", "", sub("^[^_]*_", "", flat_files))
 
 # Only compiled packages can fail at dyn.load(), so the rest are not worth
-# downloading. CRAN's own index says which; it is not behind our edge.
-cran <- read.dcf(
-  gzcon(url(
-    "https://cloud.r-project.org/src/contrib/PACKAGES.gz",
-    open = "rb"
-  )),
-  fields = c("Package", "NeedsCompilation")
+# downloading. Ask the slot's own index rather than CRAN's: CRAN lists only the
+# versions it currently ships, so a package it has archived is absent, is never
+# recognised as compiled, and is never inspected. No verdict is ever recorded
+# for it, while `union_index_records()` reads the slot's own `NeedsCompilation`
+# and happily drops it -- with nothing able to clear the flag, because the
+# repair works from recorded verdicts too. `mixedMem` and `vcpen`, both removed
+# from CRAN, were stuck out of every rhel8 per-minor index that way.
+#
+# Read it from the bucket, not over HTTPS: this runs inside R, so the edge would
+# route the request to a per-minor index and the flat objects would not all be
+# there. Same reason the object listing above comes from S3.
+index_file <- file.path(tempdir(), "flat-PACKAGES.gz")
+s3fs::s3_file_download(
+  sprintf("%s/PACKAGES.gz", contrib),
+  index_file,
+  overwrite = TRUE
 )
-compiled_names <- cran[
-  !is.na(cran[, "NeedsCompilation"]) & cran[, "NeedsCompilation"] == "yes",
-  "Package"
+index <- read.dcf(
+  gzfile(index_file),
+  fields = c("Package", "Version", "NeedsCompilation")
+)
+if (nrow(index) == 0L) {
+  stop(
+    sprintf(
+      "Read no records from %s/PACKAGES.gz; refusing to guess what is compiled.",
+      contrib
+    ),
+    call. = FALSE
+  )
+}
+# Key on the object, not the package: a slot can carry several versions and
+# only some of them compile.
+compiled_objects <- paste(
+  index[, "Package"],
+  index[, "Version"],
+  sep = "_"
+)[
+  !is.na(index[, "NeedsCompilation"]) &
+    tolower(index[, "NeedsCompilation"]) == "yes"
 ]
-keep <- flat_pkg %in% compiled_names
+keep <- paste(flat_pkg, flat_ver, sep = "_") %in% compiled_objects
 compiled <- data.frame(
   Package = flat_pkg[keep],
   Version = flat_ver[keep],