From 523424bd68775557e18439928e2e481ededf70c1 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 9 Sep 2026 18:35:33 +0000 Subject: [PATCH] fix(abi): inspect flat objects of packages CRAN has removed (#212) ## Why The backfill decided which flat objects were worth inspecting by looking the package up in **CRAN's current index**: ```r compiled_names <- cran[NeedsCompilation == "yes", "Package"] keep <- flat_pkg %in% compiled_names ``` CRAN lists only what it currently ships. A package CRAN has **archived** is absent from that index, so it is never recognised as compiled and never inspected, and no verdict is ever recorded for it. `union_index_records()` meanwhile reads the slot's *own* `NeedsCompilation`, finds `yes`, and drops the record because the source classifier flags the package and no safe verdict exists to clear it. The repair cannot rescue it either: it works from recorded verdicts, and there is no row. The result is unrecoverable, and identical in shape to #132 but from a different cause: dropped from every per-minor index, on every re-index, forever. ## Evidence `mixedMem` and `vcpen` survived a backfill, two re-indexes and a repair while missing from all three rhel8 per-minor indexes on **both** architectures: ``` rhel8 amd64 4.5: missing=3 [mixedMem timbr vcpen] rhel8 arm64 4.5: missing=2 [mixedMem vcpen] ``` Neither appears anywhere in either repair log, because neither was ever in the unsafe set. Their per-minor objects are 404 at every minor. And in CRAN's index: ``` mixedMem CRAN version=ABSENT NeedsCompilation=ABSENT vcpen CRAN version=ABSENT NeedsCompilation=ABSENT ``` Both removed from CRAN. Their flat records in our own index say `NeedsCompilation: yes`. ## What changed `NeedsCompilation` now comes from the slot's own flat index, which describes exactly the objects being inspected rather than a different repository's opinion of them. - Fetched **from the bucket**, not over HTTPS. This runs inside R, so the edge would route the request to a per-minor index and the flat objects would not all be there. Same reason the object listing above it comes from S3 (#203). - Keyed on `_`, not the package name: a slot can carry several versions and only some of them compile. - Refuses to proceed on an empty read, so an unreadable index cannot silently mean "nothing is compiled" and skip the entire slot. ## Effect Re-running the backfill on a slot will now inspect these objects and record verdicts, after which the index writer keeps the safe ones and the repair can act on the unsafe ones. Already-recorded objects are still skipped, so re-running stays cheap. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/212 --- local/backfill-flat-safety.R | 50 ++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/local/backfill-flat-safety.R b/local/backfill-flat-safety.R index a8a8bbe..59faa3c 100644 --- a/local/backfill-flat-safety.R +++ b/local/backfill-flat-safety.R @@ -75,19 +75,47 @@ flat_pkg <- sub("_.*$", "", flat_files) flat_ver <- sub("\\.tar\\.gz$", "", sub("^[^_]*_", "", flat_files)) # Only compiled packages can fail at dyn.load(), so the rest are not worth -# downloading. CRAN's own index says which; it is not behind our edge. -cran <- read.dcf( - gzcon(url( - "https://cloud.r-project.org/src/contrib/PACKAGES.gz", - open = "rb" - )), - fields = c("Package", "NeedsCompilation") +# downloading. Ask the slot's own index rather than CRAN's: CRAN lists only the +# versions it currently ships, so a package it has archived is absent, is never +# recognised as compiled, and is never inspected. No verdict is ever recorded +# for it, while `union_index_records()` reads the slot's own `NeedsCompilation` +# and happily drops it -- with nothing able to clear the flag, because the +# repair works from recorded verdicts too. `mixedMem` and `vcpen`, both removed +# from CRAN, were stuck out of every rhel8 per-minor index that way. +# +# Read it from the bucket, not over HTTPS: this runs inside R, so the edge would +# route the request to a per-minor index and the flat objects would not all be +# there. Same reason the object listing above comes from S3. +index_file <- file.path(tempdir(), "flat-PACKAGES.gz") +s3fs::s3_file_download( + sprintf("%s/PACKAGES.gz", contrib), + index_file, + overwrite = TRUE ) -compiled_names <- cran[ - !is.na(cran[, "NeedsCompilation"]) & cran[, "NeedsCompilation"] == "yes", - "Package" +index <- read.dcf( + gzfile(index_file), + fields = c("Package", "Version", "NeedsCompilation") +) +if (nrow(index) == 0L) { + stop( + sprintf( + "Read no records from %s/PACKAGES.gz; refusing to guess what is compiled.", + contrib + ), + call. = FALSE + ) +} +# Key on the object, not the package: a slot can carry several versions and +# only some of them compile. +compiled_objects <- paste( + index[, "Package"], + index[, "Version"], + sep = "_" +)[ + !is.na(index[, "NeedsCompilation"]) & + tolower(index[, "NeedsCompilation"]) == "yes" ] -keep <- flat_pkg %in% compiled_names +keep <- paste(flat_pkg, flat_ver, sep = "_") %in% compiled_objects compiled <- data.frame( Package = flat_pkg[keep], Version = flat_ver[keep],