fix(abi): inspect flat objects of packages CRAN has removed (#212)
Some checks failed
ci/crow/manual/backfill-flat-safety/5 Pipeline was successful
ci/crow/manual/backfill-flat-safety/3 Pipeline was successful
ci/crow/manual/backfill-flat-safety/17 Pipeline was successful
ci/crow/manual/backfill-flat-safety/13 Pipeline was successful
ci/crow/manual/backfill-flat-safety/15 Pipeline was successful
ci/crow/manual/backfill-flat-safety/7 Pipeline was successful
ci/crow/manual/backfill-flat-safety/1 Pipeline was successful
ci/crow/manual/backfill-flat-safety/9 Pipeline was successful
ci/crow/manual/backfill-flat-safety/11 Pipeline was successful
ci/crow/manual/backfill-flat-safety/8 Pipeline was successful
ci/crow/manual/backfill-flat-safety/12 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline failed
ci/crow/manual/backfill-flat-safety/2 Pipeline was successful
ci/crow/manual/backfill-flat-safety/10 Pipeline was successful
ci/crow/manual/backfill-flat-safety/14 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline was successful
ci/crow/manual/backfill-flat-safety/18 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/manual/backfill-flat-safety/16 Pipeline was successful
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/18 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/17 Pipeline was successful
ci/crow/manual/backfill-flat-safety/6 Pipeline was successful
ci/crow/manual/backfill-flat-safety/4 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/13 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/15 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/9 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/10 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/7 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/8 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/14 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/16 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/17 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/12 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/18 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/11 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/6 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/4 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/5 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/3 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/2 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/1 Pipeline was successful
ci/crow/manual/reindex/13 Pipeline was successful
ci/crow/manual/reindex/7 Pipeline was successful
ci/crow/manual/reindex/11 Pipeline was successful
ci/crow/manual/reindex/15 Pipeline was successful
ci/crow/manual/reindex/17 Pipeline was successful
ci/crow/manual/reindex/12 Pipeline was successful
ci/crow/manual/reindex/9 Pipeline was successful
ci/crow/manual/reindex/3 Pipeline was successful
ci/crow/manual/reindex/14 Pipeline was successful
ci/crow/manual/reindex/1 Pipeline was successful
ci/crow/manual/reindex/2 Pipeline was successful
ci/crow/manual/reindex/4 Pipeline was successful
ci/crow/manual/reindex/6 Pipeline was successful
ci/crow/manual/reindex/5 Pipeline was successful
ci/crow/manual/reindex/16 Pipeline was successful
ci/crow/manual/reindex/18 Pipeline was successful
ci/crow/manual/reindex/8 Pipeline was successful
ci/crow/manual/reindex/10 Pipeline was successful
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/1 Pipeline was successful
ci/crow/manual/verify-repo-routing/13 Pipeline was successful
ci/crow/manual/verify-repo-routing/3 Pipeline was successful
ci/crow/manual/verify-repo-routing/1 Pipeline failed
ci/crow/manual/verify-repo-routing/15 Pipeline was successful
ci/crow/manual/verify-repo-routing/2 Pipeline failed
ci/crow/manual/verify-repo-routing/4 Pipeline was successful
ci/crow/manual/verify-repo-routing/17 Pipeline was successful
ci/crow/manual/verify-repo-routing/5 Pipeline was successful
ci/crow/manual/verify-repo-routing/6 Pipeline was successful
ci/crow/manual/verify-repo-routing/9 Pipeline was successful
ci/crow/manual/verify-repo-routing/11 Pipeline was successful
ci/crow/manual/verify-repo-routing/14 Pipeline was successful
ci/crow/manual/verify-repo-routing/16 Pipeline was successful
ci/crow/manual/verify-repo-routing/10 Pipeline was successful
ci/crow/manual/verify-repo-routing/12 Pipeline was successful
ci/crow/manual/verify-repo-routing/7 Pipeline was successful
ci/crow/manual/verify-repo-routing/8 Pipeline was successful
ci/crow/manual/verify-repo-routing/18 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
Some checks failed
ci/crow/manual/backfill-flat-safety/5 Pipeline was successful
ci/crow/manual/backfill-flat-safety/3 Pipeline was successful
ci/crow/manual/backfill-flat-safety/17 Pipeline was successful
ci/crow/manual/backfill-flat-safety/13 Pipeline was successful
ci/crow/manual/backfill-flat-safety/15 Pipeline was successful
ci/crow/manual/backfill-flat-safety/7 Pipeline was successful
ci/crow/manual/backfill-flat-safety/1 Pipeline was successful
ci/crow/manual/backfill-flat-safety/9 Pipeline was successful
ci/crow/manual/backfill-flat-safety/11 Pipeline was successful
ci/crow/manual/backfill-flat-safety/8 Pipeline was successful
ci/crow/manual/backfill-flat-safety/12 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline failed
ci/crow/manual/backfill-flat-safety/2 Pipeline was successful
ci/crow/manual/backfill-flat-safety/10 Pipeline was successful
ci/crow/manual/backfill-flat-safety/14 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline was successful
ci/crow/manual/backfill-flat-safety/18 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/manual/backfill-flat-safety/16 Pipeline was successful
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/18 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/17 Pipeline was successful
ci/crow/manual/backfill-flat-safety/6 Pipeline was successful
ci/crow/manual/backfill-flat-safety/4 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/13 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/15 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/9 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/10 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/7 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/8 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/14 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/16 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/17 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/12 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/18 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/11 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/6 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/4 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/5 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/3 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/2 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/1 Pipeline was successful
ci/crow/manual/reindex/13 Pipeline was successful
ci/crow/manual/reindex/7 Pipeline was successful
ci/crow/manual/reindex/11 Pipeline was successful
ci/crow/manual/reindex/15 Pipeline was successful
ci/crow/manual/reindex/17 Pipeline was successful
ci/crow/manual/reindex/12 Pipeline was successful
ci/crow/manual/reindex/9 Pipeline was successful
ci/crow/manual/reindex/3 Pipeline was successful
ci/crow/manual/reindex/14 Pipeline was successful
ci/crow/manual/reindex/1 Pipeline was successful
ci/crow/manual/reindex/2 Pipeline was successful
ci/crow/manual/reindex/4 Pipeline was successful
ci/crow/manual/reindex/6 Pipeline was successful
ci/crow/manual/reindex/5 Pipeline was successful
ci/crow/manual/reindex/16 Pipeline was successful
ci/crow/manual/reindex/18 Pipeline was successful
ci/crow/manual/reindex/8 Pipeline was successful
ci/crow/manual/reindex/10 Pipeline was successful
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/1 Pipeline was successful
ci/crow/manual/verify-repo-routing/13 Pipeline was successful
ci/crow/manual/verify-repo-routing/3 Pipeline was successful
ci/crow/manual/verify-repo-routing/1 Pipeline failed
ci/crow/manual/verify-repo-routing/15 Pipeline was successful
ci/crow/manual/verify-repo-routing/2 Pipeline failed
ci/crow/manual/verify-repo-routing/4 Pipeline was successful
ci/crow/manual/verify-repo-routing/17 Pipeline was successful
ci/crow/manual/verify-repo-routing/5 Pipeline was successful
ci/crow/manual/verify-repo-routing/6 Pipeline was successful
ci/crow/manual/verify-repo-routing/9 Pipeline was successful
ci/crow/manual/verify-repo-routing/11 Pipeline was successful
ci/crow/manual/verify-repo-routing/14 Pipeline was successful
ci/crow/manual/verify-repo-routing/16 Pipeline was successful
ci/crow/manual/verify-repo-routing/10 Pipeline was successful
ci/crow/manual/verify-repo-routing/12 Pipeline was successful
ci/crow/manual/verify-repo-routing/7 Pipeline was successful
ci/crow/manual/verify-repo-routing/8 Pipeline was successful
ci/crow/manual/verify-repo-routing/18 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
## Why The backfill decided which flat objects were worth inspecting by looking the package up in **CRAN's current index**: ```r compiled_names <- cran[NeedsCompilation == "yes", "Package"] keep <- flat_pkg %in% compiled_names ``` CRAN lists only what it currently ships. A package CRAN has **archived** is absent from that index, so it is never recognised as compiled and never inspected, and no verdict is ever recorded for it. `union_index_records()` meanwhile reads the slot's *own* `NeedsCompilation`, finds `yes`, and drops the record because the source classifier flags the package and no safe verdict exists to clear it. The repair cannot rescue it either: it works from recorded verdicts, and there is no row. The result is unrecoverable, and identical in shape to #132 but from a different cause: dropped from every per-minor index, on every re-index, forever. ## Evidence `mixedMem` and `vcpen` survived a backfill, two re-indexes and a repair while missing from all three rhel8 per-minor indexes on **both** architectures: ``` rhel8 amd64 4.5: missing=3 [mixedMem timbr vcpen] rhel8 arm64 4.5: missing=2 [mixedMem vcpen] ``` Neither appears anywhere in either repair log, because neither was ever in the unsafe set. Their per-minor objects are 404 at every minor. And in CRAN's index: ``` mixedMem CRAN version=ABSENT NeedsCompilation=ABSENT vcpen CRAN version=ABSENT NeedsCompilation=ABSENT ``` Both removed from CRAN. Their flat records in our own index say `NeedsCompilation: yes`. ## What changed `NeedsCompilation` now comes from the slot's own flat index, which describes exactly the objects being inspected rather than a different repository's opinion of them. - Fetched **from the bucket**, not over HTTPS. This runs inside R, so the edge would route the request to a per-minor index and the flat objects would not all be there. Same reason the object listing above it comes from S3 (#203). - Keyed on `<package>_<version>`, not the package name: a slot can carry several versions and only some of them compile. - Refuses to proceed on an empty read, so an unreadable index cannot silently mean "nothing is compiled" and skip the entire slot. ## Effect Re-running the backfill on a slot will now inspect these objects and record verdicts, after which the index writer keeps the safe ones and the repair can act on the unsafe ones. Already-recorded objects are still skipped, so re-running stays cheap. Reviewed-on: #212
This commit is contained in:
parent
03e007069a
commit
523424bd68
1 changed files with 39 additions and 11 deletions
|
|
@ -75,19 +75,47 @@ flat_pkg <- sub("_.*$", "", flat_files)
|
|||
flat_ver <- sub("\\.tar\\.gz$", "", sub("^[^_]*_", "", flat_files))
|
||||
|
||||
# Only compiled packages can fail at dyn.load(), so the rest are not worth
|
||||
# downloading. CRAN's own index says which; it is not behind our edge.
|
||||
cran <- read.dcf(
|
||||
gzcon(url(
|
||||
"https://cloud.r-project.org/src/contrib/PACKAGES.gz",
|
||||
open = "rb"
|
||||
)),
|
||||
fields = c("Package", "NeedsCompilation")
|
||||
# downloading. Ask the slot's own index rather than CRAN's: CRAN lists only the
|
||||
# versions it currently ships, so a package it has archived is absent, is never
|
||||
# recognised as compiled, and is never inspected. No verdict is ever recorded
|
||||
# for it, while `union_index_records()` reads the slot's own `NeedsCompilation`
|
||||
# and happily drops it -- with nothing able to clear the flag, because the
|
||||
# repair works from recorded verdicts too. `mixedMem` and `vcpen`, both removed
|
||||
# from CRAN, were stuck out of every rhel8 per-minor index that way.
|
||||
#
|
||||
# Read it from the bucket, not over HTTPS: this runs inside R, so the edge would
|
||||
# route the request to a per-minor index and the flat objects would not all be
|
||||
# there. Same reason the object listing above comes from S3.
|
||||
index_file <- file.path(tempdir(), "flat-PACKAGES.gz")
|
||||
s3fs::s3_file_download(
|
||||
sprintf("%s/PACKAGES.gz", contrib),
|
||||
index_file,
|
||||
overwrite = TRUE
|
||||
)
|
||||
compiled_names <- cran[
|
||||
!is.na(cran[, "NeedsCompilation"]) & cran[, "NeedsCompilation"] == "yes",
|
||||
"Package"
|
||||
index <- read.dcf(
|
||||
gzfile(index_file),
|
||||
fields = c("Package", "Version", "NeedsCompilation")
|
||||
)
|
||||
if (nrow(index) == 0L) {
|
||||
stop(
|
||||
sprintf(
|
||||
"Read no records from %s/PACKAGES.gz; refusing to guess what is compiled.",
|
||||
contrib
|
||||
),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
# Key on the object, not the package: a slot can carry several versions and
|
||||
# only some of them compile.
|
||||
compiled_objects <- paste(
|
||||
index[, "Package"],
|
||||
index[, "Version"],
|
||||
sep = "_"
|
||||
)[
|
||||
!is.na(index[, "NeedsCompilation"]) &
|
||||
tolower(index[, "NeedsCompilation"]) == "yes"
|
||||
]
|
||||
keep <- flat_pkg %in% compiled_names
|
||||
keep <- paste(flat_pkg, flat_ver, sep = "_") %in% compiled_objects
|
||||
compiled <- data.frame(
|
||||
Package = flat_pkg[keep],
|
||||
Version = flat_ver[keep],
|
||||
|
|
|
|||
Loading…
Reference in a new issue