pat-s
767dd9ee5dThe backfill decided which objects were worth inspecting by looking the package up in CRAN's current index. CRAN lists only what it currently ships, so a package it has archived is absent, is never recognised as compiled, and is never inspected. No verdict is ever recorded for it. `union_index_records()` meanwhile reads the slot's *own* `NeedsCompilation`, finds `yes`, and drops the record when the source classifier flags the package and no safe verdict exists to clear it. The repair cannot rescue it either, because it works from recorded verdicts and there is no row. The record is therefore dropped from every per-minor index on every re-index, permanently. `mixedMem` and `vcpen`, both removed from CRAN, were stuck out of all three rhel8 per-minor indexes on both architectures that way, surviving a backfill, two re-indexes and a repair. Read `NeedsCompilation` from the slot's own flat index instead, which describes exactly the objects being inspected. It is fetched from the bucket rather than over HTTPS because this runs inside R, where the edge would route the request to a per-minor index. Keyed on `<package>_<version>` rather than the package name: a slot can carry several versions and only some of them compile.
199 lines
6.4 KiB
R
199 lines
6.4 KiB
R
### Record, per flat-slot object, whether it loads under every supported R minor.
|
|
#
|
|
# Usage: Rscript local/backfill-flat-safety.R <arch> <codename> [limit]
|
|
#
|
|
# `union_index_records()` keeps a carried flat record when that object is
|
|
# recorded safe, and drops it otherwise. Objects uploaded before that record
|
|
# existed have no verdict, so they are dropped: a full re-index removed ~2000
|
|
# records per per-minor index, all resolving to CRAN source for clients the
|
|
# routing serves correctly.
|
|
#
|
|
# This inspects the objects already in the bucket and writes the verdict, which
|
|
# is what restores that coverage. The question is decided by symbols, not
|
|
# heuristics: a binary fails at dyn.load() under minor M exactly when it
|
|
# references a symbol M does not export, and the volatile set is derived by
|
|
# diffing the R builds installed in this image.
|
|
#
|
|
# Only compiled packages are inspected. A package with no shared object cannot
|
|
# fail this way and is never dropped, so proving it would be wasted downloads.
|
|
#
|
|
# Resumable: objects already recorded are skipped, so a re-run costs one query.
|
|
|
|
options(error = function() {
|
|
cat("ERROR:", geterrmessage(), "\n", file = stdout())
|
|
traceback(2)
|
|
q(status = 1)
|
|
})
|
|
|
|
library(bincraft, quietly = TRUE)
|
|
|
|
args <- commandArgs(trailingOnly = TRUE)
|
|
if (length(args) < 2L) {
|
|
stop("usage: backfill-flat-safety.R <arch> <codename> [limit]", call. = FALSE)
|
|
}
|
|
arch <- args[[1L]]
|
|
codename <- args[[2L]]
|
|
limit <- if (length(args) >= 3L) as.integer(args[[3L]]) else NA_integer_
|
|
|
|
sets <- bincraft:::installed_r_symbol_sets()
|
|
if (is.null(sets)) {
|
|
stop(
|
|
"Fewer than two R minors are installed here, so nothing can be compared. Refusing to record verdicts that would all read 'safe'.", # nolint
|
|
call. = FALSE
|
|
)
|
|
}
|
|
cat(sprintf(
|
|
"Comparing R minors: %s (%d symbols vary across them)\n",
|
|
paste(names(sets), collapse = ", "),
|
|
length(bincraft:::cross_minor_volatile_symbols(sets))
|
|
))
|
|
|
|
# The flat slot is listed from S3, not from PACKAGES.gz over HTTPS. Reading the
|
|
# index from inside R sends an R User-Agent, which the edge routes to a
|
|
# per-minor index; that enumerates union records carrying `Path: <minor>`, whose
|
|
# objects do not exist at the flat path. Every download then 404s: a first run
|
|
# skipped 292 of 300 that way. The bucket cannot be routed and is authoritative.
|
|
s3fs::s3_file_system(
|
|
aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
|
aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
|
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
|
region_name = "eu-central-003",
|
|
refresh = TRUE
|
|
)
|
|
contrib <- sprintf(
|
|
"devxy-rpkgs-binaries/%s/%s/latest/src/contrib",
|
|
arch,
|
|
codename
|
|
)
|
|
objects <- s3fs::s3_dir_ls(contrib, recurse = TRUE)
|
|
relative <- sub(sprintf("^(s3://)?%s/", contrib), "", objects)
|
|
# Flat objects only: no `<minor>/` prefix and nothing under Archive/.
|
|
flat_files <- relative[
|
|
grepl("\\.tar\\.gz$", relative) & !grepl("/", relative, fixed = TRUE)
|
|
]
|
|
flat_pkg <- sub("_.*$", "", flat_files)
|
|
flat_ver <- sub("\\.tar\\.gz$", "", sub("^[^_]*_", "", flat_files))
|
|
|
|
# Only compiled packages can fail at dyn.load(), so the rest are not worth
|
|
# downloading. Ask the slot's own index rather than CRAN's: CRAN lists only the
|
|
# versions it currently ships, so a package it has archived is absent, is never
|
|
# recognised as compiled, and is never inspected. No verdict is ever recorded
|
|
# for it, while `union_index_records()` reads the slot's own `NeedsCompilation`
|
|
# and happily drops it -- with nothing able to clear the flag, because the
|
|
# repair works from recorded verdicts too. `mixedMem` and `vcpen`, both removed
|
|
# from CRAN, were stuck out of every rhel8 per-minor index that way.
|
|
#
|
|
# Read it from the bucket, not over HTTPS: this runs inside R, so the edge would
|
|
# route the request to a per-minor index and the flat objects would not all be
|
|
# there. Same reason the object listing above comes from S3.
|
|
index_file <- file.path(tempdir(), "flat-PACKAGES.gz")
|
|
s3fs::s3_file_download(
|
|
sprintf("%s/PACKAGES.gz", contrib),
|
|
index_file,
|
|
overwrite = TRUE
|
|
)
|
|
index <- read.dcf(
|
|
gzfile(index_file),
|
|
fields = c("Package", "Version", "NeedsCompilation")
|
|
)
|
|
if (nrow(index) == 0L) {
|
|
stop(
|
|
sprintf(
|
|
"Read no records from %s/PACKAGES.gz; refusing to guess what is compiled.",
|
|
contrib
|
|
),
|
|
call. = FALSE
|
|
)
|
|
}
|
|
# Key on the object, not the package: a slot can carry several versions and
|
|
# only some of them compile.
|
|
compiled_objects <- paste(
|
|
index[, "Package"],
|
|
index[, "Version"],
|
|
sep = "_"
|
|
)[
|
|
!is.na(index[, "NeedsCompilation"]) &
|
|
tolower(index[, "NeedsCompilation"]) == "yes"
|
|
]
|
|
keep <- paste(flat_pkg, flat_ver, sep = "_") %in% compiled_objects
|
|
compiled <- data.frame(
|
|
Package = flat_pkg[keep],
|
|
Version = flat_ver[keep],
|
|
stringsAsFactors = FALSE
|
|
)
|
|
cat(sprintf(
|
|
"%s/%s: %d flat objects, %d compiled and worth inspecting\n",
|
|
arch,
|
|
codename,
|
|
length(flat_files),
|
|
nrow(compiled)
|
|
))
|
|
if (!is.na(limit) && limit > 0L && limit < nrow(compiled)) {
|
|
compiled <- compiled[seq_len(limit), , drop = FALSE]
|
|
cat(sprintf("Limited to the first %d\n", limit))
|
|
}
|
|
|
|
# Fetch from the bucket as well: the edge does not rewrite tarball requests,
|
|
# but it does cache them, and the object is what the verdict is about.
|
|
safe <- 0L
|
|
unsafe <- 0L
|
|
skipped <- 0L
|
|
for (i in seq_len(nrow(compiled))) {
|
|
pkg <- compiled[i, "Package"]
|
|
ver <- compiled[i, "Version"]
|
|
tarball <- file.path(tempdir(), sprintf("%s_%s.tar.gz", pkg, ver))
|
|
ok <- tryCatch(
|
|
{
|
|
s3fs::s3_file_download(
|
|
sprintf("%s/%s_%s.tar.gz", contrib, pkg, ver),
|
|
tarball,
|
|
overwrite = TRUE
|
|
)
|
|
file.exists(tarball)
|
|
},
|
|
error = function(e) FALSE,
|
|
warning = function(w) FALSE
|
|
)
|
|
if (!ok) {
|
|
skipped <- skipped + 1L
|
|
next
|
|
}
|
|
verdict <- bincraft:::tarball_abi_verdict(tarball, sets)
|
|
unlink(tarball)
|
|
if (!isTRUE(verdict$inspected)) {
|
|
skipped <- skipped + 1L
|
|
next
|
|
}
|
|
bincraft:::flat_safety_store(
|
|
pkg,
|
|
ver,
|
|
codename,
|
|
arch,
|
|
safe = !isTRUE(verdict$sensitive),
|
|
unsupported = verdict$unsupported
|
|
)
|
|
if (isTRUE(verdict$sensitive)) {
|
|
unsafe <- unsafe + 1L
|
|
} else {
|
|
safe <- safe + 1L
|
|
}
|
|
if (i %% 250L == 0L) {
|
|
cat(sprintf(
|
|
" inspected %d/%d (%d safe, %d unsafe, %d skipped)\n",
|
|
i,
|
|
nrow(compiled),
|
|
safe,
|
|
unsafe,
|
|
skipped
|
|
))
|
|
utils::flush.console()
|
|
}
|
|
}
|
|
|
|
cat(sprintf(
|
|
"Done: %d inspected, %d safe for every minor, %d unsafe, %d skipped\n",
|
|
nrow(compiled),
|
|
safe,
|
|
unsafe,
|
|
skipped
|
|
))
|