feat(abi): skip objects that already have the per-minor builds they need (#207)
Some checks failed
ci/crow/manual/repair-unsafe-flat-objects/7 Pipeline is pending
ci/crow/manual/repair-unsafe-flat-objects/9 Pipeline failed
ci/crow/manual/repair-unsafe-flat-objects/13 Pipeline failed
ci/crow/manual/repair-unsafe-flat-objects/15 Pipeline failed
ci/crow/manual/repair-unsafe-flat-objects/8 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/10 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/14 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/16 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/17 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/11 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/18 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/12 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/4 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/2 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/3 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/5 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/6 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/1 Pipeline was successful
Some checks failed
ci/crow/manual/repair-unsafe-flat-objects/7 Pipeline is pending
ci/crow/manual/repair-unsafe-flat-objects/9 Pipeline failed
ci/crow/manual/repair-unsafe-flat-objects/13 Pipeline failed
ci/crow/manual/repair-unsafe-flat-objects/15 Pipeline failed
ci/crow/manual/repair-unsafe-flat-objects/8 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/10 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/14 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/16 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/17 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/11 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/18 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/12 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/4 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/2 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/3 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/5 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/6 Pipeline was successful
ci/crow/manual/repair-unsafe-flat-objects/1 Pipeline was successful
## Why The fleet repair (12410) was interrupted after 14 of 18 slots, having done: ``` 2522 objects enumerated | 2568 uploads | 1451 symbol overrides ``` Resuming would repeat all of it. The verdict in `abi_flat_safety` describes the **flat** object, and building a per-minor copy does not change that object, so the repair list is identical on the next run. Roughly 2500 objects of duplicated compute. ## What changed `list-unsafe-flat-objects.R` now drops rows that are already repaired: an object counts as repaired once a per-minor build exists for **every** minor that cannot load the flat one. - The bucket is the source of truth, listed directly (it cannot be routed). - Where `unsupported` is empty — unsafe without a recorded reason — a build under every minor the image carries is required, rather than assuming one suffices. - It reports how many were skipped, so an unexpectedly short list is visible rather than silent. - It **fails when the bucket cannot be listed**, because an empty listing would make every object look unrepaired and silently restore the full rebuild. ## Verification The predicate, exercised directly: ``` Rcpp needs[4.4,4.6] -> SKIP (repaired) Rcpp needs[4.4,4.5,4.6] -> REBUILD (4.5 missing) yaml needs[4.4] -> SKIP (repaired) lobstr needs[] -> REBUILD (no recorded minors: all required) ``` ## Note ~70 packages fail to build regardless — `glmmTMB`, `tmbstan`, `OpenMx`, `Cyclops`, `RSiena`, `pdftools`, `odbc` among them — all on `uvr sync --install-system-deps`. They keep their unsafe flat binaries and stay broken for the minors they cannot load under. That is a missing-system-dependency problem in the build images, not an ABI one, and needs separate attention. Reviewed-on: #207
This commit is contained in:
parent
ee4562d44c
commit
e3b7212eaf
1 changed files with 80 additions and 1 deletions
|
|
@ -37,13 +37,92 @@ on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE)
|
|||
rows <- DBI::dbGetQuery(
|
||||
con,
|
||||
paste0(
|
||||
"SELECT package, version FROM abi_flat_safety ",
|
||||
"SELECT package, version, unsupported FROM abi_flat_safety ",
|
||||
"WHERE NOT safe AND codename = $1 AND arch = $2 ",
|
||||
"ORDER BY package, version"
|
||||
),
|
||||
params = list(codename, arch)
|
||||
)
|
||||
|
||||
# Drop what is already repaired. The verdict describes the *flat* object, and a
|
||||
# per-minor build does not change it, so a resumed run would otherwise rebuild
|
||||
# everything it did last time: ~2500 objects of duplicated compute after the
|
||||
# first fleet run was interrupted.
|
||||
#
|
||||
# An object is repaired once a per-minor build exists for every minor that
|
||||
# cannot load the flat one. The bucket is the source of truth for that, and it
|
||||
# cannot be routed.
|
||||
if (nrow(rows) > 0L) {
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region_name = "eu-central-003",
|
||||
refresh = TRUE
|
||||
)
|
||||
contrib <- sprintf(
|
||||
"devxy-rpkgs-binaries/%s/%s/latest/src/contrib",
|
||||
arch,
|
||||
codename
|
||||
)
|
||||
objects <- tryCatch(
|
||||
s3fs::s3_dir_ls(contrib, recurse = TRUE),
|
||||
error = function(e) character(0)
|
||||
)
|
||||
if (length(objects) == 0L) {
|
||||
stop(
|
||||
sprintf(
|
||||
"Could not list %s; refusing to guess what is already repaired.",
|
||||
contrib
|
||||
),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
present <- sub(sprintf("^(s3://)?%s/", contrib), "", objects)
|
||||
|
||||
# No recorded minors means unsafe without a reason, so require a build under
|
||||
# every minor the image carries rather than assume one is enough.
|
||||
installed <- sub(
|
||||
"^([0-9]+[.][0-9]+).*$",
|
||||
"\\1",
|
||||
basename(list.dirs("/opt/R", recursive = FALSE))
|
||||
)
|
||||
installed <- unique(installed[grepl("^[0-9]+[.][0-9]+$", installed)])
|
||||
|
||||
repaired <- vapply(
|
||||
seq_len(nrow(rows)),
|
||||
function(i) {
|
||||
u <- rows$unsupported[[i]]
|
||||
minors <- if (is.na(u) || !nzchar(u)) {
|
||||
installed
|
||||
} else {
|
||||
trimws(strsplit(u, ",", fixed = TRUE)[[1L]])
|
||||
}
|
||||
if (length(minors) == 0L) {
|
||||
return(FALSE)
|
||||
}
|
||||
wanted <- sprintf(
|
||||
"%s/%s_%s.tar.gz",
|
||||
minors,
|
||||
rows$package[[i]],
|
||||
rows$version[[i]]
|
||||
)
|
||||
all(wanted %in% present)
|
||||
},
|
||||
logical(1L)
|
||||
)
|
||||
if (any(repaired)) {
|
||||
message(sprintf(
|
||||
"%s/%s: %d of %d already have per-minor builds for every minor that needs one; skipping those.",
|
||||
arch,
|
||||
codename,
|
||||
sum(repaired),
|
||||
nrow(rows)
|
||||
))
|
||||
}
|
||||
rows <- rows[!repaired, , drop = FALSE]
|
||||
}
|
||||
|
||||
# An empty result is ambiguous: either nothing is unsafe, or the backfill never
|
||||
# ran for this slot. Say which, rather than let the caller read silence as done.
|
||||
if (nrow(rows) == 0L) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue