Some checks failed
ci/crow/cron/auto-apply-patches Pipeline failed
ci/crow/cron/weekly-audit-missing/1 Pipeline was successful
ci/crow/cron/weekly-audit-missing/2 Pipeline was successful
ci/crow/cron/weekly-audit-missing/6 Pipeline was successful
ci/crow/cron/weekly-audit-missing/4 Pipeline was successful
ci/crow/cron/weekly-audit-missing/5 Pipeline was successful
ci/crow/cron/weekly-audit-missing/7 Pipeline was successful
ci/crow/cron/weekly-audit-missing/8 Pipeline was successful
ci/crow/cron/weekly-audit-missing/9 Pipeline was successful
ci/crow/cron/weekly-audit-missing/3 Pipeline was successful
ci/crow/cron/weekly-audit-missing/10 Pipeline was successful
ci/crow/cron/weekly-audit-missing/11 Pipeline was successful
ci/crow/cron/weekly-audit-missing/12 Pipeline was successful
ci/crow/cron/weekly-audit-missing/13 Pipeline was successful
ci/crow/cron/weekly-audit-missing/14 Pipeline was successful
ci/crow/cron/trial-build-registry/11 Pipeline was successful
ci/crow/cron/trial-build-registry/16 Pipeline was successful
ci/crow/cron/trial-build-registry/4 Pipeline was successful
ci/crow/cron/trial-build-registry/15 Pipeline was successful
ci/crow/cron/trial-build-registry/8 Pipeline was successful
ci/crow/cron/trial-build-registry/6 Pipeline was successful
ci/crow/cron/trial-build-registry/12 Pipeline was successful
ci/crow/cron/trial-build-registry/14 Pipeline was successful
ci/crow/cron/trial-build-registry/7 Pipeline was successful
ci/crow/cron/trial-build-registry/9 Pipeline was successful
ci/crow/cron/trial-build-registry/10 Pipeline was successful
ci/crow/cron/trial-build-registry/13 Pipeline was successful
ci/crow/cron/trial-build-registry/1 Pipeline was successful
ci/crow/cron/trial-build-registry/2 Pipeline was successful
ci/crow/cron/trial-build-registry/5 Pipeline was successful
ci/crow/cron/trial-build-registry/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/15 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/14 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/19 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/20 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/1 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/2 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/1 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/21 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/7 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/25 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/27 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/26 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/17 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/4 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/16 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/32 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/37 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/31 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/33 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/2 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/9 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/18 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/39 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/11 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/22 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/23 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/38 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/28 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/29 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/24 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/8 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/34 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/36 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/35 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/30 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/10 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/40 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/41 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/12 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/42 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/14 Pipeline was successful
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/1 Pipeline was successful
Speeds up the full rebuild for a new slot, e.g. the upcoming Alpine version. ## Changes - **Seed pure-R binaries** (`local/seed-pure-r.R`, `local/seed-helpers.R`), opt-in via the new `SEED_FROM` variable on `build-all-versions-install-deps`. About 74% of the ~84k candidate versions have no compiled code and cost ~20s each to build; their R objects are identical across slots. Before the build list is computed, each donor candidate is downloaded and verified, then copied server-side: - donor must be the same OS family and arch; - DESCRIPTION has `Built` (not a CRAN source fallback) with an empty platform and no `libs/`; - built by the target's primary R minor; - bincraft's `tarball_abi_verdict()` calls it portable across every installed minor (unknown means no seed). Each copy gets the `single_builds` and `abi_flat_safety` rows an upload writes, so build-all skips it and the per-minor indexes keep it. Re-runs skip existing objects and rewrite rows idempotently. - **Interleaved shards** in `local/build-all.R` via `shard_slice()`; contiguous alphabetical quarters ranged from 0.1h to 28h. - **Shared build deadline**: `build-all-versions` sets `BUILD_DEADLINE` from `BUILD_BUDGET_HOURS` (44h, repo timeout 50h); `build-all.R` and `build-newer-r.R` stop cleanly when it passes, so archiving still runs. - `codename_platform()` replaces the hard-coded codename-to-platform list. ## Testing - New unit tests for the seed verdict, candidate filter, platform names and deadline; all `local/tests` pass. - Verdicts checked on real amd64/alpine324 tarballs: R6 2.6.1/2.5.1 and jsonstat seeded, cli refused as compiled. - `crow lint .crow/` passes. - The seed has not run against B2/Postgres yet; the first bootstrap run is its end-to-end test. ## Usage Trigger `build-all-versions` with `target_arch`, `OS`, `OS_VERSION` and `SEED_FROM="alpine324 alpine323"`. The new Alpine version still needs adding to the `OS_VERSION` options once its image exists. Reviewed-on: #242
162 lines
5.6 KiB
R
162 lines
5.6 KiB
R
# Pure helpers for local/seed-pure-r.R, kept separate so local/tests can source
|
|
# them without touching S3 or the metadata DB.
|
|
|
|
# "alpine" for alpine324, "rhel" for rhel9. Seeding only ever crosses versions
|
|
# within one family: a pure-R binary has no compiled code, but install-time R
|
|
# code can still capture the build system's paths and tools.
|
|
codename_family <- function(codename) {
|
|
sub("[0-9]+$", "", codename)
|
|
}
|
|
|
|
per_minor_key <- function(keys) {
|
|
grepl("^[0-9]+\\.[0-9]+/", keys)
|
|
}
|
|
|
|
# Split `<pkg>_<ver>.tar.gz` file names; NA where a name does not parse.
|
|
split_tarball_name <- function(files) {
|
|
m <- regmatches(files, regexec("^([A-Za-z0-9.]+)_([^_]+)\\.tar\\.gz$", files))
|
|
part <- function(i) {
|
|
vapply(m, function(x) if (length(x) == 3L) x[[i]] else NA_character_, "")
|
|
}
|
|
list(package = part(2L), version = part(3L))
|
|
}
|
|
|
|
# Donor objects worth downloading and verifying for the target slot.
|
|
#
|
|
# `donor_keys`/`donor_etags` describe the donor slot, keys relative to its
|
|
# contrib dir (`curl_5.0.tar.gz`, `Archive/curl/curl_4.3.tar.gz`).
|
|
# `target_keys` are the target slot's keys, same form.
|
|
# `release_pkgs` packages on CRAN now; the build scope.
|
|
# `compiled_pkgs` packages whose current CRAN release needs compilation. Their
|
|
# archive versions are skipped as well: a seed only saves time, so guessing
|
|
# wrong in this direction costs a build, never a broken binary.
|
|
# `cran_md5` CRAN's MD5 per `<pkg>_<ver>`. An ETag equal to it is CRAN's source
|
|
# tarball served as a fallback, never a binary worth copying.
|
|
#
|
|
# Anything already present in the target, under any non-per-minor key, is left
|
|
# alone: that is either a build or an earlier seed.
|
|
seed_candidates <- function(
|
|
donor_keys,
|
|
donor_etags,
|
|
target_keys,
|
|
release_pkgs,
|
|
compiled_pkgs,
|
|
cran_md5,
|
|
exclude = character()
|
|
) {
|
|
files <- basename(donor_keys)
|
|
parsed <- split_tarball_name(files)
|
|
md5 <- if (length(cran_md5) == 0L) {
|
|
rep(NA_character_, length(files))
|
|
} else {
|
|
unname(cran_md5[paste(parsed$package, parsed$version, sep = "_")])
|
|
}
|
|
is_source <- !is.na(md5) & !is.na(donor_etags) & donor_etags == md5
|
|
present <- basename(target_keys[!per_minor_key(target_keys)])
|
|
|
|
keep <- !is.na(parsed$package) &
|
|
!per_minor_key(donor_keys) &
|
|
parsed$package %in% release_pkgs &
|
|
!parsed$package %in% compiled_pkgs &
|
|
!parsed$package %in% exclude &
|
|
!files %in% present &
|
|
!is_source
|
|
out <- data.frame(
|
|
key = donor_keys,
|
|
package = parsed$package,
|
|
version = parsed$version,
|
|
stringsAsFactors = FALSE
|
|
)[keep, , drop = FALSE]
|
|
# The same version can sit both flat and under Archive/ while the donor waits
|
|
# for process_unarchived_pkgs(); copy it once, flat first.
|
|
out <- out[order(grepl("^Archive/", out$key)), , drop = FALSE]
|
|
out <- out[!duplicated(basename(out$key)), , drop = FALSE]
|
|
rownames(out) <- NULL
|
|
out
|
|
}
|
|
|
|
# `Built: R 4.5.3; ; 2026-09-01 10:00:00 UTC; unix` -> R version and platform.
|
|
# The platform is empty exactly when the package has no compiled code.
|
|
parse_built <- function(built) {
|
|
parts <- trimws(strsplit(built, ";", fixed = TRUE)[[1L]])
|
|
list(
|
|
r_version = sub("^R\\s+", "", parts[1L]),
|
|
platform = if (length(parts) >= 2L) parts[2L] else NA_character_
|
|
)
|
|
}
|
|
|
|
# Whether one downloaded donor tarball may be published in the target slot.
|
|
#
|
|
# Every check has to pass; anything unreadable is a refusal, never a guess,
|
|
# because a refusal only means the package gets built the normal way.
|
|
# * it is a binary: DESCRIPTION carries a `Built` field (a CRAN source
|
|
# fallback does not);
|
|
# * it has no compiled code: empty `Built` platform and no libs/ directory;
|
|
# * it was built by the target's primary R minor, the one whose binaries the
|
|
# flat slot holds;
|
|
# * `abi_verdict` (bincraft's upload-time symbol inspection) calls it portable
|
|
# across every installed minor, exactly the decision a fresh upload would
|
|
# make before writing to the flat slot.
|
|
seed_verdict <- function(tarball, package, version, target_minor, abi_verdict) {
|
|
reject <- function(reason) {
|
|
list(seed = FALSE, reason = reason, unsupported = character())
|
|
}
|
|
read_members <- function() {
|
|
tryCatch(
|
|
utils::untar(tarball, list = TRUE),
|
|
error = function(e) NULL,
|
|
warning = function(w) NULL
|
|
)
|
|
}
|
|
members <- read_members()
|
|
if (is.null(members)) {
|
|
return(reject("unreadable"))
|
|
}
|
|
desc <- paste0(package, "/DESCRIPTION")
|
|
if (!desc %in% members) {
|
|
return(reject("unreadable"))
|
|
}
|
|
if (any(startsWith(members, paste0(package, "/libs/")))) {
|
|
return(reject("compiled"))
|
|
}
|
|
|
|
dir <- tempfile("seed_desc_")
|
|
on.exit(unlink(dir, recursive = TRUE, force = TRUE), add = TRUE)
|
|
fields <- tryCatch(
|
|
{
|
|
utils::untar(tarball, files = desc, exdir = dir)
|
|
read.dcf(
|
|
file.path(dir, desc),
|
|
fields = c("Package", "Version", "Built")
|
|
)
|
|
},
|
|
error = function(e) NULL,
|
|
warning = function(w) NULL
|
|
)
|
|
if (is.null(fields) || nrow(fields) != 1L) {
|
|
return(reject("unreadable"))
|
|
}
|
|
if (
|
|
!identical(unname(fields[1L, "Package"]), package) ||
|
|
!identical(unname(fields[1L, "Version"]), version)
|
|
) {
|
|
return(reject("mismatch"))
|
|
}
|
|
if (is.na(fields[1L, "Built"])) {
|
|
return(reject("source"))
|
|
}
|
|
built <- parse_built(fields[1L, "Built"])
|
|
if (is.na(built$platform) || nzchar(built$platform)) {
|
|
return(reject("compiled"))
|
|
}
|
|
built_minor <- sub("^(\\d+\\.\\d+).*$", "\\1", built$r_version)
|
|
if (!identical(built_minor, target_minor)) {
|
|
return(reject("r-minor"))
|
|
}
|
|
|
|
verdict <- abi_verdict(tarball)
|
|
if (!isTRUE(verdict$inspected) || !identical(verdict$sensitive, FALSE)) {
|
|
return(reject("abi"))
|
|
}
|
|
list(seed = TRUE, reason = "seeded", unsupported = verdict$unsupported)
|
|
}
|