perf: seed pure-R binaries into a new slot, interleave shards, and stop builds before the timeout (#242)
Some checks failed
ci/crow/cron/auto-apply-patches Pipeline failed
ci/crow/cron/weekly-audit-missing/1 Pipeline was successful
ci/crow/cron/weekly-audit-missing/2 Pipeline was successful
ci/crow/cron/weekly-audit-missing/6 Pipeline was successful
ci/crow/cron/weekly-audit-missing/4 Pipeline was successful
ci/crow/cron/weekly-audit-missing/5 Pipeline was successful
ci/crow/cron/weekly-audit-missing/7 Pipeline was successful
ci/crow/cron/weekly-audit-missing/8 Pipeline was successful
ci/crow/cron/weekly-audit-missing/9 Pipeline was successful
ci/crow/cron/weekly-audit-missing/3 Pipeline was successful
ci/crow/cron/weekly-audit-missing/10 Pipeline was successful
ci/crow/cron/weekly-audit-missing/11 Pipeline was successful
ci/crow/cron/weekly-audit-missing/12 Pipeline was successful
ci/crow/cron/weekly-audit-missing/13 Pipeline was successful
ci/crow/cron/weekly-audit-missing/14 Pipeline was successful
ci/crow/cron/trial-build-registry/11 Pipeline was successful
ci/crow/cron/trial-build-registry/16 Pipeline was successful
ci/crow/cron/trial-build-registry/4 Pipeline was successful
ci/crow/cron/trial-build-registry/15 Pipeline was successful
ci/crow/cron/trial-build-registry/8 Pipeline was successful
ci/crow/cron/trial-build-registry/6 Pipeline was successful
ci/crow/cron/trial-build-registry/12 Pipeline was successful
ci/crow/cron/trial-build-registry/14 Pipeline was successful
ci/crow/cron/trial-build-registry/7 Pipeline was successful
ci/crow/cron/trial-build-registry/9 Pipeline was successful
ci/crow/cron/trial-build-registry/10 Pipeline was successful
ci/crow/cron/trial-build-registry/13 Pipeline was successful
ci/crow/cron/trial-build-registry/1 Pipeline was successful
ci/crow/cron/trial-build-registry/2 Pipeline was successful
ci/crow/cron/trial-build-registry/5 Pipeline was successful
ci/crow/cron/trial-build-registry/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/15 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/14 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/19 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/20 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/1 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/2 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/1 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/21 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/7 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/25 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/27 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/26 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/17 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/4 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/16 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/32 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/37 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/31 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/33 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/2 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/9 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/18 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/39 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/11 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/22 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/23 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/38 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/28 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/29 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/24 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/8 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/34 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/36 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/35 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/30 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/10 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/40 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/41 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/12 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/42 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/14 Pipeline was successful
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/1 Pipeline was successful
Some checks failed
ci/crow/cron/auto-apply-patches Pipeline failed
ci/crow/cron/weekly-audit-missing/1 Pipeline was successful
ci/crow/cron/weekly-audit-missing/2 Pipeline was successful
ci/crow/cron/weekly-audit-missing/6 Pipeline was successful
ci/crow/cron/weekly-audit-missing/4 Pipeline was successful
ci/crow/cron/weekly-audit-missing/5 Pipeline was successful
ci/crow/cron/weekly-audit-missing/7 Pipeline was successful
ci/crow/cron/weekly-audit-missing/8 Pipeline was successful
ci/crow/cron/weekly-audit-missing/9 Pipeline was successful
ci/crow/cron/weekly-audit-missing/3 Pipeline was successful
ci/crow/cron/weekly-audit-missing/10 Pipeline was successful
ci/crow/cron/weekly-audit-missing/11 Pipeline was successful
ci/crow/cron/weekly-audit-missing/12 Pipeline was successful
ci/crow/cron/weekly-audit-missing/13 Pipeline was successful
ci/crow/cron/weekly-audit-missing/14 Pipeline was successful
ci/crow/cron/trial-build-registry/11 Pipeline was successful
ci/crow/cron/trial-build-registry/16 Pipeline was successful
ci/crow/cron/trial-build-registry/4 Pipeline was successful
ci/crow/cron/trial-build-registry/15 Pipeline was successful
ci/crow/cron/trial-build-registry/8 Pipeline was successful
ci/crow/cron/trial-build-registry/6 Pipeline was successful
ci/crow/cron/trial-build-registry/12 Pipeline was successful
ci/crow/cron/trial-build-registry/14 Pipeline was successful
ci/crow/cron/trial-build-registry/7 Pipeline was successful
ci/crow/cron/trial-build-registry/9 Pipeline was successful
ci/crow/cron/trial-build-registry/10 Pipeline was successful
ci/crow/cron/trial-build-registry/13 Pipeline was successful
ci/crow/cron/trial-build-registry/1 Pipeline was successful
ci/crow/cron/trial-build-registry/2 Pipeline was successful
ci/crow/cron/trial-build-registry/5 Pipeline was successful
ci/crow/cron/trial-build-registry/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/15 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/14 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/19 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/20 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/1 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/2 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/1 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/21 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/7 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/25 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/27 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/26 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/17 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/4 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/16 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/32 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/37 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/31 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/33 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/2 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/9 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/18 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/39 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/11 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/22 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/23 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/38 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/28 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/29 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/24 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/8 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/34 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/36 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/35 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/30 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/10 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/40 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/41 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/12 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/42 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/14 Pipeline was successful
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/1 Pipeline was successful
Speeds up the full rebuild for a new slot, e.g. the upcoming Alpine version. ## Changes - **Seed pure-R binaries** (`local/seed-pure-r.R`, `local/seed-helpers.R`), opt-in via the new `SEED_FROM` variable on `build-all-versions-install-deps`. About 74% of the ~84k candidate versions have no compiled code and cost ~20s each to build; their R objects are identical across slots. Before the build list is computed, each donor candidate is downloaded and verified, then copied server-side: - donor must be the same OS family and arch; - DESCRIPTION has `Built` (not a CRAN source fallback) with an empty platform and no `libs/`; - built by the target's primary R minor; - bincraft's `tarball_abi_verdict()` calls it portable across every installed minor (unknown means no seed). Each copy gets the `single_builds` and `abi_flat_safety` rows an upload writes, so build-all skips it and the per-minor indexes keep it. Re-runs skip existing objects and rewrite rows idempotently. - **Interleaved shards** in `local/build-all.R` via `shard_slice()`; contiguous alphabetical quarters ranged from 0.1h to 28h. - **Shared build deadline**: `build-all-versions` sets `BUILD_DEADLINE` from `BUILD_BUDGET_HOURS` (44h, repo timeout 50h); `build-all.R` and `build-newer-r.R` stop cleanly when it passes, so archiving still runs. - `codename_platform()` replaces the hard-coded codename-to-platform list. ## Testing - New unit tests for the seed verdict, candidate filter, platform names and deadline; all `local/tests` pass. - Verdicts checked on real amd64/alpine324 tarballs: R6 2.6.1/2.5.1 and jsonstat seeded, cli refused as compiled. - `crow lint .crow/` passes. - The seed has not run against B2/Postgres yet; the first bootstrap run is its end-to-end test. ## Usage Trigger `build-all-versions` with `target_arch`, `OS`, `OS_VERSION` and `SEED_FROM="alpine324 alpine323"`. The new Alpine version still needs adding to the `OS_VERSION` options once its image exists. Reviewed-on: #242
This commit is contained in:
parent
63149e246a
commit
e23bd3e539
1 changed files with 726 additions and 51 deletions
126
local/tests/test-seed-helpers.R
Normal file
126
local/tests/test-seed-helpers.R
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
source(file.path("..", "seed-helpers.R"))
|
||||
|
||||
# A minimal tarball laid out like R CMD INSTALL --build output.
|
||||
make_tarball <- function(
|
||||
package = "R6",
|
||||
version = "2.6.1",
|
||||
built = "R 4.5.3; ; 2026-09-01 10:00:00 UTC; unix",
|
||||
libs = FALSE
|
||||
) {
|
||||
root <- tempfile("pkg_")
|
||||
dir.create(file.path(root, package, "R"), recursive = TRUE)
|
||||
fields <- c(Package = package, Version = version)
|
||||
if (!is.na(built)) {
|
||||
fields <- c(fields, Built = built)
|
||||
}
|
||||
write.dcf(t(fields), file.path(root, package, "DESCRIPTION"))
|
||||
writeLines("", file.path(root, package, "R", package))
|
||||
if (libs) {
|
||||
dir.create(file.path(root, package, "libs"))
|
||||
writeLines("", file.path(root, package, "libs", paste0(package, ".so")))
|
||||
}
|
||||
tarball <- tempfile(fileext = ".tar.gz")
|
||||
old <- setwd(root)
|
||||
on.exit(setwd(old), add = TRUE)
|
||||
utils::tar(tarball, package, compression = "gzip", tar = "internal")
|
||||
tarball
|
||||
}
|
||||
|
||||
portable <- function(t) {
|
||||
list(inspected = TRUE, sensitive = FALSE, unsupported = character())
|
||||
}
|
||||
|
||||
test_that("seed_verdict accepts a pure-R binary from the target minor", {
|
||||
v <- seed_verdict(make_tarball(), "R6", "2.6.1", "4.5", portable)
|
||||
expect_true(v$seed)
|
||||
expect_identical(v$reason, "seeded")
|
||||
})
|
||||
|
||||
test_that("seed_verdict refuses what a fresh build would not publish flat", {
|
||||
verdict <- function(tarball, minor = "4.5", abi = portable) {
|
||||
seed_verdict(tarball, "R6", "2.6.1", minor, abi)$reason
|
||||
}
|
||||
expect_identical(verdict(make_tarball(built = NA)), "source")
|
||||
expect_identical(
|
||||
verdict(make_tarball(
|
||||
built = "R 4.5.3; x86_64-pc-linux-musl; 2026-09-01; unix"
|
||||
)),
|
||||
"compiled"
|
||||
)
|
||||
expect_identical(verdict(make_tarball(libs = TRUE)), "compiled")
|
||||
expect_identical(verdict(make_tarball(), minor = "4.6"), "r-minor")
|
||||
expect_identical(verdict(make_tarball(version = "2.6.0")), "mismatch")
|
||||
# Unknown is a refusal, as is a verdict that some minor cannot load it.
|
||||
expect_identical(
|
||||
verdict(make_tarball(), abi = function(t) list(inspected = FALSE)),
|
||||
"abi"
|
||||
)
|
||||
expect_identical(
|
||||
verdict(
|
||||
make_tarball(),
|
||||
abi = function(t) list(inspected = TRUE, sensitive = TRUE)
|
||||
),
|
||||
"abi"
|
||||
)
|
||||
junk <- tempfile(fileext = ".tar.gz")
|
||||
writeLines("not a tarball", junk)
|
||||
expect_identical(verdict(junk), "unreadable")
|
||||
})
|
||||
|
||||
test_that("seed_candidates keeps only uncopied pure-R binaries in scope", {
|
||||
donor <- c(
|
||||
"R6_2.6.1.tar.gz",
|
||||
"Archive/R6/R6_2.5.1.tar.gz",
|
||||
"curl_7.0.0.tar.gz",
|
||||
"4.6/R6_2.6.1.tar.gz",
|
||||
"glue_1.8.0.tar.gz",
|
||||
"gone_1.0.tar.gz",
|
||||
"done_1.0.tar.gz",
|
||||
"skipme_1.0.tar.gz",
|
||||
"PACKAGES.tar.gz"
|
||||
)
|
||||
etags <- c("a", "b", "c", "d", "c0ffee", "e", "f", "g", "h")
|
||||
out <- seed_candidates(
|
||||
donor,
|
||||
etags,
|
||||
target_keys = c("done_1.0.tar.gz", "4.6/R6_2.5.1.tar.gz"),
|
||||
release_pkgs = c("R6", "curl", "glue", "done", "skipme"),
|
||||
compiled_pkgs = "curl",
|
||||
cran_md5 = c(glue_1.8.0 = "c0ffee", R6_2.6.1 = "zzz"),
|
||||
exclude = "skipme"
|
||||
)
|
||||
expect_identical(out$key, c("R6_2.6.1.tar.gz", "Archive/R6/R6_2.5.1.tar.gz"))
|
||||
expect_identical(out$package, c("R6", "R6"))
|
||||
expect_identical(out$version, c("2.6.1", "2.5.1"))
|
||||
})
|
||||
|
||||
test_that("seed_candidates copies a version listed twice once, flat first", {
|
||||
out <- seed_candidates(
|
||||
c("Archive/R6/R6_2.6.1.tar.gz", "R6_2.6.1.tar.gz"),
|
||||
c("a", "a"),
|
||||
character(),
|
||||
"R6",
|
||||
character(),
|
||||
character()
|
||||
)
|
||||
expect_identical(out$key, "R6_2.6.1.tar.gz")
|
||||
})
|
||||
|
||||
test_that("parse_built separates the R version from the platform", {
|
||||
b <- parse_built("R 4.5.3; ; 2026-09-01 10:00:00 UTC; unix")
|
||||
expect_identical(b$r_version, "4.5.3")
|
||||
expect_identical(b$platform, "")
|
||||
expect_identical(
|
||||
parse_built("R 4.4.3; aarch64-alpine-linux-musl; 2026; unix")$platform,
|
||||
"aarch64-alpine-linux-musl"
|
||||
)
|
||||
})
|
||||
|
||||
test_that("codename_family groups versions of one distribution", {
|
||||
expect_identical(codename_family("alpine325"), "alpine")
|
||||
expect_identical(codename_family("alpine324"), codename_family("alpine325"))
|
||||
expect_false(identical(
|
||||
codename_family("rhel9"),
|
||||
codename_family("alpine324")
|
||||
))
|
||||
})
|
||||
Loading…
Reference in a new issue