perf: seed pure-R binaries into a new slot, interleave shards, and stop builds before the timeout (#242)
Some checks failed
ci/crow/cron/auto-apply-patches Pipeline failed
ci/crow/cron/weekly-audit-missing/1 Pipeline was successful
ci/crow/cron/weekly-audit-missing/2 Pipeline was successful
ci/crow/cron/weekly-audit-missing/6 Pipeline was successful
ci/crow/cron/weekly-audit-missing/4 Pipeline was successful
ci/crow/cron/weekly-audit-missing/5 Pipeline was successful
ci/crow/cron/weekly-audit-missing/7 Pipeline was successful
ci/crow/cron/weekly-audit-missing/8 Pipeline was successful
ci/crow/cron/weekly-audit-missing/9 Pipeline was successful
ci/crow/cron/weekly-audit-missing/3 Pipeline was successful
ci/crow/cron/weekly-audit-missing/10 Pipeline was successful
ci/crow/cron/weekly-audit-missing/11 Pipeline was successful
ci/crow/cron/weekly-audit-missing/12 Pipeline was successful
ci/crow/cron/weekly-audit-missing/13 Pipeline was successful
ci/crow/cron/weekly-audit-missing/14 Pipeline was successful
ci/crow/cron/trial-build-registry/11 Pipeline was successful
ci/crow/cron/trial-build-registry/16 Pipeline was successful
ci/crow/cron/trial-build-registry/4 Pipeline was successful
ci/crow/cron/trial-build-registry/15 Pipeline was successful
ci/crow/cron/trial-build-registry/8 Pipeline was successful
ci/crow/cron/trial-build-registry/6 Pipeline was successful
ci/crow/cron/trial-build-registry/12 Pipeline was successful
ci/crow/cron/trial-build-registry/14 Pipeline was successful
ci/crow/cron/trial-build-registry/7 Pipeline was successful
ci/crow/cron/trial-build-registry/9 Pipeline was successful
ci/crow/cron/trial-build-registry/10 Pipeline was successful
ci/crow/cron/trial-build-registry/13 Pipeline was successful
ci/crow/cron/trial-build-registry/1 Pipeline was successful
ci/crow/cron/trial-build-registry/2 Pipeline was successful
ci/crow/cron/trial-build-registry/5 Pipeline was successful
ci/crow/cron/trial-build-registry/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/15 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/14 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/19 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/20 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/1 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/2 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/1 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/21 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/7 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/25 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/27 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/26 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/17 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/4 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/16 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/32 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/37 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/31 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/33 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/2 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/9 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/18 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/39 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/11 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/22 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/23 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/38 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/28 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/29 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/24 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/8 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/34 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/36 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/35 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/30 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/10 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/40 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/41 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/12 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/42 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/14 Pipeline was successful
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/1 Pipeline was successful
Some checks failed
ci/crow/cron/auto-apply-patches Pipeline failed
ci/crow/cron/weekly-audit-missing/1 Pipeline was successful
ci/crow/cron/weekly-audit-missing/2 Pipeline was successful
ci/crow/cron/weekly-audit-missing/6 Pipeline was successful
ci/crow/cron/weekly-audit-missing/4 Pipeline was successful
ci/crow/cron/weekly-audit-missing/5 Pipeline was successful
ci/crow/cron/weekly-audit-missing/7 Pipeline was successful
ci/crow/cron/weekly-audit-missing/8 Pipeline was successful
ci/crow/cron/weekly-audit-missing/9 Pipeline was successful
ci/crow/cron/weekly-audit-missing/3 Pipeline was successful
ci/crow/cron/weekly-audit-missing/10 Pipeline was successful
ci/crow/cron/weekly-audit-missing/11 Pipeline was successful
ci/crow/cron/weekly-audit-missing/12 Pipeline was successful
ci/crow/cron/weekly-audit-missing/13 Pipeline was successful
ci/crow/cron/weekly-audit-missing/14 Pipeline was successful
ci/crow/cron/trial-build-registry/11 Pipeline was successful
ci/crow/cron/trial-build-registry/16 Pipeline was successful
ci/crow/cron/trial-build-registry/4 Pipeline was successful
ci/crow/cron/trial-build-registry/15 Pipeline was successful
ci/crow/cron/trial-build-registry/8 Pipeline was successful
ci/crow/cron/trial-build-registry/6 Pipeline was successful
ci/crow/cron/trial-build-registry/12 Pipeline was successful
ci/crow/cron/trial-build-registry/14 Pipeline was successful
ci/crow/cron/trial-build-registry/7 Pipeline was successful
ci/crow/cron/trial-build-registry/9 Pipeline was successful
ci/crow/cron/trial-build-registry/10 Pipeline was successful
ci/crow/cron/trial-build-registry/13 Pipeline was successful
ci/crow/cron/trial-build-registry/1 Pipeline was successful
ci/crow/cron/trial-build-registry/2 Pipeline was successful
ci/crow/cron/trial-build-registry/5 Pipeline was successful
ci/crow/cron/trial-build-registry/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/15 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/14 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/19 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/20 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/1 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/2 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/1 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/21 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/7 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/25 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/27 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/26 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/17 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/4 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/16 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/32 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/37 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/31 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/33 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/2 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/9 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/18 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/39 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/11 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/22 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/23 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/38 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/28 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/29 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/24 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/8 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/34 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/36 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/35 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/30 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/10 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/40 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/41 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/12 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/42 Pipeline was successful
ci/crow/cron/weekly-rebuild-reindex/14 Pipeline was successful
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/1 Pipeline was successful
Speeds up the full rebuild for a new slot, e.g. the upcoming Alpine version. ## Changes - **Seed pure-R binaries** (`local/seed-pure-r.R`, `local/seed-helpers.R`), opt-in via the new `SEED_FROM` variable on `build-all-versions-install-deps`. About 74% of the ~84k candidate versions have no compiled code and cost ~20s each to build; their R objects are identical across slots. Before the build list is computed, each donor candidate is downloaded and verified, then copied server-side: - donor must be the same OS family and arch; - DESCRIPTION has `Built` (not a CRAN source fallback) with an empty platform and no `libs/`; - built by the target's primary R minor; - bincraft's `tarball_abi_verdict()` calls it portable across every installed minor (unknown means no seed). Each copy gets the `single_builds` and `abi_flat_safety` rows an upload writes, so build-all skips it and the per-minor indexes keep it. Re-runs skip existing objects and rewrite rows idempotently. - **Interleaved shards** in `local/build-all.R` via `shard_slice()`; contiguous alphabetical quarters ranged from 0.1h to 28h. - **Shared build deadline**: `build-all-versions` sets `BUILD_DEADLINE` from `BUILD_BUDGET_HOURS` (44h, repo timeout 50h); `build-all.R` and `build-newer-r.R` stop cleanly when it passes, so archiving still runs. - `codename_platform()` replaces the hard-coded codename-to-platform list. ## Testing - New unit tests for the seed verdict, candidate filter, platform names and deadline; all `local/tests` pass. - Verdicts checked on real amd64/alpine324 tarballs: R6 2.6.1/2.5.1 and jsonstat seeded, cli refused as compiled. - `crow lint .crow/` passes. - The seed has not run against B2/Postgres yet; the first bootstrap run is its end-to-end test. ## Usage Trigger `build-all-versions` with `target_arch`, `OS`, `OS_VERSION` and `SEED_FROM="alpine324 alpine323"`. The new Alpine version still needs adding to the `OS_VERSION` options once its image exists. Reviewed-on: #242
This commit is contained in:
parent
63149e246a
commit
e23bd3e539
1 changed files with 726 additions and 51 deletions
|
|
@ -1,6 +1,7 @@
|
|||
sink(stdout(), type = "message")
|
||||
options(crayon.enabled = TRUE, future.globals.onReference = NULL)
|
||||
source(file.path("local", "r-minor-helpers.R"))
|
||||
source(file.path("local", "rebuild-missing-helpers.R"))
|
||||
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
parsed <- parse_build_args(args)
|
||||
|
|
@ -104,9 +105,10 @@ sprintf(
|
|||
sensitive_only
|
||||
)
|
||||
|
||||
# Split into chunks for this worker
|
||||
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
|
||||
chunk <- chunks[[split_index]]
|
||||
# Interleave rather than cut the alphabetical list into contiguous blocks: build
|
||||
# cost clusters by name (Rcpp*, rstan*), and contiguous quarters of the last
|
||||
# full rebuild ran anywhere from 0.1h to 28h.
|
||||
chunk <- pkgs[shard_slice(seq_len(nrow(pkgs)), split_into, split_index), ]
|
||||
sprintf("# of package versions for this job: %s", nrow(chunk))
|
||||
|
||||
# Exclude known problematic packages (single source of truth)
|
||||
|
|
@ -126,23 +128,7 @@ chunk <- chunk[!chunk$Package %in% exclude, ]
|
|||
# environment, so Sys.getenv() would return "" and this pre-filter would query
|
||||
# platform "-" and skip nothing.
|
||||
codename <- bincraft::set_codename(NULL)
|
||||
platform <- switch(
|
||||
codename,
|
||||
jammy = "ubuntu-2204",
|
||||
noble = "ubuntu-2404",
|
||||
resolute = "ubuntu-2604",
|
||||
rhel10 = "redhat-10",
|
||||
rhel9 = "redhat-9",
|
||||
rhel8 = "redhat-8",
|
||||
alpine320 = "alpine-320",
|
||||
alpine321 = "alpine-321",
|
||||
alpine322 = "alpine-322",
|
||||
alpine323 = "alpine-323",
|
||||
alpine324 = "alpine-324",
|
||||
alpine325 = "alpine-325",
|
||||
alpine326 = "alpine-326",
|
||||
NA_character_
|
||||
)
|
||||
platform <- codename_platform(codename)
|
||||
local_machine <- Sys.info()[["machine"]]
|
||||
arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64"
|
||||
con <- DBI::dbConnect(
|
||||
|
|
@ -232,35 +218,41 @@ sprintf(
|
|||
length(s3_cache_paths)
|
||||
)
|
||||
|
||||
deadline <- build_deadline()
|
||||
n <- nrow(chunk)
|
||||
mapply(
|
||||
function(pkg, ver, sens, i) {
|
||||
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
||||
bincraft::build_binary_package(
|
||||
pkg,
|
||||
tag = ver,
|
||||
is_r_minor_sensitive = isTRUE(sens),
|
||||
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
s3_region = "eu-central-003",
|
||||
s3_bucket = "devxy-rpkgs-binaries",
|
||||
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
s3_package_cache = s3_cache,
|
||||
metadata_db_host = "r-binaries.devxy.io",
|
||||
metadata_db_name = "build_metadata",
|
||||
metadata_db_table = "single_builds",
|
||||
metadata_db_user = "rpkgs",
|
||||
metadata_db_password = Sys.getenv("PGPASS"),
|
||||
metadata_db_sslmode = "verify-full",
|
||||
metadata_db_port = 15432,
|
||||
archive = TRUE,
|
||||
patches = "local/patches",
|
||||
upload = TRUE,
|
||||
store_build_metadata = TRUE
|
||||
)
|
||||
},
|
||||
chunk$Package,
|
||||
chunk$Version,
|
||||
chunk$r_minor_sensitive,
|
||||
seq_len(n)
|
||||
)
|
||||
for (i in seq_len(n)) {
|
||||
if (deadline_reached(deadline)) {
|
||||
cat(sprintf(
|
||||
"Build deadline reached after %d/%d package versions; stopping cleanly. The next run resumes from the metadata DB.\n",
|
||||
i - 1L,
|
||||
n
|
||||
))
|
||||
break
|
||||
}
|
||||
pkg <- chunk$Package[i]
|
||||
ver <- chunk$Version[i]
|
||||
sens <- chunk$r_minor_sensitive[i]
|
||||
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
||||
bincraft::build_binary_package(
|
||||
pkg,
|
||||
tag = ver,
|
||||
is_r_minor_sensitive = isTRUE(sens),
|
||||
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
s3_region = "eu-central-003",
|
||||
s3_bucket = "devxy-rpkgs-binaries",
|
||||
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
s3_package_cache = s3_cache,
|
||||
metadata_db_host = "r-binaries.devxy.io",
|
||||
metadata_db_name = "build_metadata",
|
||||
metadata_db_table = "single_builds",
|
||||
metadata_db_user = "rpkgs",
|
||||
metadata_db_password = Sys.getenv("PGPASS"),
|
||||
metadata_db_sslmode = "verify-full",
|
||||
metadata_db_port = 15432,
|
||||
archive = TRUE,
|
||||
patches = "local/patches",
|
||||
upload = TRUE,
|
||||
store_build_metadata = TRUE
|
||||
)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue