perf: seed pure-R binaries into a new slot, interleave shards, and stop builds before the timeout

- Add local/seed-pure-r.R, opt-in via SEED_FROM on install-deps: copy pure-R
  binaries server-side from a slot of the same OS family and arch, after
  verifying each is a binary, has no compiled code, was built by the primary
  R minor, and passes bincraft's cross-minor ABI check; write the
  single_builds and abi_flat_safety rows an upload would.
- Interleave build-all shards with shard_slice() instead of contiguous
  alphabetical blocks, which ranged from 0.1h to 28h per shard.
- Share one BUILD_DEADLINE (BUILD_BUDGET_HOURS, default 44h) across every
  build pass of a build-all-versions step so long runs stop cleanly and
  still archive and index what they built.
- Derive the single_builds platform with codename_platform() instead of a
  hard-coded list.
This commit is contained in:
Patrick Schratz 2026-10-24 01:58:25 +00:00
commit cdfdcd035d
No known key found for this signature in database
GPG key ID: 62050D5BC68AB6DC

View file

@ -1,6 +1,7 @@
sink(stdout(), type = "message")
options(crayon.enabled = TRUE, future.globals.onReference = NULL)
source(file.path("local", "r-minor-helpers.R"))
source(file.path("local", "rebuild-missing-helpers.R"))
args <- commandArgs(trailingOnly = TRUE)
parsed <- parse_build_args(args)
@ -104,9 +105,10 @@ sprintf(
sensitive_only
)
# Split into chunks for this worker
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
chunk <- chunks[[split_index]]
# Interleave rather than cut the alphabetical list into contiguous blocks: build
# cost clusters by name (Rcpp*, rstan*), and contiguous quarters of the last
# full rebuild ran anywhere from 0.1h to 28h.
chunk <- pkgs[shard_slice(seq_len(nrow(pkgs)), split_into, split_index), ]
sprintf("# of package versions for this job: %s", nrow(chunk))
# Exclude known problematic packages (single source of truth)
@ -126,23 +128,7 @@ chunk <- chunk[!chunk$Package %in% exclude, ]
# environment, so Sys.getenv() would return "" and this pre-filter would query
# platform "-" and skip nothing.
codename <- bincraft::set_codename(NULL)
platform <- switch(
codename,
jammy = "ubuntu-2204",
noble = "ubuntu-2404",
resolute = "ubuntu-2604",
rhel10 = "redhat-10",
rhel9 = "redhat-9",
rhel8 = "redhat-8",
alpine320 = "alpine-320",
alpine321 = "alpine-321",
alpine322 = "alpine-322",
alpine323 = "alpine-323",
alpine324 = "alpine-324",
alpine325 = "alpine-325",
alpine326 = "alpine-326",
NA_character_
)
platform <- codename_platform(codename)
local_machine <- Sys.info()[["machine"]]
arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64"
con <- DBI::dbConnect(
@ -232,35 +218,41 @@ sprintf(
length(s3_cache_paths)
)
deadline <- build_deadline()
n <- nrow(chunk)
mapply(
function(pkg, ver, sens, i) {
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
bincraft::build_binary_package(
pkg,
tag = ver,
is_r_minor_sensitive = isTRUE(sens),
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
s3_region = "eu-central-003",
s3_bucket = "devxy-rpkgs-binaries",
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
s3_package_cache = s3_cache,
metadata_db_host = "r-binaries.devxy.io",
metadata_db_name = "build_metadata",
metadata_db_table = "single_builds",
metadata_db_user = "rpkgs",
metadata_db_password = Sys.getenv("PGPASS"),
metadata_db_sslmode = "verify-full",
metadata_db_port = 15432,
archive = TRUE,
patches = "local/patches",
upload = TRUE,
store_build_metadata = TRUE
)
},
chunk$Package,
chunk$Version,
chunk$r_minor_sensitive,
seq_len(n)
)
for (i in seq_len(n)) {
if (deadline_reached(deadline)) {
cat(sprintf(
"Build deadline reached after %d/%d package versions; stopping cleanly. The next run resumes from the metadata DB.\n",
i - 1L,
n
))
break
}
pkg <- chunk$Package[i]
ver <- chunk$Version[i]
sens <- chunk$r_minor_sensitive[i]
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
bincraft::build_binary_package(
pkg,
tag = ver,
is_r_minor_sensitive = isTRUE(sens),
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
s3_region = "eu-central-003",
s3_bucket = "devxy-rpkgs-binaries",
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
s3_package_cache = s3_cache,
metadata_db_host = "r-binaries.devxy.io",
metadata_db_name = "build_metadata",
metadata_db_table = "single_builds",
metadata_db_user = "rpkgs",
metadata_db_password = Sys.getenv("PGPASS"),
metadata_db_sslmode = "verify-full",
metadata_db_port = 15432,
archive = TRUE,
patches = "local/patches",
upload = TRUE,
store_build_metadata = TRUE
)
}