pat-s
85d14ffed5The guard asked only whether the snapshot files exist. They describe S3 and CRAN state at the moment they were written, and the cache volume is per-agent, so an agent that ran an earlier pipeline keeps serving that pipeline's answer indefinitely. arm64/alpine324 reported '0 remaining' for both 4.4 and 4.6 from a snapshot listing 43 r-minor-sensitive packages, while the install-deps step of the same pipeline had just computed 6871 on another agent. Both numbers appear in one log: line 49 says 6871, line 99 says 43. That is why the slot looked finished in minutes while its own index step dropped 2407 and 2436 packages as missing. Keyed on the pipeline when the CI exposes an identifier, so a new pipeline recomputes once per agent and its shards share the result; otherwise an age check with a two hour default. Verified across five cases: missing, same pipeline, new pipeline, recent without a CI variable, and aged out.
226 lines
8.1 KiB
R
226 lines
8.1 KiB
R
sink(stdout(), type = "message")
|
|
options(crayon.enabled = TRUE, future.globals.onReference = NULL)
|
|
source(file.path("local", "r-minor-helpers.R"))
|
|
|
|
args <- commandArgs(trailingOnly = TRUE)
|
|
parsed <- parse_build_args(args)
|
|
split_into <- parsed$split_into
|
|
split_index <- parsed$split_index
|
|
ncpus <- parsed$ncpus
|
|
sensitive_only <- parsed$sensitive_only
|
|
options(Ncpus = ncpus)
|
|
|
|
# Load bincraft eagerly to avoid lazy-load memory spike during first build call
|
|
library(bincraft, quietly = TRUE)
|
|
library(future)
|
|
plan("sequential")
|
|
|
|
# The install-deps step precomputes the package snapshot into /mnt/cache, but
|
|
# that volume is per-agent: a job landing on a fresh agent (or racing
|
|
# install-deps) finds it empty. Recompute the snapshot here when any part is
|
|
# missing, so the first job on an agent repopulates the cache for the jobs that
|
|
# follow; concurrent jobs that also miss simply redo the work. Write via a
|
|
# temp file + atomic rename so a concurrent reader never sees a half-written rds.
|
|
package_cache_files <- c(
|
|
"/mnt/cache/packages/pkgs_to_build.rds",
|
|
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds",
|
|
"/mnt/cache/packages/s3_cache.rds"
|
|
)
|
|
# Existence is not freshness. The snapshot describes S3 and CRAN state at the
|
|
# moment it was written, and the volume is per-agent, so an agent that ran an
|
|
# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324
|
|
# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43
|
|
# sensitive packages, while the install-deps step in the very same pipeline had
|
|
# just computed 6871 on another agent.
|
|
#
|
|
# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes
|
|
# once per agent and its shards then share the result. Off CI, or when no such
|
|
# variable is set, fall back to an age check.
|
|
snapshot_id_path <- "/mnt/cache/packages/snapshot.id"
|
|
snapshot_ttl_hours <- as.numeric(
|
|
Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2")
|
|
)
|
|
current_snapshot_id <- ""
|
|
for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) {
|
|
val <- Sys.getenv(v, unset = "")
|
|
if (nzchar(val)) {
|
|
current_snapshot_id <- paste(v, val, sep = "=")
|
|
break
|
|
}
|
|
}
|
|
|
|
snapshot_is_stale <- function() {
|
|
if (!all(file.exists(package_cache_files))) {
|
|
return(TRUE)
|
|
}
|
|
if (nzchar(current_snapshot_id)) {
|
|
cached <- tryCatch(
|
|
readLines(snapshot_id_path, warn = FALSE)[1L],
|
|
error = function(e) NA_character_,
|
|
warning = function(w) NA_character_
|
|
)
|
|
return(!identical(cached, current_snapshot_id))
|
|
}
|
|
age_hours <- as.numeric(
|
|
difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours")
|
|
)
|
|
isTRUE(age_hours > snapshot_ttl_hours)
|
|
}
|
|
|
|
if (snapshot_is_stale()) {
|
|
message(
|
|
"Package snapshot missing or stale; recomputing via packages-to-build.R"
|
|
)
|
|
dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
|
|
save_rds_atomic <- function(obj, path) {
|
|
tmp <- paste0(path, ".tmp.", Sys.getpid())
|
|
saveRDS(obj, tmp)
|
|
file.rename(tmp, path)
|
|
}
|
|
source(file.path("local", "packages-to-build.R"))
|
|
save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds")
|
|
save_rds_atomic(
|
|
pkgs[r_minor_sensitive == TRUE],
|
|
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds"
|
|
)
|
|
if (nzchar(current_snapshot_id)) {
|
|
writeLines(current_snapshot_id, snapshot_id_path)
|
|
}
|
|
message("Package snapshot recomputed.")
|
|
}
|
|
|
|
pkgs <- if (sensitive_only) {
|
|
readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
|
} else {
|
|
readRDS("/mnt/cache/packages/pkgs_to_build.rds")
|
|
}
|
|
# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive)
|
|
if (is.null(pkgs$r_minor_sensitive)) {
|
|
pkgs$r_minor_sensitive <- FALSE
|
|
}
|
|
sprintf(
|
|
"Total# of remaining package versions: %s (sensitive_only=%s)",
|
|
nrow(pkgs),
|
|
sensitive_only
|
|
)
|
|
|
|
# Split into chunks for this worker
|
|
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
|
|
chunk <- chunks[[split_index]]
|
|
sprintf("# of package versions for this job: %s", nrow(chunk))
|
|
|
|
# Exclude known problematic packages (single source of truth)
|
|
exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]]
|
|
chunk <- chunk[!chunk$Package %in% exclude, ]
|
|
|
|
# Skip package versions already attempted in a previous run (built or errored).
|
|
# pkgs_to_build.rds is a static snapshot from the install-deps step, so on a
|
|
# restart it still lists everything an interrupted run already produced. The
|
|
# metadata DB reflects that progress, so we re-derive the remaining set here.
|
|
# We exclude *all* attempted versions, not just successful ones: a previously
|
|
# errored version is skipped by build_binary_package() anyway, so leaving it in
|
|
# the chunk only makes the job cycle through it one-by-one for no benefit.
|
|
# Derive platform + arch from the running container, mirroring the codename ->
|
|
# platform mapping bincraft uses internally. The OS/OS_VERSION selectors are
|
|
# workflow-level CI variables that are not injected into the container
|
|
# environment, so Sys.getenv() would return "" and this pre-filter would query
|
|
# platform "-" and skip nothing.
|
|
codename <- bincraft::set_codename(NULL)
|
|
platform <- switch(
|
|
codename,
|
|
jammy = "ubuntu-2204",
|
|
noble = "ubuntu-2404",
|
|
resolute = "ubuntu-2604",
|
|
rhel10 = "redhat-10",
|
|
rhel9 = "redhat-9",
|
|
rhel8 = "redhat-8",
|
|
alpine320 = "alpine-320",
|
|
alpine321 = "alpine-321",
|
|
alpine322 = "alpine-322",
|
|
alpine323 = "alpine-323",
|
|
alpine324 = "alpine-324",
|
|
alpine325 = "alpine-325",
|
|
alpine326 = "alpine-326",
|
|
NA_character_
|
|
)
|
|
local_machine <- Sys.info()[["machine"]]
|
|
arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64"
|
|
con <- DBI::dbConnect(
|
|
RPostgres::Postgres(),
|
|
dbname = "build_metadata",
|
|
host = "r-binaries.devxy.io",
|
|
port = 15432,
|
|
user = "rpkgs",
|
|
password = Sys.getenv("PGPASS"),
|
|
sslmode = "require"
|
|
)
|
|
# Scope the skip to the R minor this pass is running under. `single_builds`
|
|
# records `r_version` per attempt, but querying without it made a non-primary
|
|
# pass skip everything the primary pass had already attempted under a different
|
|
# minor - so `--sensitive-only` under 4.6 skipped packages that had only ever
|
|
# been built for 4.5, and the per-minor slots never filled. That is why
|
|
# amd64/resolute served 4000 fewer packages to a 4.6 client than to a 4.5 one.
|
|
r_minor <- paste(
|
|
R.version$major,
|
|
strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L],
|
|
sep = "."
|
|
)
|
|
built <- DBI::dbGetQuery(
|
|
con,
|
|
paste(
|
|
"SELECT name, tag FROM single_builds",
|
|
"WHERE platform = $1 AND arch = $2",
|
|
"AND substring(r_version from '^[0-9]+[.][0-9]+') = $3"
|
|
),
|
|
params = list(platform, arch, r_minor)
|
|
)
|
|
DBI::dbDisconnect(con)
|
|
before <- nrow(chunk)
|
|
chunk <- chunk[
|
|
!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag),
|
|
]
|
|
sprintf(
|
|
"Skipped %d package versions already attempted under R %s; %d remaining for this job",
|
|
before - nrow(chunk),
|
|
r_minor,
|
|
nrow(chunk)
|
|
)
|
|
|
|
# Read pre-computed S3 listing from install-deps step
|
|
# This avoids loading s3fs/reticulate/Python in the build container,
|
|
# saving significant memory for the dependency-installer subprocesses
|
|
s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
|
sprintf("S3 cache: %s files", length(s3_cache))
|
|
|
|
n <- nrow(chunk)
|
|
mapply(
|
|
function(pkg, ver, sens, i) {
|
|
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
|
bincraft::build_binary_package(
|
|
pkg,
|
|
tag = ver,
|
|
is_r_minor_sensitive = isTRUE(sens),
|
|
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
|
s3_region = "eu-central-003",
|
|
s3_bucket = "devxy-rpkgs-binaries",
|
|
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
|
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
|
s3_package_cache = s3_cache,
|
|
metadata_db_host = "r-binaries.devxy.io",
|
|
metadata_db_name = "build_metadata",
|
|
metadata_db_table = "single_builds",
|
|
metadata_db_user = "rpkgs",
|
|
metadata_db_password = Sys.getenv("PGPASS"),
|
|
metadata_db_sslmode = "require",
|
|
metadata_db_port = 15432,
|
|
archive = TRUE,
|
|
patches = "local/patches",
|
|
upload = TRUE,
|
|
store_build_metadata = TRUE
|
|
)
|
|
},
|
|
chunk$Package,
|
|
chunk$Version,
|
|
chunk$r_minor_sensitive,
|
|
seq_len(n)
|
|
)
|