pat-s
fa955c10bf
Some checks failed
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline failed
ci/crow/cron/process-updates/1 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps/1 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline failed
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps/2 Pipeline was successful
ci/crow/manual/build-all-versions/5 Pipeline was canceled
ci/crow/manual/build-all-versions/6 Pipeline was canceled
ci/crow/manual/build-all-versions/7 Pipeline was canceled
ci/crow/manual/build-all-versions/8 Pipeline was canceled
ci/crow/manual/build-all-versions/4 Pipeline failed
ci/crow/manual/build-all-versions/1 Pipeline failed
ci/crow/manual/build-all-versions/3 Pipeline failed
ci/crow/manual/build-all-versions/2 Pipeline failed
## Summary `build-all.R` reads three snapshot files from `/mnt/cache/packages/` that the `build-all-versions-install-deps` step precomputes: `pkgs_to_build.rds`, `r_minor_sensitive_pkgs.rds`, and `s3_cache.rds`. That cache volume is **per-agent**, so a build job scheduled on a different (fresh) agent than the one that ran install-deps finds the snapshot absent and dies at `readRDS` (`cannot open compressed file '/mnt/cache/packages/pkgs_to_build.rds'`). This adds a conditional guard at the top of `build-all.R`: when any of the three files is missing, it sources `local/packages-to-build.R` (which has all needed creds via `PGPASS` / `B2_S3_*` env, already present in the build step) and writes the derived `.rds` files — exactly mirroring the install-deps command. - The first build job on a fresh agent repopulates the shared cache, so subsequent jobs on that agent reuse it. - Concurrent jobs that also miss simply redo the work (accepted tradeoff vs. slow shared storage like NFS). - Saves use a temp-file + atomic `file.rename`, so a concurrent reader never sees a half-written `.rds`. Reviewed-on: #95
128 lines
4.8 KiB
R
128 lines
4.8 KiB
R
sink(stdout(), type = "message")
|
|
options(crayon.enabled = TRUE, future.globals.onReference = NULL)
|
|
source(file.path("local", "r-minor-helpers.R"))
|
|
|
|
args <- commandArgs(trailingOnly = TRUE)
|
|
parsed <- parse_build_args(args)
|
|
split_into <- parsed$split_into
|
|
split_index <- parsed$split_index
|
|
ncpus <- parsed$ncpus
|
|
sensitive_only <- parsed$sensitive_only
|
|
options(Ncpus = ncpus)
|
|
|
|
# Load bincraft eagerly to avoid lazy-load memory spike during first build call
|
|
library(bincraft, quietly = TRUE)
|
|
library(future)
|
|
plan("sequential")
|
|
|
|
# The install-deps step precomputes the package snapshot into /mnt/cache, but
|
|
# that volume is per-agent: a job landing on a fresh agent (or racing
|
|
# install-deps) finds it empty. Recompute the snapshot here when any part is
|
|
# missing, so the first job on an agent repopulates the cache for the jobs that
|
|
# follow; concurrent jobs that also miss simply redo the work. Write via a
|
|
# temp file + atomic rename so a concurrent reader never sees a half-written rds.
|
|
package_cache_files <- c(
|
|
"/mnt/cache/packages/pkgs_to_build.rds",
|
|
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds",
|
|
"/mnt/cache/packages/s3_cache.rds"
|
|
)
|
|
if (!all(file.exists(package_cache_files))) {
|
|
message("Package snapshot missing from cache; recomputing via packages-to-build.R")
|
|
dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
|
|
save_rds_atomic <- function(obj, path) {
|
|
tmp <- paste0(path, ".tmp.", Sys.getpid())
|
|
saveRDS(obj, tmp)
|
|
file.rename(tmp, path)
|
|
}
|
|
source(file.path("local", "packages-to-build.R"))
|
|
save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds")
|
|
save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
|
message("Package snapshot recomputed.")
|
|
}
|
|
|
|
pkgs <- if (sensitive_only) {
|
|
readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
|
} else {
|
|
readRDS("/mnt/cache/packages/pkgs_to_build.rds")
|
|
}
|
|
# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive)
|
|
if (is.null(pkgs$r_minor_sensitive)) {
|
|
pkgs$r_minor_sensitive <- FALSE
|
|
}
|
|
sprintf(
|
|
"Total# of remaining package versions: %s (sensitive_only=%s)",
|
|
nrow(pkgs),
|
|
sensitive_only
|
|
)
|
|
|
|
# Split into chunks for this worker
|
|
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
|
|
chunk <- chunks[[split_index]]
|
|
sprintf("# of package versions for this job: %s", nrow(chunk))
|
|
|
|
# Exclude known problematic packages (single source of truth)
|
|
exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]]
|
|
chunk <- chunk[!chunk$Package %in% exclude, ]
|
|
|
|
# Skip package versions already built in a previous run.
|
|
# pkgs_to_build.rds is a static snapshot from the install-deps step, so on a
|
|
# restart it still lists everything an interrupted run already produced. The
|
|
# metadata DB reflects that progress, so we re-derive the remaining set here.
|
|
platform <- paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-")
|
|
arch <- Sys.getenv("ARCH")
|
|
con <- DBI::dbConnect(
|
|
RPostgres::Postgres(),
|
|
dbname = "build_metadata",
|
|
host = "r-binaries.devxy.io",
|
|
port = 15432,
|
|
user = "rpkgs",
|
|
password = Sys.getenv("PGPASS"),
|
|
sslmode = "require"
|
|
)
|
|
built <- DBI::dbGetQuery(
|
|
con,
|
|
"SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2 AND error_occurred = FALSE",
|
|
params = list(platform, arch)
|
|
)
|
|
DBI::dbDisconnect(con)
|
|
before <- nrow(chunk)
|
|
chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ]
|
|
sprintf("Skipped %d already-built package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk))
|
|
|
|
# Read pre-computed S3 listing from install-deps step
|
|
# This avoids loading s3fs/reticulate/Python in the build container,
|
|
# saving significant memory for pak subprocess forks
|
|
s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
|
sprintf("S3 cache: %s files", length(s3_cache))
|
|
|
|
n <- nrow(chunk)
|
|
mapply(
|
|
function(pkg, ver, sens, i) {
|
|
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
|
bincraft::build_binary_package(
|
|
pkg,
|
|
tag = ver,
|
|
is_r_minor_sensitive = isTRUE(sens),
|
|
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
|
s3_region = "eu-central-003",
|
|
s3_bucket = "devxy-rpkgs-binaries",
|
|
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
|
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
|
s3_package_cache = s3_cache,
|
|
metadata_db_host = "r-binaries.devxy.io",
|
|
metadata_db_name = "build_metadata",
|
|
metadata_db_table = "single_builds",
|
|
metadata_db_user = "rpkgs",
|
|
metadata_db_password = Sys.getenv("PGPASS"),
|
|
metadata_db_sslmode = "require",
|
|
metadata_db_port = 15432,
|
|
archive = TRUE,
|
|
upload = TRUE,
|
|
store_build_metadata = TRUE
|
|
)
|
|
},
|
|
chunk$Package,
|
|
chunk$Version,
|
|
chunk$r_minor_sensitive,
|
|
seq_len(n)
|
|
)
|