pat-s
d668dd96c2
build-all reads a static pkgs_to_build.rds produced once by the install-deps step, so a restarted build job re-cycles every package an interrupted run already produced. Query the build-metadata DB at job start and drop the (Package, Version) pairs already built successfully for this platform/arch, so a restart only processes what is genuinely left. Relies on bincraft writing the success row only after a confirmed S3 upload (rpkgs/bincraft#56), so a DB success guarantees the binary is published.
82 lines
3 KiB
R
82 lines
3 KiB
R
sink(stdout(), type = "message")
|
|
options(crayon.enabled = TRUE, future.globals.onReference = NULL)
|
|
|
|
args <- commandArgs(trailingOnly = TRUE)
|
|
split_into <- as.integer(args[1])
|
|
split_index <- as.integer(args[2])
|
|
ncpus <- as.integer(args[3])
|
|
options(Ncpus = ncpus)
|
|
|
|
# Load bincraft eagerly to avoid lazy-load memory spike during first build call
|
|
library(bincraft, quietly = TRUE)
|
|
library(future)
|
|
plan("sequential")
|
|
|
|
# Read precomputed package+version pairs
|
|
pkgs <- readRDS("/mnt/cache/packages/pkgs_to_build.rds")
|
|
sprintf("Total# of remaining package versions: %s", nrow(pkgs))
|
|
|
|
# Split into chunks for this worker
|
|
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
|
|
chunk <- chunks[[split_index]]
|
|
sprintf("# of package versions for this job: %s", nrow(chunk))
|
|
|
|
# Exclude known problematic packages (single source of truth)
|
|
exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]]
|
|
chunk <- chunk[!chunk$Package %in% exclude, ]
|
|
|
|
# Skip package versions already built in a previous run.
|
|
# pkgs_to_build.rds is a static snapshot from the install-deps step, so on a
|
|
# restart it still lists everything an interrupted run already produced. The
|
|
# metadata DB reflects that progress, so we re-derive the remaining set here.
|
|
platform <- paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-")
|
|
arch <- Sys.getenv("ARCH")
|
|
con <- DBI::dbConnect(
|
|
RPostgres::Postgres(),
|
|
dbname = "build_metadata",
|
|
host = "r-binaries.devxy.io",
|
|
port = 15432,
|
|
user = "rpkgs",
|
|
password = Sys.getenv("PGPASS"),
|
|
sslmode = "require"
|
|
)
|
|
built <- DBI::dbGetQuery(
|
|
con,
|
|
"SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2 AND error_occurred = FALSE",
|
|
params = list(platform, arch)
|
|
)
|
|
DBI::dbDisconnect(con)
|
|
before <- nrow(chunk)
|
|
chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ]
|
|
sprintf("Skipped %d already-built package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk))
|
|
|
|
# Read pre-computed S3 listing from install-deps step
|
|
# This avoids loading s3fs/reticulate/Python in the build container,
|
|
# saving significant memory for pak subprocess forks
|
|
s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
|
sprintf("S3 cache: %s files", length(s3_cache))
|
|
|
|
n <- nrow(chunk)
|
|
mapply(function(pkg, ver, i) {
|
|
cat(sprintf("[%d/%d] %s_%s\n", i, n, pkg, ver))
|
|
bincraft::build_binary_package(
|
|
pkg,
|
|
tag = ver,
|
|
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
|
s3_region = "eu-central-003",
|
|
s3_bucket = "devxy-rpkgs-binaries",
|
|
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
|
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
|
s3_package_cache = s3_cache,
|
|
metadata_db_host = "r-binaries.devxy.io",
|
|
metadata_db_name = "build_metadata",
|
|
metadata_db_table = "single_builds",
|
|
metadata_db_user = "rpkgs",
|
|
metadata_db_password = Sys.getenv("PGPASS"),
|
|
metadata_db_sslmode = "require",
|
|
metadata_db_port = 15432,
|
|
archive = TRUE,
|
|
upload = TRUE,
|
|
store_build_metadata = TRUE
|
|
)
|
|
}, chunk$Package, chunk$Version, seq_len(n))
|