pat-s
83c761b736
## Summary
Two changes:
1. **`OS`/`OS_VERSION` manual-run dropdowns** — give these form variables explicit `options:` lists (like `target_arch` and `R_VERSION`), so the manual-run form shows dropdowns instead of free-text, in both `build-all-versions.yaml` and `build-all-versions-install-deps.yaml`. Crow form variables are independent (no cascading), so the operator still has to pick a coherent `OS` + `OS_VERSION` combination (e.g. `redhat` + `9`, not `alpine` + `jammy`).
2. **Repairs `main`** — the crow fix from PR #96 (`bc2f6f1`) was lost when that PR was squashed (only the first commit was captured). As a result `main` currently carries the `OS: ${OS}` env vars that break Crow parsing (`unable to parse variable name`) and the unfixed `build-all.R`. This PR re-applies that fix: drop the env additions and derive `platform`/`arch` inside `build-all.R` from the container (bincraft codename → platform mapping + `Sys.info()` arch).
## Notes
- `OS_VERSION` options are quoted strings so tags like `8`/`9`/`10` aren't parsed as integers.
- Validated: both YAMLs parse, `build-all.R` parses.
Reviewed-on: #97
151 lines
5.5 KiB
R
151 lines
5.5 KiB
R
sink(stdout(), type = "message")
|
|
options(crayon.enabled = TRUE, future.globals.onReference = NULL)
|
|
source(file.path("local", "r-minor-helpers.R"))
|
|
|
|
args <- commandArgs(trailingOnly = TRUE)
|
|
parsed <- parse_build_args(args)
|
|
split_into <- parsed$split_into
|
|
split_index <- parsed$split_index
|
|
ncpus <- parsed$ncpus
|
|
sensitive_only <- parsed$sensitive_only
|
|
options(Ncpus = ncpus)
|
|
|
|
# Load bincraft eagerly to avoid lazy-load memory spike during first build call
|
|
library(bincraft, quietly = TRUE)
|
|
library(future)
|
|
plan("sequential")
|
|
|
|
# The install-deps step precomputes the package snapshot into /mnt/cache, but
|
|
# that volume is per-agent: a job landing on a fresh agent (or racing
|
|
# install-deps) finds it empty. Recompute the snapshot here when any part is
|
|
# missing, so the first job on an agent repopulates the cache for the jobs that
|
|
# follow; concurrent jobs that also miss simply redo the work. Write via a
|
|
# temp file + atomic rename so a concurrent reader never sees a half-written rds.
|
|
package_cache_files <- c(
|
|
"/mnt/cache/packages/pkgs_to_build.rds",
|
|
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds",
|
|
"/mnt/cache/packages/s3_cache.rds"
|
|
)
|
|
if (!all(file.exists(package_cache_files))) {
|
|
message("Package snapshot missing from cache; recomputing via packages-to-build.R")
|
|
dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
|
|
save_rds_atomic <- function(obj, path) {
|
|
tmp <- paste0(path, ".tmp.", Sys.getpid())
|
|
saveRDS(obj, tmp)
|
|
file.rename(tmp, path)
|
|
}
|
|
source(file.path("local", "packages-to-build.R"))
|
|
save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds")
|
|
save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
|
message("Package snapshot recomputed.")
|
|
}
|
|
|
|
pkgs <- if (sensitive_only) {
|
|
readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
|
} else {
|
|
readRDS("/mnt/cache/packages/pkgs_to_build.rds")
|
|
}
|
|
# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive)
|
|
if (is.null(pkgs$r_minor_sensitive)) {
|
|
pkgs$r_minor_sensitive <- FALSE
|
|
}
|
|
sprintf(
|
|
"Total# of remaining package versions: %s (sensitive_only=%s)",
|
|
nrow(pkgs),
|
|
sensitive_only
|
|
)
|
|
|
|
# Split into chunks for this worker
|
|
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
|
|
chunk <- chunks[[split_index]]
|
|
sprintf("# of package versions for this job: %s", nrow(chunk))
|
|
|
|
# Exclude known problematic packages (single source of truth)
|
|
exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]]
|
|
chunk <- chunk[!chunk$Package %in% exclude, ]
|
|
|
|
# Skip package versions already built in a previous run.
|
|
# pkgs_to_build.rds is a static snapshot from the install-deps step, so on a
|
|
# restart it still lists everything an interrupted run already produced. The
|
|
# metadata DB reflects that progress, so we re-derive the remaining set here.
|
|
# Derive platform + arch from the running container, mirroring the codename ->
|
|
# platform mapping bincraft uses internally. The OS/OS_VERSION selectors are
|
|
# workflow-level CI variables that are not injected into the container
|
|
# environment, so Sys.getenv() would return "" and this pre-filter would query
|
|
# platform "-" and skip nothing.
|
|
codename <- bincraft::set_codename(NULL)
|
|
platform <- switch(
|
|
codename,
|
|
jammy = "ubuntu-2204",
|
|
noble = "ubuntu-2404",
|
|
resolute = "ubuntu-2604",
|
|
rhel10 = "redhat-10",
|
|
rhel9 = "redhat-9",
|
|
rhel8 = "redhat-8",
|
|
alpine320 = "alpine-320",
|
|
alpine321 = "alpine-321",
|
|
alpine322 = "alpine-322",
|
|
alpine323 = "alpine-323",
|
|
alpine324 = "alpine-324",
|
|
alpine325 = "alpine-325",
|
|
alpine326 = "alpine-326",
|
|
NA_character_
|
|
)
|
|
local_machine <- Sys.info()[["machine"]]
|
|
arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64"
|
|
con <- DBI::dbConnect(
|
|
RPostgres::Postgres(),
|
|
dbname = "build_metadata",
|
|
host = "r-binaries.devxy.io",
|
|
port = 15432,
|
|
user = "rpkgs",
|
|
password = Sys.getenv("PGPASS"),
|
|
sslmode = "require"
|
|
)
|
|
built <- DBI::dbGetQuery(
|
|
con,
|
|
"SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2 AND error_occurred = FALSE",
|
|
params = list(platform, arch)
|
|
)
|
|
DBI::dbDisconnect(con)
|
|
before <- nrow(chunk)
|
|
chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ]
|
|
sprintf("Skipped %d already-built package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk))
|
|
|
|
# Read pre-computed S3 listing from install-deps step
|
|
# This avoids loading s3fs/reticulate/Python in the build container,
|
|
# saving significant memory for pak subprocess forks
|
|
s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
|
sprintf("S3 cache: %s files", length(s3_cache))
|
|
|
|
n <- nrow(chunk)
|
|
mapply(
|
|
function(pkg, ver, sens, i) {
|
|
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
|
bincraft::build_binary_package(
|
|
pkg,
|
|
tag = ver,
|
|
is_r_minor_sensitive = isTRUE(sens),
|
|
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
|
s3_region = "eu-central-003",
|
|
s3_bucket = "devxy-rpkgs-binaries",
|
|
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
|
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
|
s3_package_cache = s3_cache,
|
|
metadata_db_host = "r-binaries.devxy.io",
|
|
metadata_db_name = "build_metadata",
|
|
metadata_db_table = "single_builds",
|
|
metadata_db_user = "rpkgs",
|
|
metadata_db_password = Sys.getenv("PGPASS"),
|
|
metadata_db_sslmode = "require",
|
|
metadata_db_port = 15432,
|
|
archive = TRUE,
|
|
upload = TRUE,
|
|
store_build_metadata = TRUE
|
|
)
|
|
},
|
|
chunk$Package,
|
|
chunk$Version,
|
|
chunk$r_minor_sensitive,
|
|
seq_len(n)
|
|
)
|