From 9b27b3e6501db1de886990263322a79c62cd86f6 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 17 Apr 2026 19:01:08 +0200 Subject: [PATCH] refactor: extract build-all R one-liner into local/build-all.R - Load bincraft eagerly via library() to avoid lazy-load memory spike - Do bulk S3 listing upfront and pass as s3_package_cache to avoid per-package S3 calls that accumulate memory and trigger fork failures - Consolidate amd64/arm64 exclude lists into single script - Much easier to read and maintain than a YAML-embedded one-liner --- .crow/build-all-versions-amd64.yaml | 2 +- .crow/build-all-versions-arm64.yaml | 2 +- local/build-all.R | 82 +++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 local/build-all.R diff --git a/.crow/build-all-versions-amd64.yaml b/.crow/build-all-versions-amd64.yaml index c2aeec7..4c10387 100644 --- a/.crow/build-all-versions-amd64.yaml +++ b/.crow/build-all-versions-amd64.yaml @@ -69,7 +69,7 @@ steps: # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' # pkgs = readRDS('/mnt/cache/pkgs_amd64.rds'); - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = readRDS('/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Total# of remaining package versions:%s', nrow(pkgs)); chunks = split(pkgs, cut(seq_len(nrow(pkgs)), $SPLIT_INTO, labels = FALSE)); chunk = chunks[[$SPLIT_INDEX]]; sprintf('# of package versions for this job:%s', nrow(chunk)); exclude = c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'Rfast', 'compareGroups', 'ff', 'GsymPoint', 'RcppDynProg', 'comtradr', 'RcppDynProg', 'FD', 'PearsonDS', 'DCluster', 'gRc', 'mixlm', 'geospt', 'fdth', 'ffmanova', 'fiery', 'ffscrapr', 'cold', 'fiery', 'RcmdrPlugin.DoE', 'RcmdrPlugin.NMBU', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.UCA', 'RcmdrPlugin.WorldFlora', 'RcmdrPlugin.aRnova', 'RcmdrPlugin.depthTools', 'RcmdrPlugin.orloca', 'RcmdrPlugin.sos', 'RcmdrPlugin.survival', 'RcmdrPlugin.temis', 'GWlasso', 'GWmodelVis'); chunk = chunk[!chunk\$Package %in% exclude, ]; library(future); plan('sequential'); n = nrow(chunk); mapply(function(pkg, ver, i) { cat(sprintf('[%d/%d] %s_%s\n', i, n, pkg, ver)); bincraft::build_binary_package(pkg, tag = ver, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE) }, chunk\$Package, chunk\$Version, seq_len(n))" 2>&1 + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 # archive missed packages - R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" backend_options: diff --git a/.crow/build-all-versions-arm64.yaml b/.crow/build-all-versions-arm64.yaml index 7f47f70..eb1a5c2 100644 --- a/.crow/build-all-versions-arm64.yaml +++ b/.crow/build-all-versions-arm64.yaml @@ -82,7 +82,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = readRDS('/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Total# of remaining package versions:%s', nrow(pkgs)); chunks = split(pkgs, cut(seq_len(nrow(pkgs)), $SPLIT_INTO, labels = FALSE)); chunk = chunks[[$SPLIT_INDEX]]; sprintf('# of package versions for this job:%s', nrow(chunk)); exclude = c('RInno', 'RcmdrPlugin.DoE', 'RcmdrPlugin.NMBU', 'RcmdrPlugin.ROC', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.UCA', 'RcmdrPlugin.WorldFlora', 'RcmdrPlugin.aRnova', 'RcmdrPlugin.depthTools', 'RcmdrPlugin.orloca', 'RcmdrPlugin.sos', 'RcmdrPlugin.survival', 'RcmdrPlugin.temis'); chunk = chunk[!chunk\$Package %in% exclude, ]; library(future); plan('sequential'); n = nrow(chunk); mapply(function(pkg, ver, i) { cat(sprintf('[%d/%d] %s_%s\n', i, n, pkg, ver)); bincraft::build_binary_package(pkg, tag = ver, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE) }, chunk\$Package, chunk\$Version, seq_len(n))" 2>&1 + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 # archive missed packages - R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" backend_options: diff --git a/local/build-all.R b/local/build-all.R new file mode 100644 index 0000000..3476a22 --- /dev/null +++ b/local/build-all.R @@ -0,0 +1,82 @@ +sink(stdout(), type = "message") +options(crayon.enabled = TRUE, future.globals.onReference = NULL) + +args <- commandArgs(trailingOnly = TRUE) +split_into <- as.integer(args[1]) +split_index <- as.integer(args[2]) +ncpus <- as.integer(args[3]) +options(Ncpus = ncpus) + +# Load bincraft eagerly to avoid lazy-load memory spike during first build call +library(bincraft, quietly = TRUE) +library(future) +plan("sequential") + +# Read precomputed package+version pairs +pkgs <- readRDS("/mnt/cache/packages/pkgs_to_build.rds") +sprintf("Total# of remaining package versions: %s", nrow(pkgs)) + +# Split into chunks for this worker +chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE)) +chunk <- chunks[[split_index]] +sprintf("# of package versions for this job: %s", nrow(chunk)) + +# Exclude known problematic packages +exclude <- c( + "RInno", "KeyboardSimulator", "R2PPT", "RWinEdt", "blatr", "excel.link", + "spectrino", "taskscheduleR", "MDSGUI", "BiplotGUI", "R2wd", "rFUSION", + "MediaNews", "doBy", "IDPmisc", "frailtypack", "afex", "FrF2", "DoE.base", + "agricolae", "doFuture", "fscaret", "PHYLOGR", "seewave", "pls", "relaimpo", + "geepack", "gggenes", "NPCirc", "repmis", "PNDSIBGE", "lidR", "poismf", + "neonstore", "MachineShop", "mvst", "MacBehaviour", "mcmcderive", "RGIFT", + "KnowBR", "netmeta", "spdep", "Rfast", "compareGroups", "ff", "GsymPoint", + "RcppDynProg", "comtradr", "FD", "PearsonDS", "DCluster", "gRc", "mixlm", + "geospt", "fdth", "ffmanova", "fiery", "ffscrapr", "cold", + "RcmdrPlugin.DoE", "RcmdrPlugin.NMBU", "RcmdrPlugin.RiskDemo", + "RcmdrPlugin.ROC", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", + "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.aRnova", + "RcmdrPlugin.depthTools", "RcmdrPlugin.orloca", "RcmdrPlugin.sos", + "RcmdrPlugin.survival", "RcmdrPlugin.temis", "GWlasso", "GWmodelVis" +) +chunk <- chunk[!chunk$Package %in% exclude, ] + +# Bulk S3 listing upfront — avoids per-package S3 calls +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) +codename <- bincraft::set_codename(NULL) +s3_cache <- basename(s3fs::s3_dir_ls( + sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", + Sys.info()[["machine"]], codename), + recurse = TRUE +)) +sprintf("S3 cache: %s files", length(s3_cache)) + +n <- nrow(chunk) +mapply(function(pkg, ver, i) { + cat(sprintf("[%d/%d] %s_%s\n", i, n, pkg, ver)) + bincraft::build_binary_package( + pkg, + tag = ver, + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + s3_package_cache = s3_cache, + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + upload = TRUE, + store_build_metadata = TRUE + ) +}, chunk$Package, chunk$Version, seq_len(n))