refactor: extract build-all R one-liner into local/build-all.R

- Load bincraft eagerly via library() to avoid lazy-load memory spike
- Do bulk S3 listing upfront and pass as s3_package_cache to avoid
  per-package S3 calls that accumulate memory and trigger fork failures
- Consolidate amd64/arm64 exclude lists into single script
- Much easier to read and maintain than a YAML-embedded one-liner

.crow/build-all-versions-amd64.yaml

@@ -69,7 +69,7 @@ steps:
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
       # pkgs = readRDS('/mnt/cache/pkgs_amd64.rds');
       - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
-      - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = readRDS('/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Total# of remaining package versions:%s', nrow(pkgs)); chunks = split(pkgs, cut(seq_len(nrow(pkgs)), $SPLIT_INTO, labels = FALSE)); chunk = chunks[[$SPLIT_INDEX]]; sprintf('# of package versions for this job:%s', nrow(chunk)); exclude = c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'Rfast', 'compareGroups', 'ff', 'GsymPoint', 'RcppDynProg', 'comtradr', 'RcppDynProg', 'FD', 'PearsonDS', 'DCluster', 'gRc', 'mixlm', 'geospt', 'fdth', 'ffmanova', 'fiery', 'ffscrapr', 'cold', 'fiery', 'RcmdrPlugin.DoE', 'RcmdrPlugin.NMBU', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.UCA', 'RcmdrPlugin.WorldFlora', 'RcmdrPlugin.aRnova', 'RcmdrPlugin.depthTools', 'RcmdrPlugin.orloca', 'RcmdrPlugin.sos', 'RcmdrPlugin.survival', 'RcmdrPlugin.temis', 'GWlasso', 'GWmodelVis'); chunk = chunk[!chunk\$Package %in% exclude, ]; library(future); plan('sequential'); n = nrow(chunk); mapply(function(pkg, ver, i) { cat(sprintf('[%d/%d] %s_%s\n', i, n, pkg, ver)); bincraft::build_binary_package(pkg, tag = ver, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE) }, chunk\$Package, chunk\$Version, seq_len(n))" 2>&1
+      - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1
       # archive missed packages
       - R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)"
     backend_options:

.crow/build-all-versions-arm64.yaml

@@ -82,7 +82,7 @@ steps:
       - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
-      - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = readRDS('/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Total# of remaining package versions:%s', nrow(pkgs)); chunks = split(pkgs, cut(seq_len(nrow(pkgs)), $SPLIT_INTO, labels = FALSE)); chunk = chunks[[$SPLIT_INDEX]]; sprintf('# of package versions for this job:%s', nrow(chunk)); exclude = c('RInno', 'RcmdrPlugin.DoE', 'RcmdrPlugin.NMBU', 'RcmdrPlugin.ROC', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.UCA', 'RcmdrPlugin.WorldFlora', 'RcmdrPlugin.aRnova', 'RcmdrPlugin.depthTools', 'RcmdrPlugin.orloca', 'RcmdrPlugin.sos', 'RcmdrPlugin.survival', 'RcmdrPlugin.temis'); chunk = chunk[!chunk\$Package %in% exclude, ]; library(future); plan('sequential'); n = nrow(chunk); mapply(function(pkg, ver, i) { cat(sprintf('[%d/%d] %s_%s\n', i, n, pkg, ver)); bincraft::build_binary_package(pkg, tag = ver, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE) }, chunk\$Package, chunk\$Version, seq_len(n))" 2>&1
+      - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1
       # archive missed packages
       - R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)"
     backend_options:

local/build-all.R

@@ -0,0 +1,82 @@
+sink(stdout(), type = "message")
+options(crayon.enabled = TRUE, future.globals.onReference = NULL)
+
+args <- commandArgs(trailingOnly = TRUE)
+split_into <- as.integer(args[1])
+split_index <- as.integer(args[2])
+ncpus <- as.integer(args[3])
+options(Ncpus = ncpus)
+
+# Load bincraft eagerly to avoid lazy-load memory spike during first build call
+library(bincraft, quietly = TRUE)
+library(future)
+plan("sequential")
+
+# Read precomputed package+version pairs
+pkgs <- readRDS("/mnt/cache/packages/pkgs_to_build.rds")
+sprintf("Total# of remaining package versions: %s", nrow(pkgs))
+
+# Split into chunks for this worker
+chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
+chunk <- chunks[[split_index]]
+sprintf("# of package versions for this job: %s", nrow(chunk))
+
+# Exclude known problematic packages
+exclude <- c(
+  "RInno", "KeyboardSimulator", "R2PPT", "RWinEdt", "blatr", "excel.link",
+  "spectrino", "taskscheduleR", "MDSGUI", "BiplotGUI", "R2wd", "rFUSION",
+  "MediaNews", "doBy", "IDPmisc", "frailtypack", "afex", "FrF2", "DoE.base",
+  "agricolae", "doFuture", "fscaret", "PHYLOGR", "seewave", "pls", "relaimpo",
+  "geepack", "gggenes", "NPCirc", "repmis", "PNDSIBGE", "lidR", "poismf",
+  "neonstore", "MachineShop", "mvst", "MacBehaviour", "mcmcderive", "RGIFT",
+  "KnowBR", "netmeta", "spdep", "Rfast", "compareGroups", "ff", "GsymPoint",
+  "RcppDynProg", "comtradr", "FD", "PearsonDS", "DCluster", "gRc", "mixlm",
+  "geospt", "fdth", "ffmanova", "fiery", "ffscrapr", "cold",
+  "RcmdrPlugin.DoE", "RcmdrPlugin.NMBU", "RcmdrPlugin.RiskDemo",
+  "RcmdrPlugin.ROC", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos",
+  "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.aRnova",
+  "RcmdrPlugin.depthTools", "RcmdrPlugin.orloca", "RcmdrPlugin.sos",
+  "RcmdrPlugin.survival", "RcmdrPlugin.temis", "GWlasso", "GWmodelVis"
+)
+chunk <- chunk[!chunk$Package %in% exclude, ]
+
+# Bulk S3 listing upfront — avoids per-package S3 calls
+s3fs::s3_file_system(
+  aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
+  aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
+  endpoint = "https://s3.eu-central-003.backblazeb2.com",
+  region_name = "eu-central-003",
+  refresh = TRUE
+)
+codename <- bincraft::set_codename(NULL)
+s3_cache <- basename(s3fs::s3_dir_ls(
+  sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib",
+    Sys.info()[["machine"]], codename),
+  recurse = TRUE
+))
+sprintf("S3 cache: %s files", length(s3_cache))
+
+n <- nrow(chunk)
+mapply(function(pkg, ver, i) {
+  cat(sprintf("[%d/%d] %s_%s\n", i, n, pkg, ver))
+  bincraft::build_binary_package(
+    pkg,
+    tag = ver,
+    s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    s3_region = "eu-central-003",
+    s3_bucket = "devxy-rpkgs-binaries",
+    s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
+    s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
+    s3_package_cache = s3_cache,
+    metadata_db_host = "r-binaries.devxy.io",
+    metadata_db_name = "build_metadata",
+    metadata_db_table = "single_builds",
+    metadata_db_user = "rpkgs",
+    metadata_db_password = Sys.getenv("PGPASS"),
+    metadata_db_sslmode = "require",
+    metadata_db_port = 15432,
+    archive = TRUE,
+    upload = TRUE,
+    store_build_metadata = TRUE
+  )
+}, chunk$Package, chunk$Version, seq_len(n))