diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 926cdbb..8e3e830 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -140,6 +140,7 @@ steps: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - . local/compiler-cache.sh - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # The primary pass must not rely on build-all-versions-install-deps having # run on *this* agent: depends_on only orders the steps, but the cache diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 4bab415..e4ef547 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -179,6 +179,8 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache + # parallel make within one package build, matched to the step's CPU limit + MAKE_JOBS: 3 PLATFORM: ${OS} OS_ID: ${OS_ID} ARCH: ${ARCH} @@ -186,6 +188,7 @@ steps: INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - . local/compiler-cache.sh - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 89badb9..e54d107 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -373,6 +373,8 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache + # parallel make within one package build, matched to the step's CPU limit + MAKE_JOBS: 3 PLATFORM: ${OS} ARCH: ${ARCH} NCPUS: 2 @@ -384,6 +386,7 @@ steps: REBUILD_BUDGET_HOURS: 20 commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - . local/compiler-cache.sh - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R diff --git a/local/build-newer-r.R b/local/build-newer-r.R index 6478200..498ca81 100644 --- a/local/build-newer-r.R +++ b/local/build-newer-r.R @@ -27,6 +27,94 @@ message(sprintf( if ("--dry-run" %in% args) { print(pkgs, row.names = FALSE) } else { + # Most candidates were built on an earlier run. Asking bincraft about each one + # costs a DB connection plus several S3 requests, about 4s a package, so the + # loop spent well over an hour per run re-confirming existing binaries. + # Decide those in bulk first; bincraft still checks whatever is left. + local_machine <- Sys.info()[["machine"]] + arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" + slot_dir <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", + arch, + bincraft::set_codename(NULL) + ) + s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE + ) + # NULL when the listing fails, so that route falls back to bincraft. + list_etags <- function(dir) { + info <- tryCatch(s3fs::s3_dir_info(dir), error = function(e) { + message(sprintf("Could not list %s: %s", dir, conditionMessage(e))) + NULL + }) + if (is.null(info)) { + return(NULL) + } + stats::setNames( + gsub('^"|"$', "", as.character(info$etag)), + basename(as.character(info$uri)) + ) + } + # The verdicts bincraft::classify_r_minor_sensitive() would read from its + # cache, fetched in one query rather than one connection per package. + read_verdicts <- function(packages) { + tryCatch( + { + con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "verify-full" + ) + on.exit(DBI::dbDisconnect(con), add = TRUE) + rows <- DBI::dbGetQuery( + con, + paste( + "SELECT package, version, r_minor_sensitive FROM abi_classification", + "WHERE classifier_sig = $1 AND package = ANY($2::text[])" + ), + params = list( + bincraft:::abi_classifier_signature(), + paste0("{", paste(packages, collapse = ","), "}") + ) + ) + stats::setNames( + as.logical(rows$r_minor_sensitive), + paste(rows$package, rows$version, sep = "_") + ) + }, + error = function(e) { + message(sprintf("Could not read ABI verdicts: %s", conditionMessage(e))) + logical() + } + ) + } + sensitive <- read_verdicts(pkgs$Package) + before <- nrow(pkgs) + pkgs <- newer_r_outstanding( + pkgs, + sensitive, + list_etags(slot_dir), + list_etags(file.path( + slot_dir, + sub("^(\\d+\\.\\d+).*", "\\1", target_r) + )), + bincraft:::cran_source_md5() + ) + message(sprintf( + "%d of %d already carry a binary at their routed path; %d left for bincraft", + before - nrow(pkgs), + before, + nrow(pkgs) + )) + options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL) for (i in seq_len(nrow(pkgs))) { message(sprintf( diff --git a/local/compiler-cache.sh b/local/compiler-cache.sh new file mode 100644 index 0000000..8e179b2 --- /dev/null +++ b/local/compiler-cache.sh @@ -0,0 +1,32 @@ +# Sourced (`. local/compiler-cache.sh`) by the build pipelines before any +# package is compiled. POSIX sh: the Alpine images run steps under busybox. +# +# Routes gcc/g++ through ccache. The images name their compilers by bare +# `gcc`/`g++` in both R's Makeconf and /root/.R/Makevars, so prepending ccache's +# masquerade dir catches every compile, CMake-driven vendored builds included. +# Wrapping CC in a Makevars instead would need R_MAKEVARS_USER, which replaces +# the images' /root/.R/Makevars (CXX_STD, the musl defines) rather than adding +# to it. The dir differs per distro: Alpine, RHEL, Debian/Ubuntu. +for dir in /usr/lib/ccache/bin /usr/lib64/ccache /usr/lib/ccache; do + if [ -x "$dir/gcc" ]; then + PATH="$dir:$PATH" + break + fi +done +export PATH + +# R compiles every package in a fresh /tmp/RtmpXXXXXX/ dir. R's default +# flags include -g, which makes ccache hash the working directory, and the line +# markers carry the absolute source path, so without these two settings no +# compile ever hits the cache across runs. +export CCACHE_BASEDIR="${CCACHE_BASEDIR:-/tmp}" +export CCACHE_NOHASHDIR=1 +# The images are rebuilt regularly, which changes the compiler's mtime but not +# its content. ccache's default mtime check would discard the cache each time. +export CCACHE_COMPILERCHECK=content + +# Parallel make inside a single package build. Match it to the step's CPU limit +# via MAKE_JOBS; an explicit MAKEFLAGS wins. +export MAKEFLAGS="${MAKEFLAGS:--j${MAKE_JOBS:-2}}" + +echo "compiler-cache: $(command -v gcc), CCACHE_DIR=${CCACHE_DIR:-unset}, MAKEFLAGS=$MAKEFLAGS" diff --git a/local/newer-r-helpers.R b/local/newer-r-helpers.R index 4b83204..b4ac689 100644 --- a/local/newer-r-helpers.R +++ b/local/newer-r-helpers.R @@ -73,3 +73,42 @@ newer_r_candidates <- function(db, primary_r, target_r) { result <- db[selected, c("Package", "Version"), drop = FALSE] result[order(result$Package), , drop = FALSE] } + +# Candidates that still need a build, decided from one bucket listing per slot +# instead of a metadata-DB connection and several S3 requests per package. +# +# Mirrors bincraft's own skip decision: a package is done when a binary sits at +# the path its cached ABI verdict routes it to, `//` when +# r-minor-sensitive and the flat `/` otherwise. A CRAN source sitting at +# that key is a fallback, not a binary (see outstanding_packages()). Whatever +# this cannot decide, no cached verdict or a slot that could not be listed, +# stays in the list, so bincraft makes that call exactly as before. +# +# `sensitive` cached verdicts for the current classifier, named `_` +# `flat_etags` listing of the flat slot as for outstanding_packages(), or +# NULL when it could not be listed +# `minor_etags` the same for this interpreter's per-minor slot +newer_r_outstanding <- function( + pkgs, + sensitive, + flat_etags, + minor_etags, + cran_md5 +) { + verdict <- unname(sensitive[paste(pkgs$Package, pkgs$Version, sep = "_")]) + version <- stats::setNames(pkgs$Version, pkgs$Package) + keep <- rep(TRUE, nrow(pkgs)) + routes <- list( + list(sensitive = FALSE, etags = flat_etags), + list(sensitive = TRUE, etags = minor_etags) + ) + for (route in routes) { + idx <- which(verdict %in% route$sensitive) + if (length(idx) == 0L || is.null(route$etags)) { + next + } + keep[idx] <- pkgs$Package[idx] %in% + outstanding_packages(pkgs$Package[idx], route$etags, version, cran_md5) + } + pkgs[keep, , drop = FALSE] +} diff --git a/local/tests/test-newer-r-helpers.R b/local/tests/test-newer-r-helpers.R index c202b7f..ad4ca49 100644 --- a/local/tests/test-newer-r-helpers.R +++ b/local/tests/test-newer-r-helpers.R @@ -1,4 +1,5 @@ source(file.path("..", "newer-r-helpers.R")) +source(file.path("..", "rebuild-missing-helpers.R")) test_that("R constraints respect patch versions, bounds, and missing fields", { expect_true(r_requirement_satisfied(NA_character_, "4.4.3")) @@ -79,3 +80,70 @@ test_that("R-devel Recommended copies cannot block ordinary CRAN dependencies", c("arules", "arulesSequences", "indirect") ) }) + +candidates <- function() { + data.frame( + Package = c( + "flat_done", + "flat_todo", + "minor_done", + "minor_flat_only", + "unclassified", + "fallback" + ), + Version = c("1.0", "1.0", "2.0", "2.0", "3.0", "4.0") + ) +} + +verdicts <- c( + flat_done_1.0 = FALSE, + flat_todo_1.0 = FALSE, + minor_done_2.0 = TRUE, + minor_flat_only_2.0 = TRUE, + fallback_4.0 = FALSE +) + +flat <- c( + flat_done_1.0.tar.gz = "aaa", + minor_flat_only_2.0.tar.gz = "bbb", + unclassified_3.0.tar.gz = "ccc", + fallback_4.0.tar.gz = "c0ffee" +) +minor <- c(minor_done_2.0.tar.gz = "ddd") +md5 <- c(fallback_4.0 = "c0ffee") + +test_that("newer_r_outstanding skips only binaries at the routed path", { + expect_identical( + newer_r_outstanding(candidates(), verdicts, flat, minor, md5)$Package, + # flat_todo: absent; minor_flat_only: sensitive, so a flat copy does not + # count; unclassified: no verdict, left to bincraft; fallback: CRAN source + c("flat_todo", "minor_flat_only", "unclassified", "fallback") + ) +}) + +test_that("newer_r_outstanding keeps a route whose slot could not be listed", { + expect_identical( + newer_r_outstanding(candidates(), verdicts, NULL, minor, md5)$Package, + c("flat_done", "flat_todo", "minor_flat_only", "unclassified", "fallback") + ) + expect_identical( + newer_r_outstanding(candidates(), verdicts, flat, NULL, md5)$Package, + c("flat_todo", "minor_done", "minor_flat_only", "unclassified", "fallback") + ) +}) + +test_that("newer_r_outstanding without verdicts or listings changes nothing", { + expect_identical( + newer_r_outstanding(candidates(), logical(), flat, minor, md5), + candidates() + ) + empty <- stats::setNames(character(), character()) + expect_identical( + nrow(newer_r_outstanding(candidates(), verdicts, empty, empty, md5)), + nrow(candidates()) + ) + expect_identical( + nrow(newer_r_outstanding(candidates()[0L, ], verdicts, flat, minor, md5)), + 0L + ) +})