From 2861d50446fa5317d1df63bf3b9c8e10b06a23af Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 13 Sep 2026 09:34:09 +0000 Subject: [PATCH] fix(build): build packages requiring newer R on compatible minors --- .crow/build-all-versions.yaml | 2 + .crow/process-updates.yaml | 2 + README.md | 17 +++++++ justfile | 7 +++ local/build-newer-r.R | 72 ++++++++++++++++++++++++++ local/newer-r-helpers.R | 75 +++++++++++++++++++++++++++ local/tests/test-newer-r-helpers.R | 81 ++++++++++++++++++++++++++++++ 7 files changed, 256 insertions(+) create mode 100644 local/build-newer-r.R create mode 100644 local/newer-r-helpers.R create mode 100644 local/tests/test-newer-r-helpers.R diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 91b7bcd..7451988 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -167,6 +167,8 @@ steps: R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true + # Also build current packages whose required dependency tree needs a newer R. + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION" $SPLIT_INTO $SPLIT_INDEX done # archive missed packages; first arg is the codename (e.g. "alpine324"), # derived via bincraft like the upload step, not paste(OS, OS_VERSION). diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 4cd95a7..a1b9698 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -207,6 +207,8 @@ steps: R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + # Also build current packages whose required dependency tree needs a newer R. + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION" done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/README.md b/README.md index d137728..99d1143 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,23 @@ This elastic server architecture offers a robust and performant backend while mi ## Technical Details +### Packages requiring a newer R + +The daily update and weekly full-build pipelines run `local/build-newer-r.R` under each non-primary installed R, alongside the ABI-sensitive pass. +It selects current CRAN versions whose `Depends: R` constraints cannot be met by the slot's primary R but can be met by the running R, including incompatibility inherited through `Depends`, `Imports`, and `LinkingTo`. +These builds retain bincraft's ABI classification and storage routing: sensitive binaries use the running minor, while safe binaries can be shared. +No shared S3 existence cache is passed, so bincraft checks the actual objects and distinguishes a source fallback from a binary. +The pass reads unfiltered CRAN metadata independently of the weekly snapshot and its primary-R failure exclusions; existing binaries and failures for the running minor retain bincraft's usual skip behavior. +Weekly workers split this extra work across their existing shards. +This does not select older archive versions for unsupported R minors or change index filtering. + +Preview the selection without building or accessing storage or build metadata: + +```sh +# Run with the target R and supply the slot's full primary R version. +/opt/R/4.6.0/bin/Rscript local/build-newer-r.R 4.4.3 --dry-run +``` + ### Creating/Updating the PACKAGES Index Files Currently, the {cranlike} and {desc} packages only work with files on a local file system. diff --git a/justfile b/justfile index 36ad0e1..524ac9a 100644 --- a/justfile +++ b/justfile @@ -95,3 +95,10 @@ check-slots: # strip a retired slot from .crow/*.yaml (dry run; pass --apply to write) retire-slot codename *flags: Rscript local/retire-slot.R {{ codename }} --config-only {{ flags }} + +# Format or lint selected R files, defaulting to local tooling. +format *files="local": + air format {{ files }} + +fix *files="local": + jarl check --fix {{ files }} diff --git a/local/build-newer-r.R b/local/build-newer-r.R new file mode 100644 index 0000000..3d7794f --- /dev/null +++ b/local/build-newer-r.R @@ -0,0 +1,72 @@ +source(file.path("local", "newer-r-helpers.R")) +source(file.path("local", "rebuild-missing-helpers.R")) + +args <- commandArgs(trailingOnly = TRUE) +pos <- args[!startsWith(args, "--")] +if (!length(pos) %in% c(1L, 3L)) { + stop("Usage: build-newer-r.R PRIMARY_R [SPLIT_INTO SPLIT_INDEX] [--dry-run]") +} +primary_r <- pos[1L] +target_r <- as.character(getRversion()) +pkgs <- newer_r_candidates(tools::CRAN_package_db(), primary_r, target_r) +exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] +pkgs <- pkgs[!pkgs$Package %in% exclude, , drop = FALSE] +if (length(pos) == 3L) { + pkgs <- pkgs[ + shard_slice(seq_len(nrow(pkgs)), pos[2L], pos[3L]), + , + drop = FALSE + ] +} +message(sprintf( + "R %s: %d package(s) unsupported by primary R %s", + target_r, + nrow(pkgs), + primary_r +)) +if ("--dry-run" %in% args) { + print(pkgs, row.names = FALSE) +} else { + options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL) + for (i in seq_len(nrow(pkgs))) { + message(sprintf( + "[%d/%d] %s %s", + i, + nrow(pkgs), + pkgs$Package[i], + pkgs$Version[i] + )) + bincraft::build_binary_package( + pkgs$Package[i], + tag = pkgs$Version[i], + # Keep bincraft's normal ABI routing: safe binaries can be shared; + # sensitive ones use this interpreter's minor for presence and output. + is_r_minor_sensitive = bincraft::classify_r_minor_sensitive( + pkgs$Package[i], + pkgs$Version[i], + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432 + ), + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + patches = "local/patches", + upload = TRUE, + store_build_metadata = TRUE + ) + } +} diff --git a/local/newer-r-helpers.R b/local/newer-r-helpers.R new file mode 100644 index 0000000..4b83204 --- /dev/null +++ b/local/newer-r-helpers.R @@ -0,0 +1,75 @@ +# Evaluate every R constraint, including patch-level and upper bounds. +r_requirement_satisfied <- function(depends, r_version) { + if (is.na(depends) || !nzchar(trimws(depends))) { + return(TRUE) + } + entries <- trimws(strsplit(depends, ",", fixed = TRUE)[[1L]]) + entries <- entries[grepl("^R(\\s|\\(|$)", entries)] + all(vapply( + entries, + function(entry) { + if (entry == "R") { + return(TRUE) + } + match <- regmatches( + entry, + regexec( + "^R\\s*\\(\\s*(>=|<=|==|>|<)\\s*([0-9]+(?:[.-][0-9]+)*)\\s*\\)$", + entry, + perl = TRUE + ) + )[[1L]] + if (length(match) != 3L) { + stop("Cannot parse R dependency: ", entry) + } + comparison <- utils::compareVersion(r_version, match[3L]) + switch( + match[2L], + ">=" = comparison >= 0L, + "<=" = comparison <= 0L, + "==" = comparison == 0L, + ">" = comparison > 0L, + "<" = comparison < 0L + ) + }, + logical(1L) + )) +} + +# Use unfiltered current CRAN metadata: available.packages() would hide exactly +# the packages that cannot run under the primary interpreter. +# Propagate R incompatibility through required dependencies, not Suggests. +newer_r_candidates <- function(db, primary_r, target_r) { + db <- as.data.frame(db, stringsAsFactors = FALSE) + # CRAN also lists R-specific Recommended copies (e.g. Matrix for R-devel). + # Prefer the ordinary contrib record, or a future-R copy can incorrectly + # block every package depending on Matrix on all released interpreters. + if ("Path" %in% names(db)) { + db <- db[order(!is.na(db$Path) & nzchar(db$Path)), , drop = FALSE] + } + db <- db[!duplicated(db$Package), , drop = FALSE] + deps <- tools::package_dependencies( + packages = db$Package, + db = as.matrix(db), + which = c("Depends", "Imports", "LinkingTo"), + recursive = FALSE + ) + supported <- function(version) { + ok <- vapply(db$Depends, r_requirement_satisfied, logical(1L), version) + repeat { + blocked <- db$Package[!ok] + next_ok <- ok & + !vapply(deps, function(x) any(x %in% blocked), logical(1L)) + if (identical(ok, next_ok)) { + return(ok) + } + ok <- next_ok + } + } + selected <- !supported(primary_r) & supported(target_r) + if ("OS_type" %in% names(db)) { + selected <- selected & (is.na(db$OS_type) | db$OS_type != "windows") + } + result <- db[selected, c("Package", "Version"), drop = FALSE] + result[order(result$Package), , drop = FALSE] +} diff --git a/local/tests/test-newer-r-helpers.R b/local/tests/test-newer-r-helpers.R new file mode 100644 index 0000000..c202b7f --- /dev/null +++ b/local/tests/test-newer-r-helpers.R @@ -0,0 +1,81 @@ +source(file.path("..", "newer-r-helpers.R")) + +test_that("R constraints respect patch versions, bounds, and missing fields", { + expect_true(r_requirement_satisfied(NA_character_, "4.4.3")) + expect_true(r_requirement_satisfied( + "Matrix (>= 1.4-0), R (>= 4.5.0)", + "4.5.3" + )) + expect_false(r_requirement_satisfied("R (>= 4.5.0)", "4.4.3")) + expect_false(r_requirement_satisfied("R (>= 4.5.4)", "4.5.3")) + expect_true(r_requirement_satisfied("R (>= 4.5), R (< 4.6)", "4.5.3")) + expect_false(r_requirement_satisfied("R (>= 4.5), R (< 4.6)", "4.6.0")) + expect_true(r_requirement_satisfied("R (== 4.5.3)", "4.5.3")) + expect_true(r_requirement_satisfied("R (> 4.5.2), R (<= 4.5.3)", "4.5.3")) + expect_true(r_requirement_satisfied("R, Rcpp (>= 1.0)", "4.4.3")) + expect_error(r_requirement_satisfied("R (~ 4.5)", "4.5.3"), "Cannot parse") +}) + +fixture <- function() { + data.frame( + Package = c( + "arules", + "arulesSequences", + "indirect", + "optional", + "ordinary", + "future", + "blocked" + ), + Version = rep("1.0", 7L), + Depends = c( + "R (>= 4.5.0)", + "arules", + NA, + NA, + "R (>= 4.0)", + "R (>= 4.7)", + NA + ), + Imports = c(NA, NA, "arulesSequences", NA, NA, NA, "arules, future"), + LinkingTo = NA_character_, + Suggests = c(NA, NA, NA, "arules", NA, NA, NA) + ) +} + +test_that("every compatible newer minor includes required dependents only", { + db <- fixture() + for (target in c("4.5.3", "4.6.0")) { + expect_identical( + newer_r_candidates(db, "4.4.3", target)$Package, + c("arules", "arulesSequences", "indirect") + ) + } + expect_equal(nrow(newer_r_candidates(db, "4.4.3", "4.4.3")), 0L) + expect_equal(nrow(newer_r_candidates(db, "4.5.3", "4.4.3")), 0L) + expect_equal(nrow(newer_r_candidates(db, "4.5.3", "4.6.0")), 0L) +}) + +test_that("LinkingTo, dependency cycles, Windows and empty metadata are handled", { + db <- fixture() + db$LinkingTo[5L] <- "arules" + db$Imports[1L] <- "ordinary" + db$OS_type <- c(NA, "windows", rep(NA_character_, 5L)) + expect_identical( + newer_r_candidates(db, "4.4.3", "4.6.0")$Package, + c("arules", "indirect", "ordinary") + ) + expect_equal(nrow(newer_r_candidates(db[FALSE, ], "4.4.3", "4.6.0")), 0L) +}) + +test_that("R-devel Recommended copies cannot block ordinary CRAN dependencies", { + db <- fixture() + db$Path <- NA_character_ + future_copy <- db[1L, ] + future_copy$Depends <- "R (>= 4.7)" + future_copy$Path <- "4.7.0/Recommended" + expect_identical( + newer_r_candidates(rbind(future_copy, db), "4.4.3", "4.6.1")$Package, + c("arules", "arulesSequences", "indirect") + ) +}) -- 2.54.0