.crow/build-all-versions.yaml

@@ -167,6 +167,8 @@ steps:
           R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
           R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
           R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true
+          # Also build current packages whose required dependency tree needs a newer R.
+          R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION" $SPLIT_INTO $SPLIT_INDEX
         done
       # archive missed packages; first arg is the codename (e.g. "alpine324"),
       # derived via bincraft like the upload step, not paste(OS, OS_VERSION).

.crow/process-updates.yaml

@@ -207,6 +207,8 @@ steps:
           R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
           R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
           R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true
+          # Also build current packages whose required dependency tree needs a newer R.
+          R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION"
         done
       - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
       - |

README.md

@@ -84,6 +84,23 @@ This elastic server architecture offers a robust and performant backend while mi
 
 ## Technical Details
 
+### Packages requiring a newer R
+
+The daily update and weekly full-build pipelines run `local/build-newer-r.R` under each non-primary installed R, alongside the ABI-sensitive pass.
+It selects current CRAN versions whose `Depends: R` constraints cannot be met by the slot's primary R but can be met by the running R, including incompatibility inherited through `Depends`, `Imports`, and `LinkingTo`.
+These builds retain bincraft's ABI classification and storage routing: sensitive binaries use the running minor, while safe binaries can be shared.
+No shared S3 existence cache is passed, so bincraft checks the actual objects and distinguishes a source fallback from a binary.
+The pass reads unfiltered CRAN metadata independently of the weekly snapshot and its primary-R failure exclusions; existing binaries and failures for the running minor retain bincraft's usual skip behavior.
+Weekly workers split this extra work across their existing shards.
+This does not select older archive versions for unsupported R minors or change index filtering.
+
+Preview the selection without building or accessing storage or build metadata:
+
+```sh
+# Run with the target R and supply the slot's full primary R version.
+/opt/R/4.6.0/bin/Rscript local/build-newer-r.R 4.4.3 --dry-run
+```
+
 ### Creating/Updating the PACKAGES Index Files
 
 Currently, the {cranlike} and {desc} packages only work with files on a local file system.

justfile

@@ -95,3 +95,10 @@ check-slots:
 # strip a retired slot from .crow/*.yaml (dry run; pass --apply to write)
 retire-slot codename *flags:
     Rscript local/retire-slot.R {{ codename }} --config-only {{ flags }}
+
+# Format or lint selected R files, defaulting to local tooling.
+format *files="local":
+    air format {{ files }}
+
+fix *files="local":
+    jarl check --fix {{ files }}

local/build-newer-r.R

@@ -0,0 +1,72 @@
+source(file.path("local", "newer-r-helpers.R"))
+source(file.path("local", "rebuild-missing-helpers.R"))
+
+args <- commandArgs(trailingOnly = TRUE)
+pos <- args[!startsWith(args, "--")]
+if (!length(pos) %in% c(1L, 3L)) {
+  stop("Usage: build-newer-r.R PRIMARY_R [SPLIT_INTO SPLIT_INDEX] [--dry-run]")
+}
+primary_r <- pos[1L]
+target_r <- as.character(getRversion())
+pkgs <- newer_r_candidates(tools::CRAN_package_db(), primary_r, target_r)
+exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]]
+pkgs <- pkgs[!pkgs$Package %in% exclude, , drop = FALSE]
+if (length(pos) == 3L) {
+  pkgs <- pkgs[
+    shard_slice(seq_len(nrow(pkgs)), pos[2L], pos[3L]),
+    ,
+    drop = FALSE
+  ]
+}
+message(sprintf(
+  "R %s: %d package(s) unsupported by primary R %s",
+  target_r,
+  nrow(pkgs),
+  primary_r
+))
+if ("--dry-run" %in% args) {
+  print(pkgs, row.names = FALSE)
+} else {
+  options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL)
+  for (i in seq_len(nrow(pkgs))) {
+    message(sprintf(
+      "[%d/%d] %s %s",
+      i,
+      nrow(pkgs),
+      pkgs$Package[i],
+      pkgs$Version[i]
+    ))
+    bincraft::build_binary_package(
+      pkgs$Package[i],
+      tag = pkgs$Version[i],
+      # Keep bincraft's normal ABI routing: safe binaries can be shared;
+      # sensitive ones use this interpreter's minor for presence and output.
+      is_r_minor_sensitive = bincraft::classify_r_minor_sensitive(
+        pkgs$Package[i],
+        pkgs$Version[i],
+        metadata_db_host = "r-binaries.devxy.io",
+        metadata_db_name = "build_metadata",
+        metadata_db_user = "rpkgs",
+        metadata_db_password = Sys.getenv("PGPASS"),
+        metadata_db_sslmode = "require",
+        metadata_db_port = 15432
+      ),
+      s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
+      s3_region = "eu-central-003",
+      s3_bucket = "devxy-rpkgs-binaries",
+      s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
+      s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
+      metadata_db_host = "r-binaries.devxy.io",
+      metadata_db_name = "build_metadata",
+      metadata_db_table = "single_builds",
+      metadata_db_user = "rpkgs",
+      metadata_db_password = Sys.getenv("PGPASS"),
+      metadata_db_sslmode = "require",
+      metadata_db_port = 15432,
+      archive = TRUE,
+      patches = "local/patches",
+      upload = TRUE,
+      store_build_metadata = TRUE
+    )
+  }
+}

local/newer-r-helpers.R

@@ -0,0 +1,75 @@
+# Evaluate every R constraint, including patch-level and upper bounds.
+r_requirement_satisfied <- function(depends, r_version) {
+  if (is.na(depends) || !nzchar(trimws(depends))) {
+    return(TRUE)
+  }
+  entries <- trimws(strsplit(depends, ",", fixed = TRUE)[[1L]])
+  entries <- entries[grepl("^R(\\s|\\(|$)", entries)]
+  all(vapply(
+    entries,
+    function(entry) {
+      if (entry == "R") {
+        return(TRUE)
+      }
+      match <- regmatches(
+        entry,
+        regexec(
+          "^R\\s*\\(\\s*(>=|<=|==|>|<)\\s*([0-9]+(?:[.-][0-9]+)*)\\s*\\)$",
+          entry,
+          perl = TRUE
+        )
+      )[[1L]]
+      if (length(match) != 3L) {
+        stop("Cannot parse R dependency: ", entry)
+      }
+      comparison <- utils::compareVersion(r_version, match[3L])
+      switch(
+        match[2L],
+        ">=" = comparison >= 0L,
+        "<=" = comparison <= 0L,
+        "==" = comparison == 0L,
+        ">" = comparison > 0L,
+        "<" = comparison < 0L
+      )
+    },
+    logical(1L)
+  ))
+}
+
+# Use unfiltered current CRAN metadata: available.packages() would hide exactly
+# the packages that cannot run under the primary interpreter.
+# Propagate R incompatibility through required dependencies, not Suggests.
+newer_r_candidates <- function(db, primary_r, target_r) {
+  db <- as.data.frame(db, stringsAsFactors = FALSE)
+  # CRAN also lists R-specific Recommended copies (e.g. Matrix for R-devel).
+  # Prefer the ordinary contrib record, or a future-R copy can incorrectly
+  # block every package depending on Matrix on all released interpreters.
+  if ("Path" %in% names(db)) {
+    db <- db[order(!is.na(db$Path) & nzchar(db$Path)), , drop = FALSE]
+  }
+  db <- db[!duplicated(db$Package), , drop = FALSE]
+  deps <- tools::package_dependencies(
+    packages = db$Package,
+    db = as.matrix(db),
+    which = c("Depends", "Imports", "LinkingTo"),
+    recursive = FALSE
+  )
+  supported <- function(version) {
+    ok <- vapply(db$Depends, r_requirement_satisfied, logical(1L), version)
+    repeat {
+      blocked <- db$Package[!ok]
+      next_ok <- ok &
+        !vapply(deps, function(x) any(x %in% blocked), logical(1L))
+      if (identical(ok, next_ok)) {
+        return(ok)
+      }
+      ok <- next_ok
+    }
+  }
+  selected <- !supported(primary_r) & supported(target_r)
+  if ("OS_type" %in% names(db)) {
+    selected <- selected & (is.na(db$OS_type) | db$OS_type != "windows")
+  }
+  result <- db[selected, c("Package", "Version"), drop = FALSE]
+  result[order(result$Package), , drop = FALSE]
+}

local/tests/test-newer-r-helpers.R

@@ -0,0 +1,81 @@
+source(file.path("..", "newer-r-helpers.R"))
+
+test_that("R constraints respect patch versions, bounds, and missing fields", {
+  expect_true(r_requirement_satisfied(NA_character_, "4.4.3"))
+  expect_true(r_requirement_satisfied(
+    "Matrix (>= 1.4-0), R (>= 4.5.0)",
+    "4.5.3"
+  ))
+  expect_false(r_requirement_satisfied("R (>= 4.5.0)", "4.4.3"))
+  expect_false(r_requirement_satisfied("R (>= 4.5.4)", "4.5.3"))
+  expect_true(r_requirement_satisfied("R (>= 4.5), R (< 4.6)", "4.5.3"))
+  expect_false(r_requirement_satisfied("R (>= 4.5), R (< 4.6)", "4.6.0"))
+  expect_true(r_requirement_satisfied("R (== 4.5.3)", "4.5.3"))
+  expect_true(r_requirement_satisfied("R (> 4.5.2), R (<= 4.5.3)", "4.5.3"))
+  expect_true(r_requirement_satisfied("R, Rcpp (>= 1.0)", "4.4.3"))
+  expect_error(r_requirement_satisfied("R (~ 4.5)", "4.5.3"), "Cannot parse")
+})
+
+fixture <- function() {
+  data.frame(
+    Package = c(
+      "arules",
+      "arulesSequences",
+      "indirect",
+      "optional",
+      "ordinary",
+      "future",
+      "blocked"
+    ),
+    Version = rep("1.0", 7L),
+    Depends = c(
+      "R (>= 4.5.0)",
+      "arules",
+      NA,
+      NA,
+      "R (>= 4.0)",
+      "R (>= 4.7)",
+      NA
+    ),
+    Imports = c(NA, NA, "arulesSequences", NA, NA, NA, "arules, future"),
+    LinkingTo = NA_character_,
+    Suggests = c(NA, NA, NA, "arules", NA, NA, NA)
+  )
+}
+
+test_that("every compatible newer minor includes required dependents only", {
+  db <- fixture()
+  for (target in c("4.5.3", "4.6.0")) {
+    expect_identical(
+      newer_r_candidates(db, "4.4.3", target)$Package,
+      c("arules", "arulesSequences", "indirect")
+    )
+  }
+  expect_equal(nrow(newer_r_candidates(db, "4.4.3", "4.4.3")), 0L)
+  expect_equal(nrow(newer_r_candidates(db, "4.5.3", "4.4.3")), 0L)
+  expect_equal(nrow(newer_r_candidates(db, "4.5.3", "4.6.0")), 0L)
+})
+
+test_that("LinkingTo, dependency cycles, Windows and empty metadata are handled", {
+  db <- fixture()
+  db$LinkingTo[5L] <- "arules"
+  db$Imports[1L] <- "ordinary"
+  db$OS_type <- c(NA, "windows", rep(NA_character_, 5L))
+  expect_identical(
+    newer_r_candidates(db, "4.4.3", "4.6.0")$Package,
+    c("arules", "indirect", "ordinary")
+  )
+  expect_equal(nrow(newer_r_candidates(db[FALSE, ], "4.4.3", "4.6.0")), 0L)
+})
+
+test_that("R-devel Recommended copies cannot block ordinary CRAN dependencies", {
+  db <- fixture()
+  db$Path <- NA_character_
+  future_copy <- db[1L, ]
+  future_copy$Depends <- "R (>= 4.7)"
+  future_copy$Path <- "4.7.0/Recommended"
+  expect_identical(
+    newer_r_candidates(rbind(future_copy, db), "4.4.3", "4.6.1")$Package,
+    c("arules", "arulesSequences", "indirect")
+  )
+})