Merge remote-tracking branch 'origin/main' into t3code/smarter-disk-pruning-macmini

# Conflicts:
#	.crow/build-all-versions-install-deps.yaml
#	.crow/build-all-versions.yaml

local/build-all.R

@@ -64,12 +64,38 @@ sprintf("# of package versions for this job: %s", nrow(chunk))
 exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]]
 chunk <- chunk[!chunk$Package %in% exclude, ]
 
-# Skip package versions already built in a previous run.
+# Skip package versions already attempted in a previous run (built or errored).
 # pkgs_to_build.rds is a static snapshot from the install-deps step, so on a
 # restart it still lists everything an interrupted run already produced. The
 # metadata DB reflects that progress, so we re-derive the remaining set here.
-platform <- paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-")
-arch <- Sys.getenv("ARCH")
+# We exclude *all* attempted versions, not just successful ones: a previously
+# errored version is skipped by build_binary_package() anyway, so leaving it in
+# the chunk only makes the job cycle through it one-by-one for no benefit.
+# Derive platform + arch from the running container, mirroring the codename ->
+# platform mapping bincraft uses internally. The OS/OS_VERSION selectors are
+# workflow-level CI variables that are not injected into the container
+# environment, so Sys.getenv() would return "" and this pre-filter would query
+# platform "-" and skip nothing.
+codename <- bincraft::set_codename(NULL)
+platform <- switch(
+  codename,
+  jammy = "ubuntu-2204",
+  noble = "ubuntu-2404",
+  resolute = "ubuntu-2604",
+  rhel10 = "redhat-10",
+  rhel9 = "redhat-9",
+  rhel8 = "redhat-8",
+  alpine320 = "alpine-320",
+  alpine321 = "alpine-321",
+  alpine322 = "alpine-322",
+  alpine323 = "alpine-323",
+  alpine324 = "alpine-324",
+  alpine325 = "alpine-325",
+  alpine326 = "alpine-326",
+  NA_character_
+)
+local_machine <- Sys.info()[["machine"]]
+arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64"
 con <- DBI::dbConnect(
   RPostgres::Postgres(),
   dbname = "build_metadata",
@@ -81,13 +107,13 @@ con <- DBI::dbConnect(
 )
 built <- DBI::dbGetQuery(
   con,
-  "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2 AND error_occurred = FALSE",
+  "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2",
   params = list(platform, arch)
 )
 DBI::dbDisconnect(con)
 before <- nrow(chunk)
 chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ]
-sprintf("Skipped %d already-built package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk))
+sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk))
 
 # Read pre-computed S3 listing from install-deps step
 # This avoids loading s3fs/reticulate/Python in the build container,
@@ -121,6 +147,7 @@ mapply(
       metadata_db_sslmode = "require",
       metadata_db_port = 15432,
       archive = TRUE,
+      patches = "local/patches",
       upload = TRUE,
       store_build_metadata = TRUE
     )

local/build-one.R

@@ -103,6 +103,7 @@ for (ver in versions) {
     force = TRUE,
     upload = TRUE,
     archive = TRUE,
+    patches = "local/patches",
     store_build_metadata = TRUE,
     s3_endpoint = s3$s3_endpoint,
     s3_region = s3$s3_region,

local/install-bincraft.R

@@ -0,0 +1,52 @@
+#!/usr/bin/env Rscript
+
+# Install the latest tagged bincraft release, resolved dynamically, so the CI
+# workflows and the build-one image never pin a hardcoded version (no more
+# editing `@vX.Y.Z` in many places on every release).
+#
+# Run with the R whose library should receive bincraft:
+#   Rscript local/install-bincraft.R
+# or, to target a specific R from a shell loop:
+#   "$RBIN" -q -e 'source("local/install-bincraft.R")'
+#
+# How it works: list the remote tags with `git ls-remote` (no token needed for
+# the public repo), keep the `vX.Y.Z` release tags, pick the highest version,
+# and install it with pak. pak is idempotent on the git ref, so re-running keeps
+# the package when it is already current and only updates when a newer tag ships.
+# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is
+# identical across git versions and `system2()` argument handling.
+
+repo_url <- Sys.getenv(
+  "BINCRAFT_GIT_URL",
+  unset = "https://codefloe.com/rpkgs/bincraft.git"
+)
+
+# GIT_TERMINAL_PROMPT=0 keeps a non-interactive run from hanging on auth.
+refs <- system2(
+  "git",
+  c("ls-remote", "--tags", repo_url),
+  stdout = TRUE,
+  stderr = FALSE,
+  env = "GIT_TERMINAL_PROMPT=0"
+)
+tags <- sub(".*refs/tags/", "", refs)
+tags <- tags[!grepl("\\^\\{\\}$", tags)] # drop dereferenced "...^{}" lines
+tags <- grep("^v[0-9]", tags, value = TRUE) # only vX.Y.Z release tags
+if (length(tags) == 0L) {
+  stop(
+    "Could not resolve any bincraft release tag from ",
+    repo_url,
+    call. = FALSE
+  )
+}
+latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][
+  1L
+]
+
+message(sprintf("Installing latest bincraft release: %s", latest))
+pak::pak(sprintf("git::%s@%s", repo_url, latest))
+message(sprintf(
+  "bincraft %s installed (%s)",
+  as.character(utils::packageVersion("bincraft")),
+  latest
+))

local/patches/README.md

@@ -0,0 +1,43 @@
+# Patch Registry
+
+This directory contains the curated registry of per-package build-time patches consumed by bincraft's `patches` argument.
+
+## Schema
+
+The registry is defined in `registry.json` as an array of patch entries. Each entry specifies lightweight build-time overrides (environment variables, configure arguments, Makevars) and optionally a source diff to apply before building.
+
+### Field semantics
+
+| Field | Type | Required | Description |
+| --- | --- | --- | --- |
+| `package` | string | yes | CRAN package name. |
+| `versions` | string | yes | `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. |
+| `platforms` | array of strings | yes | Matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). An entry matches if any listed token matches any build token. `["*"]` matches all platforms. |
+| `env` | object | no | Environment variables exported only for this package's isolated build. |
+| `configure_args` | array | no | Arguments passed as `--configure-args` to the isolated build. |
+| `makevars` | object | no | Key/value pairs written into a package-local Makevars for the isolated build. |
+| `patch` | string or null | no | Path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. |
+| `reason` | string | yes | Human explanation, surfaced in logs and metadata. |
+
+## Adding an entry
+
+To add a new patch entry:
+
+1. Add an object to the array in `registry.json` with the fields documented above.
+   Start with lightweight overrides (environment variables, configure arguments, Makevars) before resorting to source diffs.
+
+2. If a source diff is needed, place it in `local/patches/<package>/<file>.patch` and reference its path in the `patch` field.
+   For example, a diff for `RcppParallel` would go in `local/patches/RcppParallel/fix.patch` and be referenced as `"patch": "RcppParallel/fix.patch"`.
+
+3. The `reason` field should clearly explain why the patch is needed and what problem it solves.
+
+## Validation
+
+The registry is validated and applied by bincraft during the build process.
+For manual validation, run the validator from the repo root:
+
+```bash
+Rscript local/validate-patches.R
+```
+
+This validates the schema, referenced patch-file existence, and checks for duplicate entries across platforms and versions.

local/patches/RcppParallel/disable-tbb.patch

@@ -0,0 +1,16 @@
+diff --git a/src/Makevars.in b/src/Makevars.in
+index be8445f..faee771 100644
+--- a/src/Makevars.in
++++ b/src/Makevars.in
+@@ -60,7 +60,10 @@ else
+ 	endif
+ 
+ 	ifeq ($(UNAME), Linux)
+-		USE_TBB=Linux
++		# bincraft patch: the bundled Intel TBB build hangs/fails on musl (Alpine)
++		# and newer toolchains (g++ 15). Skip it (leave USE_TBB unset) and force the
++		# TinyThread backend so RcppParallel still builds.
++		PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=0
+ 	endif
+ 
+ 	ifeq ($(UNAME), SunOS)

local/patches/fs/force-vendored-libuv.patch

@@ -0,0 +1,19 @@
+diff --git a/configure b/configure
+--- a/configure
++++ b/configure
+@@ -11,6 +11,15 @@
+ PKG_TEST_HEADER="<uv.h>"
+ PKG_LIBS="-luv"
+ 
++# bincraft patch: force the vendored static libuv so the resulting binary
++# is self-contained. fs configure otherwise links system libuv whenever
++# pkg-config finds libuv-devel (installed as a build-time sysreq), yielding
++# an fs.so with NEEDED libuv.so.1 that fails to dyn.load on machines lacking
++# runtime libuv (install.packages/renv do not install SystemRequirements).
++echo "Building static libuv (bincraft: forced vendored)" 1>&2
++cp -f src/Makevars.vendor src/Makevars
++exit 0
++
+ # Use pkg-config if available
+ if [ `command -v pkg-config` ]; then
+   PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}`

local/patches/registry.json

@@ -0,0 +1,22 @@
+[
+  {
+    "package": "RcppParallel",
+    "versions": "*",
+    "platforms": ["alpine", "ubuntu-2604"],
+    "env": {},
+    "configure_args": [],
+    "makevars": {},
+    "patch": "RcppParallel/disable-tbb.patch",
+    "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); patch unsets USE_TBB and forces -DRCPP_PARALLEL_USE_TBB=0 so RcppParallel skips the bundled build and uses the TinyThread backend"
+  },
+  {
+    "package": "fs",
+    "versions": "*",
+    "platforms": ["*"],
+    "env": {},
+    "configure_args": [],
+    "makevars": {},
+    "patch": "fs/force-vendored-libuv.patch",
+    "reason": "fs 2.x configure links system libuv whenever pkg-config finds libuv-devel (installed as a build-time sysreq), producing an fs.so with NEEDED libuv.so.1. That binary fails to dyn.load on consumer machines lacking runtime libuv, because install.packages/renv do not install SystemRequirements (only pak does, and only in the build container). The patch short-circuits configure to copy src/Makevars.vendor and build the bundled static libuv (needs cmake) so the binary is self-contained on every platform. An env/pkg-config override was tried first but the rebuilt binary still linked libuv.so.1, so a source patch is used instead."
+  }
+]

local/validate-patches.R

@@ -0,0 +1,56 @@
+#!/usr/bin/env Rscript
+# Validate local/patches/registry.json: schema, referenced patch files, and
+# ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI.
+
+dir <- "local/patches"
+registry_file <- file.path(dir, "registry.json")
+if (!file.exists(registry_file)) {
+  cat("No registry.json found; nothing to validate.\n")
+  quit(status = 0L)
+}
+
+or_q <- function(x) if (is.null(x)) "?" else x
+
+reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE)
+required <- c("package", "versions", "platforms", "reason")
+errs <- character(0L)
+
+for (i in seq_along(reg)) {
+  e <- reg[[i]]
+  missing <- setdiff(required, names(e))
+  if (length(missing) > 0L) {
+    errs <- c(errs, sprintf(
+      "entry %d (%s): missing %s", i,
+      if (is.null(e$package)) "?" else e$package, toString(missing)
+    ))
+  }
+  if (!is.null(e$patch)) {
+    p <- file.path(dir, e$patch)
+    if (!file.exists(p)) {
+      errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing",
+                              i, e$package, p))
+    }
+  }
+}
+
+# Ambiguous overlap: two entries for the same package with identical platforms
+# and versions.
+keys <- vapply(reg, function(e) {
+  sprintf(
+    "%s|%s|%s",
+    or_q(e$package),
+    paste(sort(as.character(unlist(e$platforms))), collapse = ","),
+    or_q(e$versions)
+  )
+}, character(1L))
+dups <- keys[duplicated(keys)]
+if (length(dups) > 0L) {
+  errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups))))
+}
+
+if (length(errs) > 0L) {
+  cat("Patch registry validation FAILED:\n")
+  cat(paste0("  - ", errs, "\n"))
+  quit(status = 1L)
+}
+cat(sprintf("Patch registry OK (%d %s).\n", length(reg), if (length(reg) == 1L) "entry" else "entries"))