From 8afa584da39ac7c34b445f6c44ab6c73b3764cce Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 15:01:55 +0000 Subject: [PATCH 001/118] feat(local): auto-propose registry patches and track the feedback loop Implement steps 3 + 4 of issue #115 on top of the failure classifier, now that bincraft v4.4.3 applies registry patches/makevars/configure_args to the target build (not just dependencies), so a trial patched build is meaningful. - refactor the classify helpers to expose a pure build_triage_report() and a list-returning entry builder; failing-builds-report.R now renders from it - add local/propose-patches.R (step 3, "propose, do not apply"): emit a pre-filled registry.json entry for each classified, safe, unregistered failure, validate the candidate set against a temporary merged registry, and (only on request) --write it plus a proposals ledger, or --open-issue a Forgejo tracking issue; the human gate and validator/trial-build acceptance stay, and novel source diffs / unknown signatures are never proposed - add local/trial-build-patch.R: isolated bincraft build of one package with the registry applied (no upload/archive/metadata) as the pre-merge gate - add local/proposal-tracking.R + local/proposal-tracking-lib.R (step 4): signature hit rate, proposed-vs-merged, and retirement candidates, with the pure helpers covered by tests - teach validate-patches.R optional PATCH_DIR/REGISTRY_FILE overrides so a candidate registry can be validated without touching the real one - document the propose/trial-build/tracking workflow in local/patches/README.md --- local/failing-builds-classify.R | 116 +++++++- local/failing-builds-report.R | 127 ++------ local/patches/README.md | 34 +++ local/proposal-tracking-lib.R | 103 +++++++ local/proposal-tracking.R | 135 +++++++++ local/propose-patches.R | 362 +++++++++++++++++++++++ local/tests/test-proposal-tracking-lib.R | 105 +++++++ local/trial-build-patch.R | 65 ++++ local/validate-patches.R | 62 ++-- 9 files changed, 990 insertions(+), 119 deletions(-) create mode 100644 local/proposal-tracking-lib.R create mode 100644 local/proposal-tracking.R create mode 100644 local/propose-patches.R create mode 100644 local/tests/test-proposal-tracking-lib.R create mode 100644 local/trial-build-patch.R diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index b6f3430..30c168d 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -175,10 +175,10 @@ classify_error <- function(error_text, signatures = build_signatures()) { ) } -# Render a suggested registry.json entry (as a pretty JSON string) for a -# classified group, filling package/versions/platforms from the observed -# failures. Only meaningful when the signature carries a `registry` template. -propose_registry_entry <- function( +# Build a suggested registry.json entry (as an R list) for a classified group, +# filling package/versions/platforms from the observed failures. Only +# meaningful when the signature carries a `registry` template. +propose_registry_entry_list <- function( signature, package, platforms, @@ -192,7 +192,7 @@ propose_registry_entry <- function( package = package, versions = versions, # I() keeps this a JSON array even when a single platform is affected. - platforms = I(as.character(platforms)) + platforms = I(sort(unique(as.character(platforms)))) ) for (k in c("env", "configure_args", "makevars", "patch")) { if (!is.null(tmpl[[k]])) { @@ -200,5 +200,111 @@ propose_registry_entry <- function( } } entry$reason <- tmpl$reason + entry +} + +# Same, rendered as a pretty JSON string. +propose_registry_entry <- function( + signature, + package, + platforms, + versions = "*" +) { + entry <- propose_registry_entry_list(signature, package, platforms, versions) + if (is.null(entry)) { + return(NULL) + } jsonlite::toJSON(entry, auto_unbox = TRUE, pretty = TRUE, null = "null") } + +# Resolve the representative signature for a group by its id, re-attaching the +# `matched` flag the report/proposer rely on. Unknown ids -> unclassified. +resolve_signature <- function(sig_id, signatures = build_signatures()) { + hit <- Filter(function(s) s$id == sig_id, signatures) + if (length(hit) > 0L) { + s <- hit[[1L]] + s$matched <- TRUE + s + } else { + classify_error("", signatures) # unclassified fallback (matched = FALSE) + } +} + +# --------------------------------------------------------------------------- +# Triage report (shared by the report printer and the patch proposer) +# --------------------------------------------------------------------------- +# `failures` is a data.frame with at least: name, platform, arch, error_text. +# `registered_pkgs` is the set of packages that already carry a registry entry. +# Returns a list of group records (one per root-cause bucket), ordered by +# number of affected builds descending. Pure: no IO, no printing. +build_triage_report <- function( + failures, + registered_pkgs = character(0L), + signatures = build_signatures() +) { + if (nrow(failures) == 0L) { + return(list()) + } + idx <- seq_len(nrow(failures)) + failures$fingerprint <- vapply( + idx, + function(i) fingerprint_error(failures$error_text[[i]], failures$name[[i]]), + character(1L) + ) + failures$sig_id <- vapply( + idx, + function(i) classify_error(failures$error_text[[i]], signatures)$id, + character(1L) + ) + # Group by root cause: classified failures collapse by signature id (same fix + # candidate is one bucket regardless of log noise); unclassified failures fall + # back to the fingerprint so distinct unknowns stay separate for discovery. + failures$group_key <- ifelse( + failures$sig_id == "unclassified", + failures$fingerprint, + failures$sig_id + ) + groups <- split(failures, failures$group_key) + groups <- groups[order(-vapply(groups, nrow, integer(1L)))] + + lapply(unname(groups), function(g) { + sig_id <- names(sort(table(g$sig_id), decreasing = TRUE))[[1L]] + sig <- resolve_signature(sig_id, signatures) + + fp_tab <- sort(table(g$fingerprint), decreasing = TRUE) + pkgs <- sort(unique(g$name)) + plats <- sort(unique(g$platform)) + arches <- sort(unique(g$arch)) + unregistered <- setdiff(pkgs, registered_pkgs) + auto_proposable <- isTRUE(sig$auto) && sig$matched + + proposed <- list() + if (auto_proposable) { + for (p in unregistered) { + proposed[[p]] <- propose_registry_entry_list( + sig, + p, + g$platform[g$name == p] + ) + } + } + + list( + signature = sig$id, + label = sig$label, + matched = sig$matched, + auto_proposable = auto_proposable, + tier = sig$tier, + confidence = sig$confidence, + suggested_fix = sig$fix, + fingerprint = names(fp_tab)[[1L]], + fingerprint_variants = length(fp_tab), + build_count = nrow(g), + packages = pkgs, + packages_without_entry = unregistered, + platforms = plats, + arches = arches, + proposed_entries = if (length(proposed) > 0L) proposed else NULL + ) + }) +} diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R index 855f042..60f65ea 100644 --- a/local/failing-builds-report.R +++ b/local/failing-builds-report.R @@ -127,99 +127,53 @@ if (file.exists(registry_file)) { } # --------------------------------------------------------------------------- -# Fingerprint + classify every failure, then group +# Classify + group (shared logic), then render # --------------------------------------------------------------------------- -signatures <- build_signatures() -failures$fingerprint <- vapply( - seq_len(nrow(failures)), - function(i) fingerprint_error(failures$error_text[[i]], failures$name[[i]]), - character(1L) -) -failures$sig_id <- vapply( - seq_len(nrow(failures)), - function(i) classify_error(failures$error_text[[i]], signatures)$id, - character(1L) -) +report <- build_triage_report(failures, registered_pkgs) +report <- Filter(function(r) r$build_count >= min_count, report) -# Group by root cause: classified failures collapse by signature id (so the -# same fix candidate is one bucket regardless of surrounding log noise); -# unclassified failures fall back to the fingerprint so distinct unknowns stay -# separate for discovery. -failures$group_key <- ifelse( - failures$sig_id == "unclassified", - failures$fingerprint, - failures$sig_id -) -groups <- split(failures, failures$group_key) -# Order groups by number of affected builds, descending. -groups <- groups[order(-vapply(groups, nrow, integer(1L)))] - -report <- list() cat(sprintf( - "\n%d distinct failure fingerprint(s); showing groups with >= %d build(s).\n", - length(groups), + "\n%d distinct failure group(s); showing groups with >= %d build(s).\n", + length(report), min_count )) cat(strrep("=", 78L), "\n", sep = "") -for (g in groups) { - n <- nrow(g) - if (n < min_count) { - next - } - - # Representative classification: most common signature id in the group. - sig_id <- names(sort(table(g$sig_id), decreasing = TRUE))[[1L]] - sig <- Filter(function(s) s$id == sig_id, signatures) - if (length(sig) > 0L) { - sig <- sig[[1L]] - sig$matched <- TRUE - } else { - sig <- classify_error("", signatures) # unclassified fallback (matched = FALSE) - } - - fp_tab <- sort(table(g$fingerprint), decreasing = TRUE) - rep_fp <- names(fp_tab)[[1L]] - fp_variants <- length(fp_tab) - pkgs <- sort(unique(g$name)) - plats <- sort(unique(g$platform)) - arches <- sort(unique(g$arch)) - unregistered <- setdiff(pkgs, registered_pkgs) - - status <- if (!sig$matched) { +for (r in report) { + status <- if (!r$matched) { "HUMAN TRIAGE (unknown signature)" - } else if (!sig$auto) { - sprintf("HUMAN TRIAGE (classified: %s; novel source diff)", sig$id) + } else if (!r$auto_proposable) { + sprintf("HUMAN TRIAGE (classified: %s; novel source diff)", r$signature) } else { - sprintf("AUTO-PROPOSABLE (%s, %s confidence)", sig$id, sig$confidence) + sprintf("AUTO-PROPOSABLE (%s, %s confidence)", r$signature, r$confidence) } cat(sprintf( "\n[%d builds | %d pkgs | %s] %s\n", - n, - length(pkgs), - toString(plats), + r$build_count, + length(r$packages), + toString(r$platforms), status )) cat(sprintf( " fingerprint: %s%s\n", - rep_fp, - if (fp_variants > 1L) { - sprintf(" (+%d fingerprint variant(s))", fp_variants - 1L) + r$fingerprint, + if (r$fingerprint_variants > 1L) { + sprintf(" (+%d fingerprint variant(s))", r$fingerprint_variants - 1L) } else { "" } )) - if (sig$matched) { - cat(sprintf(" signature : %s\n", sig$label)) - cat(sprintf(" suggested : [%s] %s\n", sig$tier, sig$fix)) + if (r$matched) { + cat(sprintf(" signature : %s\n", r$label)) + cat(sprintf(" suggested : [%s] %s\n", r$tier, r$suggested_fix)) } - cat(sprintf(" arch : %s\n", toString(arches))) + cat(sprintf(" arch : %s\n", toString(r$arches))) cat(sprintf( " packages : %s\n", paste( vapply( - pkgs, + r$packages, function(p) { if (p %in% registered_pkgs) paste0(p, " (has entry)") else p }, @@ -229,39 +183,20 @@ for (g in groups) { ) )) - proposals <- list() - if (sig$matched && sig$auto && length(unregistered) > 0L) { + if (!is.null(r$proposed_entries)) { cat(" proposed registry entries (validate + trial-build before merge):\n") - for (p in unregistered) { - p_plats <- sort(unique(g$platform[g$name == p])) - entry <- propose_registry_entry(sig, p, p_plats) - proposals[[p]] <- entry - cat(paste0(" ", gsub("\n", "\n ", entry)), "\n", sep = "") + for (entry in r$proposed_entries) { + j <- jsonlite::toJSON( + entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "") } - } else if (sig$matched && sig$auto && length(unregistered) == 0L) { + } else if (r$auto_proposable) { cat(" (all affected packages already have a registry entry)\n") } - - report[[length(report) + 1L]] <- list( - fingerprint = rep_fp, - fingerprint_variants = fp_variants, - build_count = n, - packages = pkgs, - packages_without_entry = unregistered, - platforms = plats, - arches = arches, - signature = sig$id, - matched = sig$matched, - auto_proposable = isTRUE(sig$auto) && sig$matched, - tier = sig$tier, - confidence = sig$confidence, - suggested_fix = sig$fix, - proposed_entries = if (length(proposals) > 0L) { - lapply(proposals, jsonlite::fromJSON) - } else { - NULL - } - ) } # --------------------------------------------------------------------------- diff --git a/local/patches/README.md b/local/patches/README.md index b57cc1b..7dd769c 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -59,3 +59,37 @@ For auto-proposable groups it prints a ready-to-review registry entry; still run Novel source diffs and unknown signatures stay human-reviewed by design. Add a new signature by appending a rule to `build_signatures()` in `local/failing-builds-classify.R`; the pure helpers are covered by `local/tests/test-failing-builds-classify.R`. + +### Proposing entries (step 3: propose, do not apply) + +`local/propose-patches.R` takes the auto-proposable candidates one step further: for each classified, safe fix affecting a package with no current entry, it emits a pre-filled `registry.json` entry and validates the candidate set against a temporary merged registry (the real registry is never touched unless you ask). +The human gate stays: it never merges. + +```bash +# Default: print candidates + validation, take no action. +PGPASS=... Rscript local/propose-patches.R +# Append the candidates to registry.json + the proposals ledger (you commit + open the PR). +PGPASS=... Rscript local/propose-patches.R --write +# Or post/update a Forgejo tracking issue instead (needs FORGEJO_TOKEN). +PGPASS=... FORGEJO_TOKEN=... Rscript local/propose-patches.R --open-issue +``` + +The acceptance criteria before merging a proposal are: `validate-patches.R` passes (checked automatically), and an isolated trial build succeeds. +Run the trial build inside the failing platform's build-env image; it uploads/archives nothing and writes no metadata: + +```bash +Rscript local/trial-build-patch.R +``` + +`--write` and `--open-issue` also append to `local/patches/proposals-log.json`, a ledger of what was proposed. + +### Feedback loop (step 4) + +`local/proposal-tracking.R` reports the signature hit rate, proposed-vs-merged status (a proposal counts as merged once its package appears in the registry), and retirement candidates (registry entries whose package no longer appears in any current failure, so the upstream cause was likely fixed). +It is read-only. + +```bash +PGPASS=... Rscript local/proposal-tracking.R --json metrics.json +``` + +The pure metric/ledger helpers live in `local/proposal-tracking-lib.R` and are covered by `local/tests/test-proposal-tracking-lib.R`. diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R new file mode 100644 index 0000000..4e8a468 --- /dev/null +++ b/local/proposal-tracking-lib.R @@ -0,0 +1,103 @@ +# Pure, IO-free helpers for the proposal feedback loop (issue #115, step 4): +# a small ledger of what the classifier proposed, plus metrics derived from the +# live triage report and the registry (signature hit rate, proposed-vs-merged, +# retirement candidates). +# +# Kept free of DB/HTTP/clock so it can be sourced by the proposer, the tracker +# entrypoint, and the unit tests. Timestamps are passed in by callers. + +# Stable identity of a ledger record: one proposal per (package, signature). +ledger_key <- function(record) { + paste0( + if (is.null(record$package)) "?" else record$package, + "|", + if (is.null(record$signature)) "?" else record$signature + ) +} + +# Merge freshly-generated proposals into an existing ledger without clobbering +# history: a record whose (package, signature) already exists is left as-is +# (its status/PR/issue are preserved); genuinely new proposals are appended. +# Returns the combined list. Pure: callers stamp `proposed_at` before passing. +merge_ledger <- function(existing, new_records) { + if (is.null(existing)) { + existing <- list() + } + seen <- vapply(existing, ledger_key, character(1L)) + out <- existing + for (rec in new_records) { + if (!(ledger_key(rec) %in% seen)) { + out[[length(out) + 1L]] <- rec + seen <- c(seen, ledger_key(rec)) + } + } + out +} + +# Per-signature hit rate from a triage report (list of group records from +# build_triage_report) crossed with the set of already-registered packages. +# For each *matched* signature: how many failing builds/packages it explains, +# and how many of those packages are already addressed by a registry entry. +signature_hit_rate <- function(report, registered_pkgs = character(0L)) { + matched <- Filter(function(g) isTRUE(g$matched), report) + by_sig <- split( + matched, + vapply(matched, function(g) g$signature, character(1L)) + ) + lapply(names(by_sig), function(sig) { + grps <- by_sig[[sig]] + pkgs <- unique(unlist(lapply(grps, function(g) g$packages))) + addressed <- intersect(pkgs, registered_pkgs) + list( + signature = sig, + builds = sum(vapply(grps, function(g) g$build_count, integer(1L))), + packages = length(pkgs), + addressed = length(addressed), + open = length(setdiff(pkgs, registered_pkgs)), + auto_proposable = any(vapply( + grps, + function(g) isTRUE(g$auto_proposable), + logical(1L) + )) + ) + }) +} + +# Proposed-vs-merged: a ledger record counts as "merged" once its package +# appears in the registry. Returns per-record status plus a rollup. +proposed_vs_merged <- function(ledger, registered_pkgs = character(0L)) { + if (is.null(ledger)) { + ledger <- list() + } + rows <- lapply(ledger, function(rec) { + merged <- !is.null(rec$package) && rec$package %in% registered_pkgs + list( + package = rec$package, + signature = rec$signature, + status = if (merged) "merged" else (rec$status %||% "proposed"), + merged = merged + ) + }) + list( + records = rows, + total = length(rows), + merged = sum(vapply(rows, function(r) isTRUE(r$merged), logical(1L))) + ) +} + +# Registry entries whose package no longer appears in any current failure are +# retirement candidates: the upstream cause was likely fixed, so the entry can +# be reviewed for removal. `failing_pkgs` is the set of currently-failing +# package names (from the live report). +retirement_candidates <- function(registry_entries, failing_pkgs) { + if (is.null(registry_entries)) { + registry_entries <- list() + } + keep <- Filter( + function(e) !is.null(e$package) && !(e$package %in% failing_pkgs), + registry_entries + ) + vapply(keep, function(e) as.character(e$package), character(1L)) +} + +`%||%` <- function(a, b) if (is.null(a)) b else a diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R new file mode 100644 index 0000000..60bbd10 --- /dev/null +++ b/local/proposal-tracking.R @@ -0,0 +1,135 @@ +#!/usr/bin/env Rscript + +# Feedback loop for the failure classifier (issue #115, step 4): report the +# signature hit rate, proposed-vs-merged status, and registry entries that look +# retirable, so the rule set can improve and stale entries can be pruned. +# Read-only: queries `single_builds` and reads the registry + proposals ledger. +# +# Usage: +# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(DBI, quietly = TRUE) + library(RPostgres, quietly = TRUE) + library(jsonlite, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "failing-builds-classify.R")) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +args <- commandArgs(trailingOnly = TRUE) +json_out <- local({ + i <- match("--json", args) + if (!is.na(i) && i < length(args)) args[[i + 1L]] else NA_character_ +}) + +if (nchar(Sys.getenv("PGPASS")) == 0L) { + stop("PGPASS env var is not set; a DB password is required.") +} + +registry_file <- file.path(script_dir, "patches", "registry.json") +ledger_file <- file.path(script_dir, "patches", "proposals-log.json") + +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(DBI::dbDisconnect(con), add = TRUE) + +failures <- DBI::dbGetQuery( + con, + paste0( + "SELECT name, tag, platform, arch, r_version, timestamp, error_text ", + "FROM single_builds WHERE error_occurred = TRUE AND removed = FALSE" + ) +) + +existing_entries <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +registered_pkgs <- unique(vapply( + existing_entries, + function(e) as.character(e$package %||% ""), + character(1L) +)) +ledger <- if (file.exists(ledger_file)) { + jsonlite::fromJSON(ledger_file, simplifyVector = FALSE) +} else { + list() +} + +report <- build_triage_report(failures, registered_pkgs) +failing_pkgs <- unique(unlist(lapply(report, function(g) g$packages))) + +hit <- signature_hit_rate(report, registered_pkgs) +pvm <- proposed_vs_merged(ledger, registered_pkgs) +retire <- retirement_candidates(existing_entries, failing_pkgs) + +cat(sprintf( + "Feedback loop over %d failing builds, %d registry entries, %d ledger records.\n", + nrow(failures), + length(existing_entries), + length(ledger) +)) + +cat("\nSignature hit rate:\n") +cat(strrep("-", 60L), "\n", sep = "") +for (h in hit) { + cat(sprintf( + " %-26s %3d builds | %2d pkgs | %2d addressed | %2d open%s\n", + h$signature, + h$builds, + h$packages, + h$addressed, + h$open, + if (isTRUE(h$auto_proposable)) " [auto]" else "" + )) +} + +cat(sprintf("\nProposed vs merged: %d / %d merged.\n", pvm$merged, pvm$total)) +for (r in pvm$records) { + cat(sprintf(" %-24s %-26s %s\n", r$package, r$signature, r$status)) +} + +cat(sprintf( + "\nRetirement candidates (%d): registry entries with no current failure.\n", + length(retire) +)) +if (length(retire) > 0L) { + cat(paste0(" - ", retire, "\n"), sep = "") +} else { + cat(" (none)\n") +} + +if (!is.na(json_out)) { + jsonlite::write_json( + list( + signature_hit_rate = hit, + proposed_vs_merged = pvm, + retirement_candidates = retire + ), + json_out, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf("\nWrote metrics to %s\n", json_out)) +} diff --git a/local/propose-patches.R b/local/propose-patches.R new file mode 100644 index 0000000..20583ba --- /dev/null +++ b/local/propose-patches.R @@ -0,0 +1,362 @@ +#!/usr/bin/env Rscript + +# Propose registry patches for classified, auto-proposable build failures +# (issue #115, step 3: "propose, do not apply"). Queries `single_builds`, +# classifies failures, and for each *known, safe* fix lever affecting a package +# with no current registry entry, emits a pre-filled `registry.json` entry. +# +# The human gate stays: this never merges. Acceptance before merge is +# 1. `local/validate-patches.R` passes (checked here against a candidate +# registry), and +# 2. an isolated trial build succeeds -- run `local/trial-build-patch.R ` +# (this script prints the exact command per candidate). +# Novel source diffs and unknown signatures are never proposed (they carry +# `auto = FALSE` in the signature table). +# +# Usage: +# PGPASS=... Rscript local/propose-patches.R [--platform P] [--arch A] [--min N] +# (default) print candidates + validation, take no action +# --write append candidates to local/patches/registry.json and +# the proposals ledger (commit + open a PR yourself) +# --open-issue post/update a Forgejo tracking issue (needs FORGEJO_TOKEN) +# --json PATH also write the machine-readable candidate list to PATH + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(DBI, quietly = TRUE) + library(RPostgres, quietly = TRUE) + library(jsonlite, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "failing-builds-classify.R")) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +# --------------------------------------------------------------------------- +# Arguments +# --------------------------------------------------------------------------- +args <- commandArgs(trailingOnly = TRUE) +opt_val <- function(flag, default = NA_character_) { + i <- match(flag, args) + if (!is.na(i) && i < length(args)) args[[i + 1L]] else default +} +platform <- opt_val("--platform", Sys.getenv("PLATFORM", "")) +arch <- opt_val("--arch", Sys.getenv("ARCH", "")) +json_out <- opt_val("--json") +min_count <- suppressWarnings(as.integer(opt_val("--min", "1"))) +if (is.na(min_count)) { + min_count <- 1L +} +do_write <- "--write" %in% args +do_issue <- "--open-issue" %in% args + +if (nchar(Sys.getenv("PGPASS")) == 0L) { + stop("PGPASS env var is not set; a DB password is required.") +} + +registry_file <- file.path(script_dir, "patches", "registry.json") +ledger_file <- file.path(script_dir, "patches", "proposals-log.json") + +# --------------------------------------------------------------------------- +# Query failing builds (same shape as failing-builds-report.R) +# --------------------------------------------------------------------------- +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(DBI::dbDisconnect(con), add = TRUE) + +where <- "error_occurred = TRUE AND removed = FALSE" +params <- list() +if (nzchar(platform)) { + where <- paste0(where, " AND platform = $", length(params) + 1L) + params <- c(params, platform) +} +if (nzchar(arch)) { + where <- paste0(where, " AND arch = $", length(params) + 1L) + params <- c(params, arch) +} +failures <- DBI::dbGetQuery( + con, + paste0( + "SELECT name, tag, platform, arch, r_version, timestamp, error_text ", + "FROM single_builds WHERE ", + where + ), + params = if (length(params) > 0L) params else NULL +) +cat(sprintf("Queried single_builds: %d failing builds\n", nrow(failures))) + +# --------------------------------------------------------------------------- +# Existing registry + classification +# --------------------------------------------------------------------------- +existing_entries <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +registered_pkgs <- unique(vapply( + existing_entries, + function(e) as.character(e$package %||% ""), + character(1L) +)) + +report <- build_triage_report(failures, registered_pkgs) +report <- Filter(function(r) r$build_count >= min_count, report) + +# Flatten auto-proposable groups into candidate records. +candidates <- list() +for (r in report) { + if (is.null(r$proposed_entries)) { + next + } + for (pkg in names(r$proposed_entries)) { + candidates[[length(candidates) + 1L]] <- list( + package = pkg, + signature = r$signature, + confidence = r$confidence, + tier = r$tier, + entry = r$proposed_entries[[pkg]] + ) + } +} + +if (length(candidates) == 0L) { + cat( + "No auto-proposable candidates (nothing classified, safe, and unregistered).\n" + ) + q(status = 0) +} + +cat(sprintf( + "\n%d candidate registry %s:\n", + length(candidates), + if (length(candidates) == 1L) "entry" else "entries" +)) +cat(strrep("=", 78L), "\n", sep = "") +for (c in candidates) { + j <- jsonlite::toJSON( + c$entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf( + "\n# %s [signature: %s, %s confidence]\n", + c$package, + c$signature, + c$confidence + )) + cat(j, "\n", sep = "") + cat(sprintf( + " acceptance gate: PGPASS=... Rscript local/trial-build-patch.R %s\n", + c$package + )) +} + +# --------------------------------------------------------------------------- +# Validate the candidate set against a temporary merged registry (never +# touches the real registry unless --write succeeds). +# --------------------------------------------------------------------------- +rscript <- file.path(R.home("bin"), "Rscript") +candidate_entries <- lapply(candidates, function(c) c$entry) +merged <- c(existing_entries, candidate_entries) + +validate_registry <- function(entries) { + tmp <- tempfile(fileext = ".json") + on.exit(unlink(tmp), add = TRUE) + jsonlite::write_json( + entries, + tmp, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + status <- system2( + rscript, + file.path(script_dir, "validate-patches.R"), + env = c( + paste0("PATCH_DIR=", file.path(script_dir, "patches")), + paste0("REGISTRY_FILE=", tmp) + ), + stdout = TRUE, + stderr = TRUE + ) + cat(paste0(" ", status, "\n"), sep = "") + identical(attr(status, "status"), NULL) # NULL status attr == exit 0 +} + +cat("\n", strrep("=", 78L), "\n", sep = "") +cat("Validating candidate registry:\n") +ok <- validate_registry(merged) +if (!ok) { + stop("Candidate registry failed validation; not writing or proposing.") +} + +if (!is.na(json_out)) { + jsonlite::write_json( + candidates, + json_out, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf("Wrote candidate list to %s\n", json_out)) +} + +# --------------------------------------------------------------------------- +# Actions (default: none) +# --------------------------------------------------------------------------- +now <- format(Sys.time(), "%Y-%m-%d %H:%M:%S") +new_ledger_records <- lapply(candidates, function(c) { + list( + package = c$package, + signature = c$signature, + tier = c$tier, + confidence = c$confidence, + versions = c$entry$versions, + platforms = as.character(c$entry$platforms), + proposed_at = now, + status = "proposed" + ) +}) +load_ledger <- function() { + if (file.exists(ledger_file)) { + jsonlite::fromJSON(ledger_file, simplifyVector = FALSE) + } else { + list() + } +} +save_ledger <- function(led) { + jsonlite::write_json( + led, + ledger_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) +} + +if (do_write) { + jsonlite::write_json( + merged, + registry_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) + cat(sprintf( + "\nWrote %d entries to %s and updated %s.\n", + length(candidate_entries), + registry_file, + ledger_file + )) + cat( + "Next: run the acceptance-gate trial build(s), then commit and open a PR.\n" + ) +} else if (do_issue) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-issue requires FORGEJO_TOKEN.") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + issue_title <- "Auto-proposed registry patches (classifier #115)" + body_lines <- c( + sprintf("_Generated %s from `single_builds` failures._", now), + "", + "The failure classifier proposes these registry entries for packages with no current entry.", + "Each is a known, safe fix lever; **review, trial-build, and open a PR** -- nothing is applied automatically.", + "" + ) + for (c in candidates) { + j <- jsonlite::toJSON( + c$entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + body_lines <- c( + body_lines, + sprintf( + "### %s (%s, %s confidence)", + c$package, + c$signature, + c$confidence + ), + "", + "```json", + as.character(j), + "```", + sprintf( + "Acceptance gate: `PGPASS=... Rscript local/trial-build-patch.R %s`", + c$package + ), + "" + ) + } + new_body <- paste(body_lines, collapse = "\n") + + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(issue_title, reserved = TRUE) + ) + existing <- httr2::request(search_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + existing, + function(x) identical(x$title, issue_title), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- existing[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("Updated tracking issue #%d.\n", num)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/issues", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = issue_title, body = new_body)) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf("Opened tracking issue #%d.\n", created$number)) + } + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) +} else { + cat( + "\nDry run: no changes made. Re-run with --write or --open-issue to act.\n" + ) +} diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R new file mode 100644 index 0000000..55da4b6 --- /dev/null +++ b/local/tests/test-proposal-tracking-lib.R @@ -0,0 +1,105 @@ +source(file.path("..", "proposal-tracking-lib.R")) +source(file.path("..", "failing-builds-classify.R")) + +mk_failures <- function() { + data.frame( + name = c("StanHeaders", "rstan", "RcppParallel", "somepkg"), + platform = c("alpine-321", "alpine-321", "alpine-320", "redhat-9"), + arch = c("amd64", "arm64", "amd64", "amd64"), + error_text = c( + "fatal error: tbb/tbb_stddef.h: No such file or directory", + "In file: tbb/tbb_stddef.h: No such file or directory", + "Error: USE_TBB=Linux is not supported; bundled TBB on musl", + "some unmatched failure" + ), + stringsAsFactors = FALSE + ) +} + +test_that("merge_ledger appends new proposals and preserves existing history", { + existing <- list(list( + package = "fs", + signature = "system-libuv-link-leak", + status = "merged" + )) + new <- list( + list( + package = "fs", + signature = "system-libuv-link-leak", + status = "proposed" + ), + list( + package = "StanHeaders", + signature = "tbb-stddef-removed", + status = "proposed" + ) + ) + merged <- merge_ledger(existing, new) + expect_length(merged, 2L) # fs is deduped, StanHeaders added + fs <- Filter(function(r) r$package == "fs", merged)[[1L]] + expect_identical(fs$status, "merged") # existing status preserved, not clobbered +}) + +test_that("merge_ledger handles an empty/NULL starting ledger", { + new <- list(list(package = "x", signature = "s")) + expect_length(merge_ledger(NULL, new), 1L) + expect_length(merge_ledger(list(), new), 1L) +}) + +test_that("signature_hit_rate splits addressed vs open per signature", { + report <- build_triage_report(mk_failures(), registered_pkgs = "RcppParallel") + hit <- signature_hit_rate(report, registered_pkgs = "RcppParallel") + tbb <- Filter(function(h) h$signature == "tbb-stddef-removed", hit)[[1L]] + expect_identical(tbb$packages, 2L) # StanHeaders + rstan + expect_identical(tbb$addressed, 0L) + expect_identical(tbb$open, 2L) + expect_true(tbb$auto_proposable) + + rcpp <- Filter(function(h) h$signature == "rcppparallel-bundled-tbb", hit)[[ + 1L + ]] + expect_identical(rcpp$addressed, 1L) # already registered + expect_identical(rcpp$open, 0L) + + # Unclassified failures never appear as a signature. + expect_false( + "unclassified" %in% vapply(hit, function(h) h$signature, character(1L)) + ) +}) + +test_that("proposed_vs_merged marks a package merged once it is registered", { + ledger <- list( + list( + package = "StanHeaders", + signature = "tbb-stddef-removed", + status = "proposed" + ), + list( + package = "rstan", + signature = "tbb-stddef-removed", + status = "proposed" + ) + ) + pvm <- proposed_vs_merged(ledger, registered_pkgs = "StanHeaders") + expect_identical(pvm$total, 2L) + expect_identical(pvm$merged, 1L) + stan <- Filter(function(r) r$package == "StanHeaders", pvm$records)[[1L]] + expect_identical(stan$status, "merged") +}) + +test_that("retirement_candidates flags entries whose package no longer fails", { + entries <- list( + list(package = "RcppParallel"), + list(package = "oldpkg") + ) + # RcppParallel still fails; oldpkg does not -> only oldpkg is retirable. + out <- retirement_candidates( + entries, + failing_pkgs = c("RcppParallel", "StanHeaders") + ) + expect_identical(out, "oldpkg") + expect_length( + retirement_candidates(entries, failing_pkgs = c("RcppParallel", "oldpkg")), + 0L + ) +}) diff --git a/local/trial-build-patch.R b/local/trial-build-patch.R new file mode 100644 index 0000000..4e8030a --- /dev/null +++ b/local/trial-build-patch.R @@ -0,0 +1,65 @@ +#!/usr/bin/env Rscript + +# Acceptance gate for a proposed registry patch (issue #115, step 3): build one +# package in isolation with the current `local/patches` registry applied, and +# report whether it succeeds. Nothing is uploaded, archived, or written to the +# metadata DB -- `patchhash` keeps the trial binary out of the real cache. +# +# Must run inside a build-env image (the same image the failing platform uses), +# because it invokes the real compiler toolchain via bincraft. +# +# Usage: +# Rscript local/trial-build-patch.R [tag] +# +# Exit status: 0 if the patched build succeeds, 1 otherwise -- so it can gate a +# CI step or a manual pre-merge check. + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 1L) { + stop("usage: Rscript local/trial-build-patch.R [tag]") +} +package <- args[[1L]] +tag <- if (length(args) >= 2L) args[[2L]] else NULL + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +patches_dir <- file.path( + if (is.na(script_path)) "local" else dirname(script_path), + "patches" +) + +suppressPackageStartupMessages(library(bincraft, quietly = TRUE)) + +cat(sprintf( + "Trial build: %s%s with registry %s (no upload/archive/metadata)\n", + package, + if (is.null(tag)) "" else sprintf(" @ %s", tag), + patches_dir +)) + +ok <- tryCatch( + { + bincraft::build_binary_package( + package, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ) + TRUE + }, + error = function(e) { + cat(sprintf("Trial build FAILED: %s\n", conditionMessage(e))) + FALSE + } +) + +if (ok) { + cat(sprintf("Trial build OK: %s builds with the proposed patch.\n", package)) + q(status = 0) +} +q(status = 1) diff --git a/local/validate-patches.R b/local/validate-patches.R index 5f5aace..b20aea1 100644 --- a/local/validate-patches.R +++ b/local/validate-patches.R @@ -1,9 +1,17 @@ #!/usr/bin/env Rscript # Validate local/patches/registry.json: schema, referenced patch files, and # ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. +# +# Defaults to local/patches/registry.json. To validate a candidate registry +# without touching the real one (e.g. from the patch proposer), set: +# PATCH_DIR directory patch-file paths resolve against (default local/patches) +# REGISTRY_FILE registry.json to validate (default /registry.json) -dir <- "local/patches" -registry_file <- file.path(dir, "registry.json") +dir <- Sys.getenv("PATCH_DIR", unset = "local/patches") +registry_file <- Sys.getenv( + "REGISTRY_FILE", + unset = file.path(dir, "registry.json") +) if (!file.exists(registry_file)) { cat("No registry.json found; nothing to validate.\n") quit(status = 0L) @@ -19,33 +27,47 @@ for (i in seq_along(reg)) { e <- reg[[i]] missing <- setdiff(required, names(e)) if (length(missing) > 0L) { - errs <- c(errs, sprintf( - "entry %d (%s): missing %s", i, - if (is.null(e$package)) "?" else e$package, toString(missing) - )) + errs <- c( + errs, + sprintf( + "entry %d (%s): missing %s", + i, + if (is.null(e$package)) "?" else e$package, + toString(missing) + ) + ) } if (!is.null(e$patch)) { p <- file.path(dir, e$patch) if (!file.exists(p)) { - errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing", - i, e$package, p)) + errs <- c( + errs, + sprintf("entry %d (%s): patch file '%s' missing", i, e$package, p) + ) } } } # Ambiguous overlap: two entries for the same package with identical platforms # and versions. -keys <- vapply(reg, function(e) { - sprintf( - "%s|%s|%s", - or_q(e$package), - paste(sort(as.character(unlist(e$platforms))), collapse = ","), - or_q(e$versions) - ) -}, character(1L)) +keys <- vapply( + reg, + function(e) { + sprintf( + "%s|%s|%s", + or_q(e$package), + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + or_q(e$versions) + ) + }, + character(1L) +) dups <- keys[duplicated(keys)] if (length(dups) > 0L) { - errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups)))) + errs <- c( + errs, + sprintf("ambiguous duplicate entries: %s", toString(unique(dups))) + ) } if (length(errs) > 0L) { @@ -53,4 +75,8 @@ if (length(errs) > 0L) { cat(paste0(" - ", errs, "\n")) quit(status = 1L) } -cat(sprintf("Patch registry OK (%d %s).\n", length(reg), if (length(reg) == 1L) "entry" else "entries")) +cat(sprintf( + "Patch registry OK (%d %s).\n", + length(reg), + if (length(reg) == 1L) "entry" else "entries" +)) From 1c297c01bf14af1100b6ec3bff79b13d86ec47eb Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 15:03:24 +0000 Subject: [PATCH 002/118] feat(local): auto-propose registry patches and track the feedback loop (#117) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements steps 3 + 4 of #115, building on the classifier merged in #116. Now that bincraft **v4.4.3** applies registry `patch`/`makevars`/`configure_args` to the *target* package build (previously deps-only), a trial patched build is a meaningful acceptance gate, so the "propose" half is viable. ## Step 3 — propose, do not apply - **`local/propose-patches.R`** — for each classified, safe fix affecting a package with no current registry entry, emits a pre-filled `registry.json` entry and validates the candidate set against a *temporary* merged registry (the real one is never touched unless asked). - default: print candidates + validation, **take no action** - `--write`: append entries to `registry.json` + the proposals ledger (you commit + open the PR) - `--open-issue`: post/update a Forgejo tracking issue (reuses the weekly-audit `httr2` + `FORGEJO_TOKEN` pattern) - **`local/trial-build-patch.R`** — isolated bincraft build of one package with the registry applied (no upload/archive/metadata; `patchhash` keeps it out of the real cache). Exit 0/1, so it gates a CI step or manual pre-merge check. The human gate stays: nothing merges. Acceptance = `validate-patches.R` passes (checked automatically) **and** the trial build succeeds. Novel source diffs and unknown signatures are never proposed (they carry `auto = FALSE`). ## Step 4 — feedback loop - **`local/proposal-tracking.R`** (read-only) — signature hit rate (builds/pkgs/addressed/open per signature), proposed-vs-merged (a proposal counts merged once its package is in the registry), and retirement candidates (registry entries whose package no longer fails, i.e. likely fixed upstream). - **`local/proposal-tracking-lib.R`** — the pure metric/ledger helpers. ## Supporting changes - Refactored the classify helpers to expose a pure `build_triage_report()` + a list-returning entry builder; `failing-builds-report.R` now renders from the shared function (no behaviour change). - `validate-patches.R` gains optional `PATCH_DIR`/`REGISTRY_FILE` overrides (backward-compatible) so a candidate registry can be validated in isolation. - Documented the propose/trial-build/tracking workflow in `local/patches/README.md`. ## Verification - 71 unit tests pass (incl. new `test-proposal-tracking-lib.R`) under the Dockerized R 4.5.3 build env. - All pre-commit hooks pass (`air-format`, `validate-patches`, prettier, etc.). - Smoke-tested all three entrypoints end-to-end with a stubbed DB: dry-run, `--write` (produces a registry that passes the canonical validator + a valid ledger, then reverted), and the tracker. Closes #115 Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/117 --- local/failing-builds-classify.R | 116 +++++++- local/failing-builds-report.R | 127 ++------ local/patches/README.md | 34 +++ local/proposal-tracking-lib.R | 103 +++++++ local/proposal-tracking.R | 135 +++++++++ local/propose-patches.R | 362 +++++++++++++++++++++++ local/tests/test-proposal-tracking-lib.R | 105 +++++++ local/trial-build-patch.R | 65 ++++ local/validate-patches.R | 62 ++-- 9 files changed, 990 insertions(+), 119 deletions(-) create mode 100644 local/proposal-tracking-lib.R create mode 100644 local/proposal-tracking.R create mode 100644 local/propose-patches.R create mode 100644 local/tests/test-proposal-tracking-lib.R create mode 100644 local/trial-build-patch.R diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index b6f3430..30c168d 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -175,10 +175,10 @@ classify_error <- function(error_text, signatures = build_signatures()) { ) } -# Render a suggested registry.json entry (as a pretty JSON string) for a -# classified group, filling package/versions/platforms from the observed -# failures. Only meaningful when the signature carries a `registry` template. -propose_registry_entry <- function( +# Build a suggested registry.json entry (as an R list) for a classified group, +# filling package/versions/platforms from the observed failures. Only +# meaningful when the signature carries a `registry` template. +propose_registry_entry_list <- function( signature, package, platforms, @@ -192,7 +192,7 @@ propose_registry_entry <- function( package = package, versions = versions, # I() keeps this a JSON array even when a single platform is affected. - platforms = I(as.character(platforms)) + platforms = I(sort(unique(as.character(platforms)))) ) for (k in c("env", "configure_args", "makevars", "patch")) { if (!is.null(tmpl[[k]])) { @@ -200,5 +200,111 @@ propose_registry_entry <- function( } } entry$reason <- tmpl$reason + entry +} + +# Same, rendered as a pretty JSON string. +propose_registry_entry <- function( + signature, + package, + platforms, + versions = "*" +) { + entry <- propose_registry_entry_list(signature, package, platforms, versions) + if (is.null(entry)) { + return(NULL) + } jsonlite::toJSON(entry, auto_unbox = TRUE, pretty = TRUE, null = "null") } + +# Resolve the representative signature for a group by its id, re-attaching the +# `matched` flag the report/proposer rely on. Unknown ids -> unclassified. +resolve_signature <- function(sig_id, signatures = build_signatures()) { + hit <- Filter(function(s) s$id == sig_id, signatures) + if (length(hit) > 0L) { + s <- hit[[1L]] + s$matched <- TRUE + s + } else { + classify_error("", signatures) # unclassified fallback (matched = FALSE) + } +} + +# --------------------------------------------------------------------------- +# Triage report (shared by the report printer and the patch proposer) +# --------------------------------------------------------------------------- +# `failures` is a data.frame with at least: name, platform, arch, error_text. +# `registered_pkgs` is the set of packages that already carry a registry entry. +# Returns a list of group records (one per root-cause bucket), ordered by +# number of affected builds descending. Pure: no IO, no printing. +build_triage_report <- function( + failures, + registered_pkgs = character(0L), + signatures = build_signatures() +) { + if (nrow(failures) == 0L) { + return(list()) + } + idx <- seq_len(nrow(failures)) + failures$fingerprint <- vapply( + idx, + function(i) fingerprint_error(failures$error_text[[i]], failures$name[[i]]), + character(1L) + ) + failures$sig_id <- vapply( + idx, + function(i) classify_error(failures$error_text[[i]], signatures)$id, + character(1L) + ) + # Group by root cause: classified failures collapse by signature id (same fix + # candidate is one bucket regardless of log noise); unclassified failures fall + # back to the fingerprint so distinct unknowns stay separate for discovery. + failures$group_key <- ifelse( + failures$sig_id == "unclassified", + failures$fingerprint, + failures$sig_id + ) + groups <- split(failures, failures$group_key) + groups <- groups[order(-vapply(groups, nrow, integer(1L)))] + + lapply(unname(groups), function(g) { + sig_id <- names(sort(table(g$sig_id), decreasing = TRUE))[[1L]] + sig <- resolve_signature(sig_id, signatures) + + fp_tab <- sort(table(g$fingerprint), decreasing = TRUE) + pkgs <- sort(unique(g$name)) + plats <- sort(unique(g$platform)) + arches <- sort(unique(g$arch)) + unregistered <- setdiff(pkgs, registered_pkgs) + auto_proposable <- isTRUE(sig$auto) && sig$matched + + proposed <- list() + if (auto_proposable) { + for (p in unregistered) { + proposed[[p]] <- propose_registry_entry_list( + sig, + p, + g$platform[g$name == p] + ) + } + } + + list( + signature = sig$id, + label = sig$label, + matched = sig$matched, + auto_proposable = auto_proposable, + tier = sig$tier, + confidence = sig$confidence, + suggested_fix = sig$fix, + fingerprint = names(fp_tab)[[1L]], + fingerprint_variants = length(fp_tab), + build_count = nrow(g), + packages = pkgs, + packages_without_entry = unregistered, + platforms = plats, + arches = arches, + proposed_entries = if (length(proposed) > 0L) proposed else NULL + ) + }) +} diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R index 855f042..60f65ea 100644 --- a/local/failing-builds-report.R +++ b/local/failing-builds-report.R @@ -127,99 +127,53 @@ if (file.exists(registry_file)) { } # --------------------------------------------------------------------------- -# Fingerprint + classify every failure, then group +# Classify + group (shared logic), then render # --------------------------------------------------------------------------- -signatures <- build_signatures() -failures$fingerprint <- vapply( - seq_len(nrow(failures)), - function(i) fingerprint_error(failures$error_text[[i]], failures$name[[i]]), - character(1L) -) -failures$sig_id <- vapply( - seq_len(nrow(failures)), - function(i) classify_error(failures$error_text[[i]], signatures)$id, - character(1L) -) +report <- build_triage_report(failures, registered_pkgs) +report <- Filter(function(r) r$build_count >= min_count, report) -# Group by root cause: classified failures collapse by signature id (so the -# same fix candidate is one bucket regardless of surrounding log noise); -# unclassified failures fall back to the fingerprint so distinct unknowns stay -# separate for discovery. -failures$group_key <- ifelse( - failures$sig_id == "unclassified", - failures$fingerprint, - failures$sig_id -) -groups <- split(failures, failures$group_key) -# Order groups by number of affected builds, descending. -groups <- groups[order(-vapply(groups, nrow, integer(1L)))] - -report <- list() cat(sprintf( - "\n%d distinct failure fingerprint(s); showing groups with >= %d build(s).\n", - length(groups), + "\n%d distinct failure group(s); showing groups with >= %d build(s).\n", + length(report), min_count )) cat(strrep("=", 78L), "\n", sep = "") -for (g in groups) { - n <- nrow(g) - if (n < min_count) { - next - } - - # Representative classification: most common signature id in the group. - sig_id <- names(sort(table(g$sig_id), decreasing = TRUE))[[1L]] - sig <- Filter(function(s) s$id == sig_id, signatures) - if (length(sig) > 0L) { - sig <- sig[[1L]] - sig$matched <- TRUE - } else { - sig <- classify_error("", signatures) # unclassified fallback (matched = FALSE) - } - - fp_tab <- sort(table(g$fingerprint), decreasing = TRUE) - rep_fp <- names(fp_tab)[[1L]] - fp_variants <- length(fp_tab) - pkgs <- sort(unique(g$name)) - plats <- sort(unique(g$platform)) - arches <- sort(unique(g$arch)) - unregistered <- setdiff(pkgs, registered_pkgs) - - status <- if (!sig$matched) { +for (r in report) { + status <- if (!r$matched) { "HUMAN TRIAGE (unknown signature)" - } else if (!sig$auto) { - sprintf("HUMAN TRIAGE (classified: %s; novel source diff)", sig$id) + } else if (!r$auto_proposable) { + sprintf("HUMAN TRIAGE (classified: %s; novel source diff)", r$signature) } else { - sprintf("AUTO-PROPOSABLE (%s, %s confidence)", sig$id, sig$confidence) + sprintf("AUTO-PROPOSABLE (%s, %s confidence)", r$signature, r$confidence) } cat(sprintf( "\n[%d builds | %d pkgs | %s] %s\n", - n, - length(pkgs), - toString(plats), + r$build_count, + length(r$packages), + toString(r$platforms), status )) cat(sprintf( " fingerprint: %s%s\n", - rep_fp, - if (fp_variants > 1L) { - sprintf(" (+%d fingerprint variant(s))", fp_variants - 1L) + r$fingerprint, + if (r$fingerprint_variants > 1L) { + sprintf(" (+%d fingerprint variant(s))", r$fingerprint_variants - 1L) } else { "" } )) - if (sig$matched) { - cat(sprintf(" signature : %s\n", sig$label)) - cat(sprintf(" suggested : [%s] %s\n", sig$tier, sig$fix)) + if (r$matched) { + cat(sprintf(" signature : %s\n", r$label)) + cat(sprintf(" suggested : [%s] %s\n", r$tier, r$suggested_fix)) } - cat(sprintf(" arch : %s\n", toString(arches))) + cat(sprintf(" arch : %s\n", toString(r$arches))) cat(sprintf( " packages : %s\n", paste( vapply( - pkgs, + r$packages, function(p) { if (p %in% registered_pkgs) paste0(p, " (has entry)") else p }, @@ -229,39 +183,20 @@ for (g in groups) { ) )) - proposals <- list() - if (sig$matched && sig$auto && length(unregistered) > 0L) { + if (!is.null(r$proposed_entries)) { cat(" proposed registry entries (validate + trial-build before merge):\n") - for (p in unregistered) { - p_plats <- sort(unique(g$platform[g$name == p])) - entry <- propose_registry_entry(sig, p, p_plats) - proposals[[p]] <- entry - cat(paste0(" ", gsub("\n", "\n ", entry)), "\n", sep = "") + for (entry in r$proposed_entries) { + j <- jsonlite::toJSON( + entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "") } - } else if (sig$matched && sig$auto && length(unregistered) == 0L) { + } else if (r$auto_proposable) { cat(" (all affected packages already have a registry entry)\n") } - - report[[length(report) + 1L]] <- list( - fingerprint = rep_fp, - fingerprint_variants = fp_variants, - build_count = n, - packages = pkgs, - packages_without_entry = unregistered, - platforms = plats, - arches = arches, - signature = sig$id, - matched = sig$matched, - auto_proposable = isTRUE(sig$auto) && sig$matched, - tier = sig$tier, - confidence = sig$confidence, - suggested_fix = sig$fix, - proposed_entries = if (length(proposals) > 0L) { - lapply(proposals, jsonlite::fromJSON) - } else { - NULL - } - ) } # --------------------------------------------------------------------------- diff --git a/local/patches/README.md b/local/patches/README.md index b57cc1b..7dd769c 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -59,3 +59,37 @@ For auto-proposable groups it prints a ready-to-review registry entry; still run Novel source diffs and unknown signatures stay human-reviewed by design. Add a new signature by appending a rule to `build_signatures()` in `local/failing-builds-classify.R`; the pure helpers are covered by `local/tests/test-failing-builds-classify.R`. + +### Proposing entries (step 3: propose, do not apply) + +`local/propose-patches.R` takes the auto-proposable candidates one step further: for each classified, safe fix affecting a package with no current entry, it emits a pre-filled `registry.json` entry and validates the candidate set against a temporary merged registry (the real registry is never touched unless you ask). +The human gate stays: it never merges. + +```bash +# Default: print candidates + validation, take no action. +PGPASS=... Rscript local/propose-patches.R +# Append the candidates to registry.json + the proposals ledger (you commit + open the PR). +PGPASS=... Rscript local/propose-patches.R --write +# Or post/update a Forgejo tracking issue instead (needs FORGEJO_TOKEN). +PGPASS=... FORGEJO_TOKEN=... Rscript local/propose-patches.R --open-issue +``` + +The acceptance criteria before merging a proposal are: `validate-patches.R` passes (checked automatically), and an isolated trial build succeeds. +Run the trial build inside the failing platform's build-env image; it uploads/archives nothing and writes no metadata: + +```bash +Rscript local/trial-build-patch.R +``` + +`--write` and `--open-issue` also append to `local/patches/proposals-log.json`, a ledger of what was proposed. + +### Feedback loop (step 4) + +`local/proposal-tracking.R` reports the signature hit rate, proposed-vs-merged status (a proposal counts as merged once its package appears in the registry), and retirement candidates (registry entries whose package no longer appears in any current failure, so the upstream cause was likely fixed). +It is read-only. + +```bash +PGPASS=... Rscript local/proposal-tracking.R --json metrics.json +``` + +The pure metric/ledger helpers live in `local/proposal-tracking-lib.R` and are covered by `local/tests/test-proposal-tracking-lib.R`. diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R new file mode 100644 index 0000000..4e8a468 --- /dev/null +++ b/local/proposal-tracking-lib.R @@ -0,0 +1,103 @@ +# Pure, IO-free helpers for the proposal feedback loop (issue #115, step 4): +# a small ledger of what the classifier proposed, plus metrics derived from the +# live triage report and the registry (signature hit rate, proposed-vs-merged, +# retirement candidates). +# +# Kept free of DB/HTTP/clock so it can be sourced by the proposer, the tracker +# entrypoint, and the unit tests. Timestamps are passed in by callers. + +# Stable identity of a ledger record: one proposal per (package, signature). +ledger_key <- function(record) { + paste0( + if (is.null(record$package)) "?" else record$package, + "|", + if (is.null(record$signature)) "?" else record$signature + ) +} + +# Merge freshly-generated proposals into an existing ledger without clobbering +# history: a record whose (package, signature) already exists is left as-is +# (its status/PR/issue are preserved); genuinely new proposals are appended. +# Returns the combined list. Pure: callers stamp `proposed_at` before passing. +merge_ledger <- function(existing, new_records) { + if (is.null(existing)) { + existing <- list() + } + seen <- vapply(existing, ledger_key, character(1L)) + out <- existing + for (rec in new_records) { + if (!(ledger_key(rec) %in% seen)) { + out[[length(out) + 1L]] <- rec + seen <- c(seen, ledger_key(rec)) + } + } + out +} + +# Per-signature hit rate from a triage report (list of group records from +# build_triage_report) crossed with the set of already-registered packages. +# For each *matched* signature: how many failing builds/packages it explains, +# and how many of those packages are already addressed by a registry entry. +signature_hit_rate <- function(report, registered_pkgs = character(0L)) { + matched <- Filter(function(g) isTRUE(g$matched), report) + by_sig <- split( + matched, + vapply(matched, function(g) g$signature, character(1L)) + ) + lapply(names(by_sig), function(sig) { + grps <- by_sig[[sig]] + pkgs <- unique(unlist(lapply(grps, function(g) g$packages))) + addressed <- intersect(pkgs, registered_pkgs) + list( + signature = sig, + builds = sum(vapply(grps, function(g) g$build_count, integer(1L))), + packages = length(pkgs), + addressed = length(addressed), + open = length(setdiff(pkgs, registered_pkgs)), + auto_proposable = any(vapply( + grps, + function(g) isTRUE(g$auto_proposable), + logical(1L) + )) + ) + }) +} + +# Proposed-vs-merged: a ledger record counts as "merged" once its package +# appears in the registry. Returns per-record status plus a rollup. +proposed_vs_merged <- function(ledger, registered_pkgs = character(0L)) { + if (is.null(ledger)) { + ledger <- list() + } + rows <- lapply(ledger, function(rec) { + merged <- !is.null(rec$package) && rec$package %in% registered_pkgs + list( + package = rec$package, + signature = rec$signature, + status = if (merged) "merged" else (rec$status %||% "proposed"), + merged = merged + ) + }) + list( + records = rows, + total = length(rows), + merged = sum(vapply(rows, function(r) isTRUE(r$merged), logical(1L))) + ) +} + +# Registry entries whose package no longer appears in any current failure are +# retirement candidates: the upstream cause was likely fixed, so the entry can +# be reviewed for removal. `failing_pkgs` is the set of currently-failing +# package names (from the live report). +retirement_candidates <- function(registry_entries, failing_pkgs) { + if (is.null(registry_entries)) { + registry_entries <- list() + } + keep <- Filter( + function(e) !is.null(e$package) && !(e$package %in% failing_pkgs), + registry_entries + ) + vapply(keep, function(e) as.character(e$package), character(1L)) +} + +`%||%` <- function(a, b) if (is.null(a)) b else a diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R new file mode 100644 index 0000000..60bbd10 --- /dev/null +++ b/local/proposal-tracking.R @@ -0,0 +1,135 @@ +#!/usr/bin/env Rscript + +# Feedback loop for the failure classifier (issue #115, step 4): report the +# signature hit rate, proposed-vs-merged status, and registry entries that look +# retirable, so the rule set can improve and stale entries can be pruned. +# Read-only: queries `single_builds` and reads the registry + proposals ledger. +# +# Usage: +# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(DBI, quietly = TRUE) + library(RPostgres, quietly = TRUE) + library(jsonlite, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "failing-builds-classify.R")) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +args <- commandArgs(trailingOnly = TRUE) +json_out <- local({ + i <- match("--json", args) + if (!is.na(i) && i < length(args)) args[[i + 1L]] else NA_character_ +}) + +if (nchar(Sys.getenv("PGPASS")) == 0L) { + stop("PGPASS env var is not set; a DB password is required.") +} + +registry_file <- file.path(script_dir, "patches", "registry.json") +ledger_file <- file.path(script_dir, "patches", "proposals-log.json") + +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(DBI::dbDisconnect(con), add = TRUE) + +failures <- DBI::dbGetQuery( + con, + paste0( + "SELECT name, tag, platform, arch, r_version, timestamp, error_text ", + "FROM single_builds WHERE error_occurred = TRUE AND removed = FALSE" + ) +) + +existing_entries <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +registered_pkgs <- unique(vapply( + existing_entries, + function(e) as.character(e$package %||% ""), + character(1L) +)) +ledger <- if (file.exists(ledger_file)) { + jsonlite::fromJSON(ledger_file, simplifyVector = FALSE) +} else { + list() +} + +report <- build_triage_report(failures, registered_pkgs) +failing_pkgs <- unique(unlist(lapply(report, function(g) g$packages))) + +hit <- signature_hit_rate(report, registered_pkgs) +pvm <- proposed_vs_merged(ledger, registered_pkgs) +retire <- retirement_candidates(existing_entries, failing_pkgs) + +cat(sprintf( + "Feedback loop over %d failing builds, %d registry entries, %d ledger records.\n", + nrow(failures), + length(existing_entries), + length(ledger) +)) + +cat("\nSignature hit rate:\n") +cat(strrep("-", 60L), "\n", sep = "") +for (h in hit) { + cat(sprintf( + " %-26s %3d builds | %2d pkgs | %2d addressed | %2d open%s\n", + h$signature, + h$builds, + h$packages, + h$addressed, + h$open, + if (isTRUE(h$auto_proposable)) " [auto]" else "" + )) +} + +cat(sprintf("\nProposed vs merged: %d / %d merged.\n", pvm$merged, pvm$total)) +for (r in pvm$records) { + cat(sprintf(" %-24s %-26s %s\n", r$package, r$signature, r$status)) +} + +cat(sprintf( + "\nRetirement candidates (%d): registry entries with no current failure.\n", + length(retire) +)) +if (length(retire) > 0L) { + cat(paste0(" - ", retire, "\n"), sep = "") +} else { + cat(" (none)\n") +} + +if (!is.na(json_out)) { + jsonlite::write_json( + list( + signature_hit_rate = hit, + proposed_vs_merged = pvm, + retirement_candidates = retire + ), + json_out, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf("\nWrote metrics to %s\n", json_out)) +} diff --git a/local/propose-patches.R b/local/propose-patches.R new file mode 100644 index 0000000..20583ba --- /dev/null +++ b/local/propose-patches.R @@ -0,0 +1,362 @@ +#!/usr/bin/env Rscript + +# Propose registry patches for classified, auto-proposable build failures +# (issue #115, step 3: "propose, do not apply"). Queries `single_builds`, +# classifies failures, and for each *known, safe* fix lever affecting a package +# with no current registry entry, emits a pre-filled `registry.json` entry. +# +# The human gate stays: this never merges. Acceptance before merge is +# 1. `local/validate-patches.R` passes (checked here against a candidate +# registry), and +# 2. an isolated trial build succeeds -- run `local/trial-build-patch.R ` +# (this script prints the exact command per candidate). +# Novel source diffs and unknown signatures are never proposed (they carry +# `auto = FALSE` in the signature table). +# +# Usage: +# PGPASS=... Rscript local/propose-patches.R [--platform P] [--arch A] [--min N] +# (default) print candidates + validation, take no action +# --write append candidates to local/patches/registry.json and +# the proposals ledger (commit + open a PR yourself) +# --open-issue post/update a Forgejo tracking issue (needs FORGEJO_TOKEN) +# --json PATH also write the machine-readable candidate list to PATH + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(DBI, quietly = TRUE) + library(RPostgres, quietly = TRUE) + library(jsonlite, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "failing-builds-classify.R")) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +# --------------------------------------------------------------------------- +# Arguments +# --------------------------------------------------------------------------- +args <- commandArgs(trailingOnly = TRUE) +opt_val <- function(flag, default = NA_character_) { + i <- match(flag, args) + if (!is.na(i) && i < length(args)) args[[i + 1L]] else default +} +platform <- opt_val("--platform", Sys.getenv("PLATFORM", "")) +arch <- opt_val("--arch", Sys.getenv("ARCH", "")) +json_out <- opt_val("--json") +min_count <- suppressWarnings(as.integer(opt_val("--min", "1"))) +if (is.na(min_count)) { + min_count <- 1L +} +do_write <- "--write" %in% args +do_issue <- "--open-issue" %in% args + +if (nchar(Sys.getenv("PGPASS")) == 0L) { + stop("PGPASS env var is not set; a DB password is required.") +} + +registry_file <- file.path(script_dir, "patches", "registry.json") +ledger_file <- file.path(script_dir, "patches", "proposals-log.json") + +# --------------------------------------------------------------------------- +# Query failing builds (same shape as failing-builds-report.R) +# --------------------------------------------------------------------------- +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(DBI::dbDisconnect(con), add = TRUE) + +where <- "error_occurred = TRUE AND removed = FALSE" +params <- list() +if (nzchar(platform)) { + where <- paste0(where, " AND platform = $", length(params) + 1L) + params <- c(params, platform) +} +if (nzchar(arch)) { + where <- paste0(where, " AND arch = $", length(params) + 1L) + params <- c(params, arch) +} +failures <- DBI::dbGetQuery( + con, + paste0( + "SELECT name, tag, platform, arch, r_version, timestamp, error_text ", + "FROM single_builds WHERE ", + where + ), + params = if (length(params) > 0L) params else NULL +) +cat(sprintf("Queried single_builds: %d failing builds\n", nrow(failures))) + +# --------------------------------------------------------------------------- +# Existing registry + classification +# --------------------------------------------------------------------------- +existing_entries <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +registered_pkgs <- unique(vapply( + existing_entries, + function(e) as.character(e$package %||% ""), + character(1L) +)) + +report <- build_triage_report(failures, registered_pkgs) +report <- Filter(function(r) r$build_count >= min_count, report) + +# Flatten auto-proposable groups into candidate records. +candidates <- list() +for (r in report) { + if (is.null(r$proposed_entries)) { + next + } + for (pkg in names(r$proposed_entries)) { + candidates[[length(candidates) + 1L]] <- list( + package = pkg, + signature = r$signature, + confidence = r$confidence, + tier = r$tier, + entry = r$proposed_entries[[pkg]] + ) + } +} + +if (length(candidates) == 0L) { + cat( + "No auto-proposable candidates (nothing classified, safe, and unregistered).\n" + ) + q(status = 0) +} + +cat(sprintf( + "\n%d candidate registry %s:\n", + length(candidates), + if (length(candidates) == 1L) "entry" else "entries" +)) +cat(strrep("=", 78L), "\n", sep = "") +for (c in candidates) { + j <- jsonlite::toJSON( + c$entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf( + "\n# %s [signature: %s, %s confidence]\n", + c$package, + c$signature, + c$confidence + )) + cat(j, "\n", sep = "") + cat(sprintf( + " acceptance gate: PGPASS=... Rscript local/trial-build-patch.R %s\n", + c$package + )) +} + +# --------------------------------------------------------------------------- +# Validate the candidate set against a temporary merged registry (never +# touches the real registry unless --write succeeds). +# --------------------------------------------------------------------------- +rscript <- file.path(R.home("bin"), "Rscript") +candidate_entries <- lapply(candidates, function(c) c$entry) +merged <- c(existing_entries, candidate_entries) + +validate_registry <- function(entries) { + tmp <- tempfile(fileext = ".json") + on.exit(unlink(tmp), add = TRUE) + jsonlite::write_json( + entries, + tmp, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + status <- system2( + rscript, + file.path(script_dir, "validate-patches.R"), + env = c( + paste0("PATCH_DIR=", file.path(script_dir, "patches")), + paste0("REGISTRY_FILE=", tmp) + ), + stdout = TRUE, + stderr = TRUE + ) + cat(paste0(" ", status, "\n"), sep = "") + identical(attr(status, "status"), NULL) # NULL status attr == exit 0 +} + +cat("\n", strrep("=", 78L), "\n", sep = "") +cat("Validating candidate registry:\n") +ok <- validate_registry(merged) +if (!ok) { + stop("Candidate registry failed validation; not writing or proposing.") +} + +if (!is.na(json_out)) { + jsonlite::write_json( + candidates, + json_out, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf("Wrote candidate list to %s\n", json_out)) +} + +# --------------------------------------------------------------------------- +# Actions (default: none) +# --------------------------------------------------------------------------- +now <- format(Sys.time(), "%Y-%m-%d %H:%M:%S") +new_ledger_records <- lapply(candidates, function(c) { + list( + package = c$package, + signature = c$signature, + tier = c$tier, + confidence = c$confidence, + versions = c$entry$versions, + platforms = as.character(c$entry$platforms), + proposed_at = now, + status = "proposed" + ) +}) +load_ledger <- function() { + if (file.exists(ledger_file)) { + jsonlite::fromJSON(ledger_file, simplifyVector = FALSE) + } else { + list() + } +} +save_ledger <- function(led) { + jsonlite::write_json( + led, + ledger_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) +} + +if (do_write) { + jsonlite::write_json( + merged, + registry_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) + cat(sprintf( + "\nWrote %d entries to %s and updated %s.\n", + length(candidate_entries), + registry_file, + ledger_file + )) + cat( + "Next: run the acceptance-gate trial build(s), then commit and open a PR.\n" + ) +} else if (do_issue) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-issue requires FORGEJO_TOKEN.") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + issue_title <- "Auto-proposed registry patches (classifier #115)" + body_lines <- c( + sprintf("_Generated %s from `single_builds` failures._", now), + "", + "The failure classifier proposes these registry entries for packages with no current entry.", + "Each is a known, safe fix lever; **review, trial-build, and open a PR** -- nothing is applied automatically.", + "" + ) + for (c in candidates) { + j <- jsonlite::toJSON( + c$entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + body_lines <- c( + body_lines, + sprintf( + "### %s (%s, %s confidence)", + c$package, + c$signature, + c$confidence + ), + "", + "```json", + as.character(j), + "```", + sprintf( + "Acceptance gate: `PGPASS=... Rscript local/trial-build-patch.R %s`", + c$package + ), + "" + ) + } + new_body <- paste(body_lines, collapse = "\n") + + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(issue_title, reserved = TRUE) + ) + existing <- httr2::request(search_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + existing, + function(x) identical(x$title, issue_title), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- existing[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("Updated tracking issue #%d.\n", num)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/issues", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = issue_title, body = new_body)) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf("Opened tracking issue #%d.\n", created$number)) + } + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) +} else { + cat( + "\nDry run: no changes made. Re-run with --write or --open-issue to act.\n" + ) +} diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R new file mode 100644 index 0000000..55da4b6 --- /dev/null +++ b/local/tests/test-proposal-tracking-lib.R @@ -0,0 +1,105 @@ +source(file.path("..", "proposal-tracking-lib.R")) +source(file.path("..", "failing-builds-classify.R")) + +mk_failures <- function() { + data.frame( + name = c("StanHeaders", "rstan", "RcppParallel", "somepkg"), + platform = c("alpine-321", "alpine-321", "alpine-320", "redhat-9"), + arch = c("amd64", "arm64", "amd64", "amd64"), + error_text = c( + "fatal error: tbb/tbb_stddef.h: No such file or directory", + "In file: tbb/tbb_stddef.h: No such file or directory", + "Error: USE_TBB=Linux is not supported; bundled TBB on musl", + "some unmatched failure" + ), + stringsAsFactors = FALSE + ) +} + +test_that("merge_ledger appends new proposals and preserves existing history", { + existing <- list(list( + package = "fs", + signature = "system-libuv-link-leak", + status = "merged" + )) + new <- list( + list( + package = "fs", + signature = "system-libuv-link-leak", + status = "proposed" + ), + list( + package = "StanHeaders", + signature = "tbb-stddef-removed", + status = "proposed" + ) + ) + merged <- merge_ledger(existing, new) + expect_length(merged, 2L) # fs is deduped, StanHeaders added + fs <- Filter(function(r) r$package == "fs", merged)[[1L]] + expect_identical(fs$status, "merged") # existing status preserved, not clobbered +}) + +test_that("merge_ledger handles an empty/NULL starting ledger", { + new <- list(list(package = "x", signature = "s")) + expect_length(merge_ledger(NULL, new), 1L) + expect_length(merge_ledger(list(), new), 1L) +}) + +test_that("signature_hit_rate splits addressed vs open per signature", { + report <- build_triage_report(mk_failures(), registered_pkgs = "RcppParallel") + hit <- signature_hit_rate(report, registered_pkgs = "RcppParallel") + tbb <- Filter(function(h) h$signature == "tbb-stddef-removed", hit)[[1L]] + expect_identical(tbb$packages, 2L) # StanHeaders + rstan + expect_identical(tbb$addressed, 0L) + expect_identical(tbb$open, 2L) + expect_true(tbb$auto_proposable) + + rcpp <- Filter(function(h) h$signature == "rcppparallel-bundled-tbb", hit)[[ + 1L + ]] + expect_identical(rcpp$addressed, 1L) # already registered + expect_identical(rcpp$open, 0L) + + # Unclassified failures never appear as a signature. + expect_false( + "unclassified" %in% vapply(hit, function(h) h$signature, character(1L)) + ) +}) + +test_that("proposed_vs_merged marks a package merged once it is registered", { + ledger <- list( + list( + package = "StanHeaders", + signature = "tbb-stddef-removed", + status = "proposed" + ), + list( + package = "rstan", + signature = "tbb-stddef-removed", + status = "proposed" + ) + ) + pvm <- proposed_vs_merged(ledger, registered_pkgs = "StanHeaders") + expect_identical(pvm$total, 2L) + expect_identical(pvm$merged, 1L) + stan <- Filter(function(r) r$package == "StanHeaders", pvm$records)[[1L]] + expect_identical(stan$status, "merged") +}) + +test_that("retirement_candidates flags entries whose package no longer fails", { + entries <- list( + list(package = "RcppParallel"), + list(package = "oldpkg") + ) + # RcppParallel still fails; oldpkg does not -> only oldpkg is retirable. + out <- retirement_candidates( + entries, + failing_pkgs = c("RcppParallel", "StanHeaders") + ) + expect_identical(out, "oldpkg") + expect_length( + retirement_candidates(entries, failing_pkgs = c("RcppParallel", "oldpkg")), + 0L + ) +}) diff --git a/local/trial-build-patch.R b/local/trial-build-patch.R new file mode 100644 index 0000000..4e8030a --- /dev/null +++ b/local/trial-build-patch.R @@ -0,0 +1,65 @@ +#!/usr/bin/env Rscript + +# Acceptance gate for a proposed registry patch (issue #115, step 3): build one +# package in isolation with the current `local/patches` registry applied, and +# report whether it succeeds. Nothing is uploaded, archived, or written to the +# metadata DB -- `patchhash` keeps the trial binary out of the real cache. +# +# Must run inside a build-env image (the same image the failing platform uses), +# because it invokes the real compiler toolchain via bincraft. +# +# Usage: +# Rscript local/trial-build-patch.R [tag] +# +# Exit status: 0 if the patched build succeeds, 1 otherwise -- so it can gate a +# CI step or a manual pre-merge check. + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 1L) { + stop("usage: Rscript local/trial-build-patch.R [tag]") +} +package <- args[[1L]] +tag <- if (length(args) >= 2L) args[[2L]] else NULL + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +patches_dir <- file.path( + if (is.na(script_path)) "local" else dirname(script_path), + "patches" +) + +suppressPackageStartupMessages(library(bincraft, quietly = TRUE)) + +cat(sprintf( + "Trial build: %s%s with registry %s (no upload/archive/metadata)\n", + package, + if (is.null(tag)) "" else sprintf(" @ %s", tag), + patches_dir +)) + +ok <- tryCatch( + { + bincraft::build_binary_package( + package, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ) + TRUE + }, + error = function(e) { + cat(sprintf("Trial build FAILED: %s\n", conditionMessage(e))) + FALSE + } +) + +if (ok) { + cat(sprintf("Trial build OK: %s builds with the proposed patch.\n", package)) + q(status = 0) +} +q(status = 1) diff --git a/local/validate-patches.R b/local/validate-patches.R index 5f5aace..b20aea1 100644 --- a/local/validate-patches.R +++ b/local/validate-patches.R @@ -1,9 +1,17 @@ #!/usr/bin/env Rscript # Validate local/patches/registry.json: schema, referenced patch files, and # ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. +# +# Defaults to local/patches/registry.json. To validate a candidate registry +# without touching the real one (e.g. from the patch proposer), set: +# PATCH_DIR directory patch-file paths resolve against (default local/patches) +# REGISTRY_FILE registry.json to validate (default /registry.json) -dir <- "local/patches" -registry_file <- file.path(dir, "registry.json") +dir <- Sys.getenv("PATCH_DIR", unset = "local/patches") +registry_file <- Sys.getenv( + "REGISTRY_FILE", + unset = file.path(dir, "registry.json") +) if (!file.exists(registry_file)) { cat("No registry.json found; nothing to validate.\n") quit(status = 0L) @@ -19,33 +27,47 @@ for (i in seq_along(reg)) { e <- reg[[i]] missing <- setdiff(required, names(e)) if (length(missing) > 0L) { - errs <- c(errs, sprintf( - "entry %d (%s): missing %s", i, - if (is.null(e$package)) "?" else e$package, toString(missing) - )) + errs <- c( + errs, + sprintf( + "entry %d (%s): missing %s", + i, + if (is.null(e$package)) "?" else e$package, + toString(missing) + ) + ) } if (!is.null(e$patch)) { p <- file.path(dir, e$patch) if (!file.exists(p)) { - errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing", - i, e$package, p)) + errs <- c( + errs, + sprintf("entry %d (%s): patch file '%s' missing", i, e$package, p) + ) } } } # Ambiguous overlap: two entries for the same package with identical platforms # and versions. -keys <- vapply(reg, function(e) { - sprintf( - "%s|%s|%s", - or_q(e$package), - paste(sort(as.character(unlist(e$platforms))), collapse = ","), - or_q(e$versions) - ) -}, character(1L)) +keys <- vapply( + reg, + function(e) { + sprintf( + "%s|%s|%s", + or_q(e$package), + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + or_q(e$versions) + ) + }, + character(1L) +) dups <- keys[duplicated(keys)] if (length(dups) > 0L) { - errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups)))) + errs <- c( + errs, + sprintf("ambiguous duplicate entries: %s", toString(unique(dups))) + ) } if (length(errs) > 0L) { @@ -53,4 +75,8 @@ if (length(errs) > 0L) { cat(paste0(" - ", errs, "\n")) quit(status = 1L) } -cat(sprintf("Patch registry OK (%d %s).\n", length(reg), if (length(reg) == 1L) "entry" else "entries")) +cat(sprintf( + "Patch registry OK (%d %s).\n", + length(reg), + if (length(reg) == 1L) "entry" else "entries" +)) From 1b6e2b506ebdc989c590819e0faa4e30ac4c5f3f Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 15:05:22 +0000 Subject: [PATCH 003/118] ci(crow): add weekly patch-proposal + feedback-loop pipeline Add `.crow/weekly-patch-proposals.yaml`, a single (non-matrix) job that runs the classifier over all of `single_builds` weekly: it posts/updates a Forgejo tracking issue with the auto-proposable registry entries via `propose-patches.R --open-issue`, then logs the step-4 metrics via `proposal-tracking.R`. Clones read-only; the only write is the issue. Register the `weekly-patch-proposals` cron in the crow UI, or trigger manually with `task=weekly-patch-proposals`. --- .crow/weekly-patch-proposals.yaml | 57 +++++++++++++++++++++++++++++++ local/patches/README.md | 5 +++ 2 files changed, 62 insertions(+) create mode 100644 .crow/weekly-patch-proposals.yaml diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml new file mode 100644 index 0000000..8fa45df --- /dev/null +++ b/.crow/weekly-patch-proposals.yaml @@ -0,0 +1,57 @@ +# Weekly failure-triage proposals (issue #115, steps 3 + 4). +# Classifies the recorded `single_builds` failures and: +# 1. posts/updates a Forgejo tracking issue with the auto-proposable registry +# entries (human reviews, trial-builds, and opens the PR -- nothing merges), and +# 2. prints the feedback-loop metrics (signature hit rate, proposed-vs-merged, +# retirement candidates) to the run log. +# Global across platforms (the classifier groups over all of single_builds), so +# a single job -- no matrix. Clones read-only; the only write is the Forgejo +# issue via FORGEJO_TOKEN. +# +# Register the cron in the crow UI as `weekly-patch-proposals`, or run manually: +# woodpecker-cli pipeline create --var task=weekly-patch-proposals --branch=main 7 +when: + - event: manual + evaluate: 'task == "weekly-patch-proposals"' + - event: cron + cron: weekly-patch-proposals + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Propose registry patches + track feedback loop' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + R_VERSION: 4.5.3 + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue + - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 2000m + limits: + memory: 2Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/local/patches/README.md b/local/patches/README.md index 7dd769c..e875217 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -93,3 +93,8 @@ PGPASS=... Rscript local/proposal-tracking.R --json metrics.json ``` The pure metric/ledger helpers live in `local/proposal-tracking-lib.R` and are covered by `local/tests/test-proposal-tracking-lib.R`. + +### Scheduled run + +`.crow/weekly-patch-proposals.yaml` runs both steps weekly (register the `weekly-patch-proposals` cron in the crow UI): it posts/updates a Forgejo tracking issue with the auto-proposable entries and logs the feedback-loop metrics. +It clones read-only; the only write is the tracking issue. From af481f5129f1129f3937594c2335fdb4dc81dc37 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 15:17:16 +0000 Subject: [PATCH 004/118] ci(crow): add weekly patch-proposal + feedback-loop pipeline (#118) Adds the scheduled pipeline for the failure-triage proposer/tracker landed in #117 (issue #115, steps 3 + 4). ## `.crow/weekly-patch-proposals.yaml` A single (non-matrix) job -- the classifier groups over all of `single_builds`, so unlike the per-platform `weekly-audit-missing` it runs once globally. Weekly it: 1. `propose-patches.R --open-issue` -- posts/updates a Forgejo tracking issue with the auto-proposable registry entries (human reviews, trial-builds, opens the PR; nothing merges). 2. `proposal-tracking.R` -- logs the feedback-loop metrics (signature hit rate, proposed-vs-merged, retirement candidates). Low blast radius: clones read-only (`REPO_RO_TOKEN`); the only write is the tracking issue via `FORGEJO_TOKEN`. It builds nothing (no B2 keys, no bincraft). Triggers: `cron: weekly-patch-proposals` (register the cron in the crow UI) or manual with `task=weekly-patch-proposals`, matching the `archive-missed-packages` convention. Documented under "Scheduled run" in `local/patches/README.md`. `yamllint` and the other pre-commit hooks pass. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/118 --- .crow/weekly-patch-proposals.yaml | 57 +++++++++++++++++++++++++++++++ local/patches/README.md | 5 +++ 2 files changed, 62 insertions(+) create mode 100644 .crow/weekly-patch-proposals.yaml diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml new file mode 100644 index 0000000..8fa45df --- /dev/null +++ b/.crow/weekly-patch-proposals.yaml @@ -0,0 +1,57 @@ +# Weekly failure-triage proposals (issue #115, steps 3 + 4). +# Classifies the recorded `single_builds` failures and: +# 1. posts/updates a Forgejo tracking issue with the auto-proposable registry +# entries (human reviews, trial-builds, and opens the PR -- nothing merges), and +# 2. prints the feedback-loop metrics (signature hit rate, proposed-vs-merged, +# retirement candidates) to the run log. +# Global across platforms (the classifier groups over all of single_builds), so +# a single job -- no matrix. Clones read-only; the only write is the Forgejo +# issue via FORGEJO_TOKEN. +# +# Register the cron in the crow UI as `weekly-patch-proposals`, or run manually: +# woodpecker-cli pipeline create --var task=weekly-patch-proposals --branch=main 7 +when: + - event: manual + evaluate: 'task == "weekly-patch-proposals"' + - event: cron + cron: weekly-patch-proposals + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Propose registry patches + track feedback loop' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + R_VERSION: 4.5.3 + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue + - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 2000m + limits: + memory: 2Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/local/patches/README.md b/local/patches/README.md index 7dd769c..e875217 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -93,3 +93,8 @@ PGPASS=... Rscript local/proposal-tracking.R --json metrics.json ``` The pure metric/ledger helpers live in `local/proposal-tracking-lib.R` and are covered by `local/tests/test-proposal-tracking-lib.R`. + +### Scheduled run + +`.crow/weekly-patch-proposals.yaml` runs both steps weekly (register the `weekly-patch-proposals` cron in the crow UI): it posts/updates a Forgejo tracking issue with the auto-proposable entries and logs the feedback-loop metrics. +It clones read-only; the only write is the tracking issue. From 77508f8fc5f8d5f82ffcbc7121adb60593137cf6 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 18:17:28 +0200 Subject: [PATCH 005/118] ci: clean weekly-patch-proposals trigger --- .crow/weekly-patch-proposals.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 8fa45df..f956f3f 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -7,12 +7,8 @@ # Global across platforms (the classifier groups over all of single_builds), so # a single job -- no matrix. Clones read-only; the only write is the Forgejo # issue via FORGEJO_TOKEN. -# -# Register the cron in the crow UI as `weekly-patch-proposals`, or run manually: -# woodpecker-cli pipeline create --var task=weekly-patch-proposals --branch=main 7 when: - event: manual - evaluate: 'task == "weekly-patch-proposals"' - event: cron cron: weekly-patch-proposals From f1dd661213acdefdd7ee52cadb20bd0ddad9524b Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 17:11:54 +0000 Subject: [PATCH 006/118] fix(local): route multi-signature packages to human triage in the proposer (#119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem A live `propose-patches.R --open-issue` run aborted with: ``` Patch registry validation FAILED: - ambiguous duplicate entries: hmmTMB|ubuntu-2604|*, imt|ubuntu-2604|*, refundBayes|ubuntu-2604|* Error: Candidate registry failed validation; not writing or proposing. ``` Root cause: a package whose failing builds match **more than one** auto-proposable signature (e.g. some logs hit `tbb-stddef-removed`, others `rcppparallel-bundled-tbb`) landed in two separate signature groups, so the proposer emitted two candidate entries with the same `package|platforms|versions` key. The validator's ambiguous-duplicate check then failed the whole candidate set, aborting the run — so even the unambiguous candidates (e.g. `vacalibration`) never got proposed. ## Fix Add a pure, tested `dedupe_candidates()` that: - keeps a package's candidate only when it maps to a **single** signature (collapsing an accidental repeat under the same signature), and - routes a package that maps to **multiple** signatures (conflicting fix tiers -- makevars vs source patch) to **human triage** instead of guessing between them. The proposer prints the skipped ambiguous packages and proceeds with the clean candidates, so one ambiguous package no longer blocks the rest. This matches the issue #115 guardrail: don't auto-pick when the fix is ambiguous. ## Verification - New unit tests in `test-proposal-tracking-lib.R` cover the single-signature, multi-signature (conflict), duplicate-under-one-signature, and empty cases. - Reproduced the original failure end-to-end with a stubbed DB (`hmmTMB` ambiguous + `vacalibration` clean): `hmmTMB` is now listed under "Ambiguous -> human triage", `vacalibration` is proposed, and the candidate registry validates (`Patch registry OK`). - Full suite: 78 tests pass; all pre-commit hooks pass. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/119 --- local/proposal-tracking-lib.R | 31 +++++++++++++++++++++++ local/propose-patches.R | 18 ++++++++++++- local/tests/test-proposal-tracking-lib.R | 32 ++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 4e8a468..2edd9c8 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -6,6 +6,37 @@ # Kept free of DB/HTTP/clock so it can be sourced by the proposer, the tracker # entrypoint, and the unit tests. Timestamps are passed in by callers. +# Split proposal candidates into the ones safe to emit and the ambiguous ones. +# A candidate is a list with at least `package` and `signature`. A package that +# maps to more than one distinct auto-proposable signature is genuinely +# ambiguous (two conflicting fix tiers, e.g. makevars vs source patch): emitting +# both would create colliding registry entries, so those are routed to human +# triage instead of guessed at. Returns list(keep = ..., ambiguous = ...), where +# `ambiguous` is a named list of package -> the distinct signatures seen. +dedupe_candidates <- function(candidates) { + if (length(candidates) == 0L) { + return(list(keep = list(), ambiguous = list())) + } + pkgs <- vapply(candidates, function(c) as.character(c$package), character(1L)) + by_pkg <- split(candidates, pkgs) + keep <- list() + ambiguous <- list() + for (pkg in names(by_pkg)) { + cs <- by_pkg[[pkg]] + sigs <- unique(vapply( + cs, + function(c) as.character(c$signature), + character(1L) + )) + if (length(sigs) == 1L) { + keep[[length(keep) + 1L]] <- cs[[1L]] # one signature -> take the first + } else { + ambiguous[[pkg]] <- sigs + } + } + list(keep = keep, ambiguous = ambiguous) +} + # Stable identity of a ledger record: one proposal per (package, signature). ledger_key <- function(record) { paste0( diff --git a/local/propose-patches.R b/local/propose-patches.R index 20583ba..d43668a 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -135,9 +135,25 @@ for (r in report) { } } +# A package that maps to more than one auto-proposable signature is ambiguous +# (conflicting fix tiers) and would collide on the same registry key; route it +# to human triage instead of emitting both. +split_candidates <- dedupe_candidates(candidates) +candidates <- split_candidates$keep +if (length(split_candidates$ambiguous) > 0L) { + cat("\nAmbiguous (multiple signatures) -> human triage, not proposed:\n") + for (pkg in names(split_candidates$ambiguous)) { + cat(sprintf( + " %s: %s\n", + pkg, + toString(split_candidates$ambiguous[[pkg]]) + )) + } +} + if (length(candidates) == 0L) { cat( - "No auto-proposable candidates (nothing classified, safe, and unregistered).\n" + "\nNo auto-proposable candidates (nothing classified, safe, unregistered, and unambiguous).\n" ) q(status = 0) } diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index 55da4b6..a2fbe05 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -46,6 +46,38 @@ test_that("merge_ledger handles an empty/NULL starting ledger", { expect_length(merge_ledger(list(), new), 1L) }) +test_that("dedupe_candidates keeps single-signature pkgs, routes conflicts to triage", { + candidates <- list( + list(package = "StanHeaders", signature = "tbb-stddef-removed"), + list(package = "hmmTMB", signature = "tbb-stddef-removed"), + list(package = "hmmTMB", signature = "rcppparallel-bundled-tbb") # conflict + ) + out <- dedupe_candidates(candidates) + kept <- vapply(out$keep, function(c) c$package, character(1L)) + expect_identical(sort(kept), "StanHeaders") # hmmTMB dropped as ambiguous + expect_true("hmmTMB" %in% names(out$ambiguous)) + expect_setequal( + out$ambiguous$hmmTMB, + c("tbb-stddef-removed", "rcppparallel-bundled-tbb") + ) +}) + +test_that("dedupe_candidates collapses a package repeated under one signature", { + candidates <- list( + list(package = "rstan", signature = "tbb-stddef-removed"), + list(package = "rstan", signature = "tbb-stddef-removed") + ) + out <- dedupe_candidates(candidates) + expect_length(out$keep, 1L) + expect_length(out$ambiguous, 0L) +}) + +test_that("dedupe_candidates handles the empty list", { + out <- dedupe_candidates(list()) + expect_length(out$keep, 0L) + expect_length(out$ambiguous, 0L) +}) + test_that("signature_hit_rate splits addressed vs open per signature", { report <- build_triage_report(mk_failures(), registered_pkgs = "RcppParallel") hit <- signature_hit_rate(report, registered_pkgs = "RcppParallel") From 9bc49731591c5d506ecf894364f61e3853723742 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 18:40:41 +0000 Subject: [PATCH 007/118] fix(local): stop proposing per-dependent patches for dependency-cascade failures (#121) ## Problem The scheduled `--open-issue` run (issue #120) posted **800+** proposed entries, all `rcppparallel-bundled-tbb` on `ubuntu-2604`, each pre-filling `RcppParallel/disable-tbb.patch` for an unrelated package (`ACEsimFit`, `AovBay`, `AdaptGauss`, ...). Root cause: those packages fail on ubuntu-2604 only because their **RcppParallel dependency** does not build there. bincraft records RcppParallel's TBB error text against each dependent, so every one of them matches the `rcppparallel-bundled-tbb` signature. Proposing a RcppParallel source patch for `ACEsimFit` is not just noise -- it is **broken**: the diff targets RcppParallel's `Makevars.in`, so it cannot apply to a dependent's source. ## Fix Add an optional `applies_to` field that pins a package-specific fix to the package it targets: - A signature whose fix is a curated per-package source patch (`rcppparallel-bundled-tbb` -> `RcppParallel`) is only ever proposed **for that package**. - Other packages matching the signature are downstream failures, reported as **"blocked on ``"** and never proposed a bogus entry. Fix the dependency once and the whole cascade clears. Generic levers (the `tbb-stddef-removed` makevars fix) are unaffected -- they carry no `applies_to`, so they still apply to any matching package. ## Changes - gate proposal generation in `build_triage_report()` on `applies_to`; add a `blocked_on` field to group records - surface blocked groups in both `failing-builds-report.R` and `propose-patches.R` instead of the misleading "already registered" note - document `applies_to` in the signature-table header ## Verification - New tests: RcppParallel dependents are blocked (no `proposed_entries`, `blocked_on = "RcppParallel"`), while RcppParallel *itself* is still proposed when it is the failing package. - Reproduced the avalanche end-to-end with a stubbed DB: 5 dependents now report "Blocked on RcppParallel: 5 package(s) fail because RcppParallel does not build" and **zero** candidates are emitted; a genuine `tbb-stddef` proposal in the same run is unaffected. - Full suite: 84 tests pass; all pre-commit hooks pass. Refs #120. Note: the underlying build problem (RcppParallel failing on ubuntu-2604 despite its registry entry) is real and separate -- this PR stops the classifier from spamming broken per-dependent proposals about it. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/121 --- local/failing-builds-classify.R | 31 ++++++++++++++++++- local/failing-builds-report.R | 7 +++++ local/propose-patches.R | 15 ++++++++++ local/tests/test-failing-builds-classify.R | 35 ++++++++++++++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index 30c168d..4ce30cf 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -15,6 +15,11 @@ # would require a brand-new source diff for a previously-unseen package stay # `auto = FALSE` -> classified, but always routed to human triage, per the # issue's guardrail against shipping autonomous novel source diffs. +# `applies_to` (optional) pins a package-specific fix to the package it targets: +# a signature whose registry patch belongs to one package (e.g. RcppParallel's +# disable-tbb patch) is only proposed for that package. Other packages matching +# the signature are downstream failures blocked on that dependency, not +# individually patchable, so they are reported but never proposed an entry. # # Seeded from the existing registry entries and known recurring failures; add # a row here as new signatures are confirmed. Order matters: the first match @@ -45,6 +50,14 @@ build_signatures <- function() { tier = "patch", confidence = "high", auto = TRUE, + # The fix is a RcppParallel-specific source patch, so it is only ever + # proposed for RcppParallel itself. The hundreds of packages that fail + # merely because their RcppParallel *dependency* did not build carry + # RcppParallel's error text and match this signature too -- they are + # blocked on RcppParallel, not individually patchable (applying this diff + # to their source is meaningless). `applies_to` pins the proposal so those + # downstream failures are never proposed a bogus per-package entry. + applies_to = "RcppParallel", fix = "apply the curated RcppParallel/disable-tbb.patch so the bundled TBB build is skipped and the TinyThread backend is used", example = "RcppParallel", registry = list( @@ -278,9 +291,23 @@ build_triage_report <- function( unregistered <- setdiff(pkgs, registered_pkgs) auto_proposable <- isTRUE(sig$auto) && sig$matched + # A signature whose fix is package-specific (`applies_to`) may only be + # proposed for that package. Everything else matching it is a downstream + # failure blocked on that package (e.g. RcppParallel dependents carrying + # RcppParallel's own error text) -- never propose those a bogus entry. + proposable <- unregistered + blocked_on <- NULL + if (!is.null(sig$applies_to)) { + proposable <- intersect(unregistered, sig$applies_to) + downstream <- setdiff(pkgs, sig$applies_to) + if (length(downstream) > 0L) { + blocked_on <- sig$applies_to + } + } + proposed <- list() if (auto_proposable) { - for (p in unregistered) { + for (p in proposable) { proposed[[p]] <- propose_registry_entry_list( sig, p, @@ -297,6 +324,8 @@ build_triage_report <- function( tier = sig$tier, confidence = sig$confidence, suggested_fix = sig$fix, + applies_to = sig$applies_to, + blocked_on = blocked_on, fingerprint = names(fp_tab)[[1L]], fingerprint_variants = length(fp_tab), build_count = nrow(g), diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R index 60f65ea..51f5bc7 100644 --- a/local/failing-builds-report.R +++ b/local/failing-builds-report.R @@ -194,6 +194,13 @@ for (r in report) { ) cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "") } + } else if (!is.null(r$blocked_on)) { + cat(sprintf( + " (blocked on %s -- these %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", + toString(r$blocked_on), + length(r$packages), + toString(r$blocked_on) + )) } else if (r$auto_proposable) { cat(" (all affected packages already have a registry entry)\n") } diff --git a/local/propose-patches.R b/local/propose-patches.R index d43668a..7ff195b 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -135,6 +135,21 @@ for (r in report) { } } +# Groups blocked on a dependency (e.g. RcppParallel dependents) are reported, +# not proposed: fixing the named dependency clears them all at once. +blocked <- Filter(function(r) !is.null(r$blocked_on), report) +if (length(blocked) > 0L) { + cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") + for (r in blocked) { + cat(sprintf( + " %s: %d package(s) fail because %s does not build\n", + toString(r$blocked_on), + length(r$packages), + toString(r$blocked_on) + )) + } +} + # A package that maps to more than one auto-proposable signature is ambiguous # (conflicting fix tiers) and would collide on the same registry key; route it # to human triage instead of emitting both. diff --git a/local/tests/test-failing-builds-classify.R b/local/tests/test-failing-builds-classify.R index 66cd26b..a365e14 100644 --- a/local/tests/test-failing-builds-classify.R +++ b/local/tests/test-failing-builds-classify.R @@ -85,3 +85,38 @@ test_that("propose_registry_entry fills a schema-valid entry for a known lever", # No template -> no proposal (unclassified path). expect_null(propose_registry_entry(classify_error("weird"), "x", "alpine")) }) + +test_that("RcppParallel dependents are blocked, not proposed a per-package patch", { + # Hundreds of packages fail on ubuntu-2604 only because their RcppParallel + # dependency fails to build, so they carry RcppParallel's TBB error text. + failures <- data.frame( + name = c("ACEsimFit", "AovBay", "AdaptGauss"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + grp <- Filter(function(g) g$signature == "rcppparallel-bundled-tbb", report)[[ + 1L + ]] + # No bogus per-dependent entries (the patch targets RcppParallel's source). + expect_null(grp$proposed_entries) + expect_identical(grp$blocked_on, "RcppParallel") +}) + +test_that("RcppParallel itself is still proposed when it is the failing package", { + failures <- data.frame( + name = "RcppParallel", + platform = "ubuntu-2604", + arch = "amd64", + error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + grp <- report[[1L]] + expect_identical(grp$signature, "rcppparallel-bundled-tbb") + expect_false(is.null(grp$proposed_entries)) + expect_true("RcppParallel" %in% names(grp$proposed_entries)) + expect_null(grp$blocked_on) # RcppParallel is the target, not a dependent +}) From 21a2fe9e6cfe93efcd653c461e5fb0fac9fc1d49 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 15 Jul 2026 07:46:29 +0000 Subject: [PATCH 008/118] feat(local): report unclassified and dependency-blocked failures for discovery (#122) ## Why Issue #120 (the auto-proposed-patches issue) only lists **auto-proposable** fixes -- currently just the TBB signatures. So a reasonable read of it was "TBB is our only build failure", when in fact three whole categories are simply not shown there: - **Unclassified failures** -- anything that doesn't match a seeded signature is routed to human triage and never appears (we've only seeded TBB and libuv signatures). - **Dependency-blocked failures** -- the ~800 RcppParallel dependents (post #121) are still failing; they only show as a log line. - Human-only signatures (libuv). These blind spots are exactly where the *next* signatures should come from, so they deserve the same visibility as the proposals. ## What Extend the feedback-loop tracker to surface the classifier's blind spots: - **`unclassified_summary()`** -- groups every unknown-signature failure by normalised fingerprint, ranked by build count, capped with an explicit `dropped_groups` count (no silent truncation), each with example packages + platforms. These are the candidates for new `build_signatures()` rules. - **`blocked_summary()`** -- lists each dependency (e.g. RcppParallel) and how many dependents wait on it. - `proposal-tracking.R` prints both sections, and a new **`--open-issue`** mode posts/updates a *"Unclassified build failures (needs signatures) (#115)"* Forgejo issue. - The weekly crow pipeline now runs the tracker with `--open-issue`, so it maintains a second tracking issue alongside the proposals one. Read-only on the DB; the only writes are the two issues. ## Verification - New tests cover `unclassified_summary` (ranking + both caps) and `blocked_summary`. - Tracker smoke with a stubbed DB (proposable + blocked + unclassified mix) prints the hit rate, `Blocked on a dependency: RcppParallel: 2 dependent(s)`, and `Unclassified failures ... [2 builds | 2 pkgs] ld: undefined reference ...`. - Full suite: 95 tests pass; all pre-commit hooks pass (air, prettier, markdownlint, yamllint, validate-patches). Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/122 --- .crow/weekly-patch-proposals.yaml | 15 +- local/patches/README.md | 12 +- local/proposal-tracking-lib.R | 44 ++++++ local/proposal-tracking.R | 179 ++++++++++++++++++++++- local/tests/test-proposal-tracking-lib.R | 43 ++++++ 5 files changed, 278 insertions(+), 15 deletions(-) diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index f956f3f..0284e93 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -1,12 +1,13 @@ # Weekly failure-triage proposals (issue #115, steps 3 + 4). # Classifies the recorded `single_builds` failures and: -# 1. posts/updates a Forgejo tracking issue with the auto-proposable registry -# entries (human reviews, trial-builds, and opens the PR -- nothing merges), and -# 2. prints the feedback-loop metrics (signature hit rate, proposed-vs-merged, -# retirement candidates) to the run log. +# 1. posts/updates a Forgejo issue with the auto-proposable registry entries +# (human reviews, trial-builds, and opens the PR -- nothing merges), +# 2. posts/updates a second Forgejo issue with the classifier's blind spots +# (unclassified failures = candidates for new signatures, and groups +# blocked on a dependency), and prints the feedback-loop metrics to the log. # Global across platforms (the classifier groups over all of single_builds), so -# a single job -- no matrix. Clones read-only; the only write is the Forgejo -# issue via FORGEJO_TOKEN. +# a single job -- no matrix. Clones read-only; the only writes are the two +# Forgejo issues via FORGEJO_TOKEN. when: - event: manual - event: cron @@ -36,7 +37,7 @@ steps: - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R + - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: kubernetes: resources: diff --git a/local/patches/README.md b/local/patches/README.md index e875217..992395f 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -86,15 +86,21 @@ Rscript local/trial-build-patch.R ### Feedback loop (step 4) `local/proposal-tracking.R` reports the signature hit rate, proposed-vs-merged status (a proposal counts as merged once its package appears in the registry), and retirement candidates (registry entries whose package no longer appears in any current failure, so the upstream cause was likely fixed). -It is read-only. +It also surfaces the classifier's **blind spots**: the unclassified failures (candidates for a new signature) and the groups blocked on a dependency build, so the unknown buckets get the same visibility as the proposals. +It is read-only on the DB, with an optional Forgejo issue as the only write. ```bash +# Print the metrics + blind spots. PGPASS=... Rscript local/proposal-tracking.R --json metrics.json +# Post/update a "needs signatures" tracking issue with the unclassified failures. +PGPASS=... FORGEJO_TOKEN=... Rscript local/proposal-tracking.R --open-issue ``` +The unclassified groups are the natural place to discover which new signatures are worth adding to `build_signatures()`. The pure metric/ledger helpers live in `local/proposal-tracking-lib.R` and are covered by `local/tests/test-proposal-tracking-lib.R`. ### Scheduled run -`.crow/weekly-patch-proposals.yaml` runs both steps weekly (register the `weekly-patch-proposals` cron in the crow UI): it posts/updates a Forgejo tracking issue with the auto-proposable entries and logs the feedback-loop metrics. -It clones read-only; the only write is the tracking issue. +`.crow/weekly-patch-proposals.yaml` runs both steps weekly (register the `weekly-patch-proposals` cron in the crow UI). +It posts/updates two Forgejo issues -- one with the auto-proposable entries, one with the unclassified/blocked failures -- and logs the feedback-loop metrics. +It clones read-only; the only writes are the two tracking issues. diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 2edd9c8..5efc87c 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -131,4 +131,48 @@ retirement_candidates <- function(registry_entries, failing_pkgs) { vapply(keep, function(e) as.character(e$package), character(1L)) } +# Discovery view: the failure groups the classifier could NOT auto-propose, so +# they get the same visibility as proposals. `report` is a build_triage_report +# result. Returns the unclassified groups (unknown signature -> candidates for a +# new signature) ranked by build count, capped to `max_groups`, each with up to +# `max_pkgs` example packages. `dropped_groups`/`packages_truncated` record any +# cap so nothing is silently hidden. +unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) { + un <- Filter(function(g) !isTRUE(g$matched), report) + un <- un[order(-vapply(un, function(g) g$build_count, integer(1L)))] + shown <- utils::head(un, max_groups) + groups <- lapply(shown, function(g) { + pkgs <- g$packages + list( + fingerprint = g$fingerprint, + build_count = g$build_count, + n_packages = length(pkgs), + packages = utils::head(pkgs, max_pkgs), + packages_truncated = length(pkgs) > max_pkgs, + platforms = g$platforms + ) + }) + list( + groups = groups, + total_groups = length(un), + dropped_groups = max(0L, length(un) - length(shown)), + total_builds = sum(vapply(un, function(g) g$build_count, integer(1L))) + ) +} + +# Groups blocked on a dependency (a package-specific fix pinned via `applies_to` +# whose dependents merely carry its error): report the dependency + how many +# dependents wait on it, so fixing it once is recognised as clearing the batch. +blocked_summary <- function(report, max_pkgs = 15L) { + bl <- Filter(function(g) !is.null(g$blocked_on), report) + lapply(bl, function(g) { + list( + blocked_on = g$blocked_on, + n_packages = length(g$packages), + packages = utils::head(g$packages, max_pkgs), + packages_truncated = length(g$packages) > max_pkgs + ) + }) +} + `%||%` <- function(a, b) if (is.null(a)) b else a diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R index 60bbd10..4d82a11 100644 --- a/local/proposal-tracking.R +++ b/local/proposal-tracking.R @@ -1,12 +1,16 @@ #!/usr/bin/env Rscript # Feedback loop for the failure classifier (issue #115, step 4): report the -# signature hit rate, proposed-vs-merged status, and registry entries that look -# retirable, so the rule set can improve and stale entries can be pruned. -# Read-only: queries `single_builds` and reads the registry + proposals ledger. +# signature hit rate, proposed-vs-merged status, registry entries that look +# retirable, and -- so the classifier's blind spots get the same visibility as +# its proposals -- the failures it could NOT classify (candidates for new +# signatures) plus the groups blocked on a dependency build. +# Read-only on the DB/registry; the only optional write is the Forgejo issue. # # Usage: -# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] +# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] [--open-issue] +# --open-issue post/update a Forgejo issue listing the unclassified and +# dependency-blocked failures (needs FORGEJO_TOKEN) options(error = function() { cat("ERROR:", geterrmessage(), "\n", file = stdout()) @@ -33,6 +37,7 @@ json_out <- local({ i <- match("--json", args) if (!is.na(i) && i < length(args)) args[[i + 1L]] else NA_character_ }) +do_issue <- "--open-issue" %in% args if (nchar(Sys.getenv("PGPASS")) == 0L) { stop("PGPASS env var is not set; a DB password is required.") @@ -119,12 +124,52 @@ if (length(retire) > 0L) { cat(" (none)\n") } +# --------------------------------------------------------------------------- +# Blind spots: failures the classifier could not auto-propose. +# --------------------------------------------------------------------------- +blocked <- blocked_summary(report) +unmatched <- unclassified_summary(report) + +cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") +if (length(blocked) > 0L) { + for (b in blocked) { + cat(sprintf(" %s: %d dependent(s) waiting\n", b$blocked_on, b$n_packages)) + } +} else { + cat(" (none)\n") +} + +cat(sprintf( + "\nUnclassified failures (candidates for new signatures): %d group(s), %d builds.\n", + unmatched$total_groups, + unmatched$total_builds +)) +cat(strrep("-", 60L), "\n", sep = "") +for (g in unmatched$groups) { + cat(sprintf( + " [%d builds | %d pkgs] %s\n e.g. %s%s\n", + g$build_count, + g$n_packages, + g$fingerprint, + toString(g$packages), + if (isTRUE(g$packages_truncated)) ", ..." else "" + )) +} +if (unmatched$dropped_groups > 0L) { + cat(sprintf( + " (+%d more unclassified group(s) not shown)\n", + unmatched$dropped_groups + )) +} + if (!is.na(json_out)) { jsonlite::write_json( list( signature_hit_rate = hit, proposed_vs_merged = pvm, - retirement_candidates = retire + retirement_candidates = retire, + blocked = blocked, + unclassified = unmatched ), json_out, auto_unbox = TRUE, @@ -133,3 +178,127 @@ if (!is.na(json_out)) { ) cat(sprintf("\nWrote metrics to %s\n", json_out)) } + +# --------------------------------------------------------------------------- +# Optionally publish the blind-spots to a Forgejo tracking issue. +# --------------------------------------------------------------------------- +if (do_issue) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-issue requires FORGEJO_TOKEN.") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + issue_title <- "Unclassified build failures (needs signatures) (#115)" + now <- format(Sys.time(), "%Y-%m-%d %H:%M:%S") + + body_lines <- c( + sprintf("_Generated %s from `single_builds` failures._", now), + "", + "Failures the classifier could **not** auto-propose a fix for.", + "Each unclassified group is a candidate for a new signature in `local/failing-builds-classify.R`; the blocked groups clear once the named dependency builds.", + "" + ) + + body_lines <- c(body_lines, "## Blocked on a dependency", "") + if (length(blocked) > 0L) { + for (b in blocked) { + body_lines <- c( + body_lines, + sprintf( + "- **%s**: %d dependent(s) waiting (e.g. %s%s)", + b$blocked_on, + b$n_packages, + toString(b$packages), + if (isTRUE(b$packages_truncated)) ", ..." else "" + ) + ) + } + } else { + body_lines <- c(body_lines, "_None._") + } + + body_lines <- c( + body_lines, + "", + sprintf( + "## Unclassified failures (%d groups, %d builds)", + unmatched$total_groups, + unmatched$total_builds + ), + "" + ) + if (length(unmatched$groups) > 0L) { + for (g in unmatched$groups) { + body_lines <- c( + body_lines, + sprintf( + "### %d builds / %d pkg(s)", + g$build_count, + g$n_packages + ), + "", + sprintf("Fingerprint: `%s`", g$fingerprint), + sprintf( + "Packages: %s%s", + toString(g$packages), + if (isTRUE(g$packages_truncated)) ", ..." else "" + ), + sprintf("Platforms: %s", toString(g$platforms)), + "" + ) + } + if (unmatched$dropped_groups > 0L) { + body_lines <- c( + body_lines, + sprintf("_(+%d more group(s) not shown.)_", unmatched$dropped_groups) + ) + } + } else { + body_lines <- c(body_lines, "_None -- every failure is classified._") + } + new_body <- paste(body_lines, collapse = "\n") + + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(issue_title, reserved = TRUE) + ) + existing <- httr2::request(search_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + existing, + function(x) identical(x$title, issue_title), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- existing[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("\nUpdated tracking issue #%d.\n", num)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/issues", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = issue_title, body = new_body)) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf("\nOpened tracking issue #%d.\n", created$number)) + } +} diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index a2fbe05..02a29f1 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -119,6 +119,49 @@ test_that("proposed_vs_merged marks a package merged once it is registered", { expect_identical(stan$status, "merged") }) +test_that("unclassified_summary ranks unknown groups and caps output", { + failures <- data.frame( + name = c("a", "b", "c", "d", "solo"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = c( + # 4 builds share one unknown fingerprint; 1 build a different unknown. + rep("mystery linker meltdown at stage 3", 4L), + "a totally different unknown boom" + ), + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + s <- unclassified_summary(report, max_groups = 30L, max_pkgs = 2L) + expect_identical(s$total_groups, 2L) + expect_identical(s$total_builds, 5L) + # Largest group first, and its example packages are capped at max_pkgs. + expect_identical(s$groups[[1L]]$build_count, 4L) + expect_length(s$groups[[1L]]$packages, 2L) + expect_true(s$groups[[1L]]$packages_truncated) + + # max_groups cap is reported, not silently dropped. + capped <- unclassified_summary(report, max_groups = 1L) + expect_length(capped$groups, 1L) + expect_identical(capped$dropped_groups, 1L) +}) + +test_that("blocked_summary lists each dependency and its dependent count", { + failures <- data.frame( + name = c("ACEsimFit", "AovBay", "AdaptGauss"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + b <- blocked_summary(report, max_pkgs = 2L) + expect_length(b, 1L) + expect_identical(b[[1L]]$blocked_on, "RcppParallel") + expect_identical(b[[1L]]$n_packages, 3L) + expect_true(b[[1L]]$packages_truncated) +}) + test_that("retirement_candidates flags entries whose package no longer fails", { entries <- list( list(package = "RcppParallel"), From 4bca17e4acae7abea58e478ac56f1725b90cb5dc Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 15 Jul 2026 08:28:15 +0000 Subject: [PATCH 009/118] feat(local): auto-apply registry patches with a build-env trial-build gate (#124) Closes the classifier loop (issue #115, step 3): turns the auto-proposable candidates into an actual PR, gated by a real trial build **in our own build-env images**. Chosen model (from the design discussion): **autonomous PR**, **PR-first with a CI trial-build gate**, **bounded top-N batch** per run. ## Creating the patch PR - `propose-patches.R` gains: - `--limit N` -- act on the top-N candidates by failure volume; the rest defer to the next run (logged). - `--open-pr` -- write the entries onto the reused `auto/registry-patch-proposals` branch, push (with `REPO_RW_TOKEN`), and open/update **one** PR via the Forgejo API (so re-runs update the same PR instead of piling up). - `.crow/auto-apply-patches.yaml` -- a single job that runs `--open-pr --limit` on a cron/manual trigger. Needs `FORGEJO_TOKEN` + a write-scoped `REPO_RW_TOKEN`. Novel source diffs and unknown signatures are still never proposed; nothing merges. ## The merge gate (our build-env images) - `.crow/trial-build-registry.yaml` -- matrixed over the real `OS/IMG` build-env matrix (alpine:3.24, redhat:8/9/10, ubuntu:jammy/noble/**resolute** for ubuntu-2604). Each platform runs `local/trial-build-registry.R`, which diffs the branch registry against `main` and trial-builds **only the entries the branch adds** that apply to that platform, inside `reg.devxy.io/rpkgs/build-env-*`. Green only if every new entry builds; a platform with no new entries is a fast no-op. Nothing is uploaded/archived/recorded. - The base-registry read **fails loud** if it can't read `registry.json` at `main`, rather than silently treating the base as empty and trial-building the whole registry. ## Notes / follow-up - The repo uses **no `pull_request` triggers**, so the gate runs manually or on a cron against the branch (`--var patch_branch=...`). Wiring it to fire automatically on the PR needs `event: pull_request` enabled on the Forgejo webhook -- a one-line addition once that's on. - Two new crons to register in the crow UI: `auto-apply-patches` and `trial-build-registry`. New secret needed: `REPO_RW_TOKEN` (write scope) for the push. ## Verification - New pure helpers `entry_applies_to_os()` / `new_registry_packages()` covered by tests (platform codename/family/wildcard matching; added-vs-unchanged entry detection; per-platform filtering). - `--limit` smoke (stubbed DB): top-2 by volume proposed, 3 deferred, candidate registry validates. - Full suite: 105 tests pass; all pre-commit hooks pass (air, prettier, markdownlint, yamllint, validate-patches). Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/124 --- .crow/auto-apply-patches.yaml | 65 ++++++++ .crow/trial-build-registry.yaml | 140 +++++++++++++++++ local/patches/README.md | 13 ++ local/proposal-tracking-lib.R | 33 ++++ local/propose-patches.R | 188 ++++++++++++++++++++++- local/tests/test-proposal-tracking-lib.R | 51 ++++++ local/trial-build-registry.R | 145 +++++++++++++++++ 7 files changed, 633 insertions(+), 2 deletions(-) create mode 100644 .crow/auto-apply-patches.yaml create mode 100644 .crow/trial-build-registry.yaml create mode 100644 local/trial-build-registry.R diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml new file mode 100644 index 0000000..15cc1ee --- /dev/null +++ b/.crow/auto-apply-patches.yaml @@ -0,0 +1,65 @@ +# Auto-apply registry patches (issue #115, step 3 automation). +# Classifies `single_builds` failures and, for the top-N auto-proposable +# candidates by failure volume, writes the registry entries onto the reused +# `auto/registry-patch-proposals` branch and opens/updates a single PR. +# Nothing merges: the `trial-build-registry` pipeline is the merge gate, and a +# human reviews the PR. Novel source diffs / unknown signatures are never +# proposed. Global across platforms, so a single job -- no matrix. +# +# Needs a write token (REPO_RW_TOKEN) to push and FORGEJO_TOKEN to open the PR. +# Register the `auto-apply-patches` cron in the crow UI, or run manually: +# woodpecker-cli pipeline create --var task=auto-apply-patches --branch=main 7 +variables: + patch_limit: + description: 'Max candidates to propose per run (top by failure volume).' + default: '10' + +when: + - event: manual + evaluate: 'task == "auto-apply-patches"' + - event: cron + cron: auto-apply-patches + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Auto-apply registry patches' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + REPO_RW_TOKEN: + from_secret: REPO_RW_TOKEN + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + GIT_USER: devxy-bot + GIT_EMAIL: bot@devxy.io + PATCH_LIMIT: ${patch_limit} + R_VERSION: 4.5.3 + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 2000m + limits: + memory: 2Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml new file mode 100644 index 0000000..1ac71e8 --- /dev/null +++ b/.crow/trial-build-registry.yaml @@ -0,0 +1,140 @@ +# Merge gate for the auto-patch PR (issue #115, step 3). +# For each platform, trial-builds every registry entry the auto-patch branch +# ADDS (vs main) in that platform's own `reg.devxy.io/rpkgs/build-env-*` image, +# with the registry applied. A row with no new entries for its platform is a +# fast no-op. The pipeline is green only if every new entry builds, so it gates +# the PR before merge. Nothing is uploaded/archived/recorded. +# +# The repo uses no `pull_request` triggers, so this runs manually against the +# branch (or on a cron); point it at the auto-patch branch via `patch_branch`: +# woodpecker-cli pipeline create --var task=trial-build-registry \ +# --var patch_branch=auto/registry-patch-proposals --branch=main 7 +variables: + patch_branch: + description: 'Branch whose new registry entries to trial-build.' + default: auto/registry-patch-proposals + +when: + - event: manual + evaluate: 'task == "trial-build-registry"' + - event: cron + cron: trial-build-registry + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + +steps: + - name: 'Trial-build new registry entries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + PLATFORM: ${OS} + ARCH: ${ARCH} + R_VERSION: ${R_VERSION} + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q --branch ${patch_branch} https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - git fetch -q origin main + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/Rscript local/trial-build-registry.R origin/main + backend_options: + kubernetes: + resources: + requests: + memory: 2Gi + cpu: 2000m + limits: + memory: 4Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/local/patches/README.md b/local/patches/README.md index 992395f..455a96c 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -81,6 +81,19 @@ Run the trial build inside the failing platform's build-env image; it uploads/ar Rscript local/trial-build-patch.R ``` +#### Autonomous PR + trial-build gate + +`--open-pr` closes the loop: it writes the top-N candidates (by failure volume) onto the reused `auto/registry-patch-proposals` branch and opens/updates a single PR. +`.crow/auto-apply-patches.yaml` runs this on a cron (needs `FORGEJO_TOKEN` and a write-scoped `REPO_RW_TOKEN`). + +```bash +# Bounded batch; opens/updates one PR. +PGPASS=... FORGEJO_TOKEN=... REPO_RW_TOKEN=... Rscript local/propose-patches.R --open-pr --limit 10 +``` + +The merge gate is `.crow/trial-build-registry.yaml`: matrixed over the build-env images, each platform trial-builds only the entries the branch **adds** (`local/trial-build-registry.R`, which diffs the registry against `main`) and is green only if every new entry builds. +The repo uses no `pull_request` triggers, so this gate runs manually or on a cron against the auto-patch branch (`--var patch_branch=...`); wiring it to fire on the PR needs `event: pull_request` enabled on the forge. + `--write` and `--open-issue` also append to `local/patches/proposals-log.json`, a ledger of what was proposed. ### Feedback loop (step 4) diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 5efc87c..80c4ab2 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -175,4 +175,37 @@ blocked_summary <- function(report, max_pkgs = 15L) { }) } +# Does a registry entry's `platforms` apply to a build on `os` (e.g. +# "ubuntu-2604")? Mirrors bincraft's token match: an entry applies if any of its +# platform tokens is "*", the OS codename, or the distro family ("ubuntu"). +entry_applies_to_os <- function(entry_platforms, os) { + toks <- as.character(unlist(entry_platforms)) + family <- sub("-.*$", "", os) # ubuntu-2604 -> ubuntu + any(toks %in% c("*", os, family)) +} + +# Registry entries present in `current` but not in `base` (matched on +# package|platforms|versions), optionally restricted to those that apply to a +# given `os`. Used by the trial-build gate to build only the entries a PR adds. +new_registry_packages <- function(current, base, os = NULL) { + key <- function(e) { + sprintf( + "%s|%s|%s", + e$package %||% "?", + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + e$versions %||% "?" + ) + } + base_keys <- vapply(base %||% list(), key, character(1L)) + added <- Filter(function(e) !(key(e) %in% base_keys), current %||% list()) + if (!is.null(os)) { + added <- Filter(function(e) entry_applies_to_os(e$platforms, os), added) + } + unique(vapply( + added, + function(e) as.character(e$package %||% ""), + character(1L) + )) +} + `%||%` <- function(a, b) if (is.null(a)) b else a diff --git a/local/propose-patches.R b/local/propose-patches.R index 7ff195b..fb45ea9 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -15,10 +15,17 @@ # # Usage: # PGPASS=... Rscript local/propose-patches.R [--platform P] [--arch A] [--min N] +# [--limit N] # (default) print candidates + validation, take no action # --write append candidates to local/patches/registry.json and # the proposals ledger (commit + open a PR yourself) # --open-issue post/update a Forgejo tracking issue (needs FORGEJO_TOKEN) +# --open-pr write the entries, push the `auto/registry-patch-proposals` +# branch, and open/update a PR autonomously (needs +# FORGEJO_TOKEN, and REPO_RW_TOKEN to push in CI). The +# `trial-build-registry` pipeline is the merge gate. +# --limit N only act on the top-N candidates by failure volume +# (bounded batch; the rest are picked up on the next run) # --json PATH also write the machine-readable candidate list to PATH options(error = function() { @@ -58,6 +65,8 @@ if (is.na(min_count)) { } do_write <- "--write" %in% args do_issue <- "--open-issue" %in% args +do_pr <- "--open-pr" %in% args +limit <- suppressWarnings(as.integer(opt_val("--limit", NA_character_))) if (nchar(Sys.getenv("PGPASS")) == 0L) { stop("PGPASS env var is not set; a DB password is required.") @@ -65,6 +74,8 @@ if (nchar(Sys.getenv("PGPASS")) == 0L) { registry_file <- file.path(script_dir, "patches", "registry.json") ledger_file <- file.path(script_dir, "patches", "proposals-log.json") +# Branch the autonomous PR reuses, so re-runs update one PR instead of piling up. +pr_branch <- "auto/registry-patch-proposals" # --------------------------------------------------------------------------- # Query failing builds (same shape as failing-builds-report.R) @@ -118,7 +129,8 @@ registered_pkgs <- unique(vapply( report <- build_triage_report(failures, registered_pkgs) report <- Filter(function(r) r$build_count >= min_count, report) -# Flatten auto-proposable groups into candidate records. +# Flatten auto-proposable groups into candidate records, carrying the group's +# build volume so we can prioritise the highest-impact fixes. candidates <- list() for (r in report) { if (is.null(r$proposed_entries)) { @@ -130,6 +142,7 @@ for (r in report) { signature = r$signature, confidence = r$confidence, tier = r$tier, + build_count = r$build_count, entry = r$proposed_entries[[pkg]] ) } @@ -173,6 +186,22 @@ if (length(candidates) == 0L) { q(status = 0) } +# Prioritise by failure volume, then apply --limit so one run tackles a bounded +# batch (the rest are picked up on the next run). +candidates <- candidates[order( + -vapply(candidates, function(c) c$build_count %||% 0L, integer(1L)) +)] +deferred <- 0L +if (!is.na(limit) && limit >= 0L && length(candidates) > limit) { + deferred <- length(candidates) - limit + candidates <- utils::head(candidates, limit) + cat(sprintf( + "\nLimiting to top %d candidate(s) by failure volume; %d deferred to a later run.\n", + limit, + deferred + )) +} + cat(sprintf( "\n%d candidate registry %s:\n", length(candidates), @@ -386,8 +415,163 @@ if (do_write) { cat(sprintf("Opened tracking issue #%d.\n", created$number)) } save_ledger(merge_ledger(load_ledger(), new_ledger_records)) +} else if (do_pr) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-pr requires FORGEJO_TOKEN (to open the PR).") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + + # Write the entries + ledger, then commit them onto the reused auto branch. + jsonlite::write_json( + merged, + registry_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) + + git <- function(...) { + st <- system2("git", c(...), stdout = TRUE, stderr = TRUE) + if (!identical(attr(st, "status"), NULL)) { + stop(sprintf( + "git %s failed:\n%s", + paste(..., collapse = " "), + paste(st, collapse = "\n") + )) + } + invisible(st) + } + git("config", "user.name", Sys.getenv("GIT_USER", "devxy-bot")) + git( + "config", + "user.email", + Sys.getenv("GIT_EMAIL", "bot@devxy.io") + ) + git("checkout", "-B", pr_branch) + git( + "add", + file.path(script_dir, "patches", "registry.json"), + ledger_file + ) + git( + "commit", + "-m", + sprintf( + "feat(patches): auto-propose %d registry %s from classified failures", + length(candidate_entries), + if (length(candidate_entries) == 1L) "entry" else "entries" + ) + ) + # Push with a write token when provided (CI); otherwise rely on origin creds. + rw_token <- Sys.getenv("REPO_RW_TOKEN") + push_target <- if (nchar(rw_token) > 0L) { + sprintf( + "https://%s:%s@git.devxy.io/%s.git", + Sys.getenv("GIT_USER", "devxy-bot"), + rw_token, + repo + ) + } else { + "origin" + } + git("push", "-f", push_target, sprintf("HEAD:refs/heads/%s", pr_branch)) + + pr_title <- sprintf( + "feat(patches): auto-proposed registry patches (%s)", + now + ) + body_lines <- c( + sprintf( + "_Auto-generated %s by `local/propose-patches.R --open-pr` from classified `single_builds` failures._", + now + ), + "", + sprintf( + "Adds %d known-lever registry %s (top by failure volume%s).", + length(candidate_entries), + if (length(candidate_entries) == 1L) "entry" else "entries", + if (deferred > 0L) { + sprintf("; %d deferred to a later run", deferred) + } else { + "" + } + ), + "", + "**Merge gate:** the `trial-build-registry` pipeline builds each new entry in its target build-env image; merge only once it is green.", + "Novel source diffs and unknown signatures are never auto-proposed.", + "", + "| package | signature | platforms |", + "| --- | --- | --- |" + ) + for (c in candidates) { + body_lines <- c( + body_lines, + sprintf( + "| %s | %s | %s |", + c$package, + c$signature, + toString(as.character(c$entry$platforms)) + ) + ) + } + new_body <- paste(body_lines, collapse = "\n") + + # One PR per reused branch: update if open, else create. + pulls_url <- sprintf( + "%s/repos/%s/pulls?state=open&limit=50", + forgejo_base, + repo + ) + open_pulls <- httr2::request(pulls_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + open_pulls, + function(p) identical(p$head$ref, pr_branch), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- open_pulls[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/pulls/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = pr_title, body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("\nUpdated auto-patch PR #%d (branch %s).\n", num, pr_branch)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/pulls", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list( + title = pr_title, + head = pr_branch, + base = "main", + body = new_body + )) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf( + "\nOpened auto-patch PR #%d (branch %s).\n", + created$number, + pr_branch + )) + } } else { cat( - "\nDry run: no changes made. Re-run with --write or --open-issue to act.\n" + "\nDry run: no changes made. Re-run with --write, --open-issue, or --open-pr to act.\n" ) } diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index 02a29f1..05cd0e9 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -162,6 +162,57 @@ test_that("blocked_summary lists each dependency and its dependent count", { expect_true(b[[1L]]$packages_truncated) }) +test_that("entry_applies_to_os matches codename, family, and wildcard", { + expect_true(entry_applies_to_os(list("ubuntu-2604"), "ubuntu-2604")) + expect_true(entry_applies_to_os(list("ubuntu"), "ubuntu-2604")) # family + expect_true(entry_applies_to_os(list("*"), "ubuntu-2604")) + expect_true(entry_applies_to_os(list("alpine", "ubuntu-2604"), "ubuntu-2604")) + expect_false(entry_applies_to_os(list("alpine-324"), "ubuntu-2604")) + expect_false(entry_applies_to_os(list("ubuntu-2404"), "ubuntu-2604")) # other codename +}) + +test_that("new_registry_packages returns only added entries for the platform", { + base <- list( + list( + package = "RcppParallel", + platforms = list("alpine", "ubuntu-2604"), + versions = "*" + ) + ) + current <- list( + base[[1L]], # unchanged -> not "new" + list(package = "BFpack", platforms = list("ubuntu-2604"), versions = "*"), + list( + package = "someAlpinePkg", + platforms = list("alpine-324"), + versions = "*" + ) + ) + # For ubuntu-2604: only the newly-added BFpack (RcppParallel is unchanged, + # someAlpinePkg does not apply to this OS). + expect_identical( + new_registry_packages(current, base, os = "ubuntu-2604"), + "BFpack" + ) + # For alpine-324: the alpine package is new and applies. + expect_identical( + new_registry_packages(current, base, os = "alpine-324"), + "someAlpinePkg" + ) + # Without an OS filter, both additions are returned. + expect_setequal( + new_registry_packages(current, base), + c("BFpack", "someAlpinePkg") + ) + # A changed platform set on the same package counts as a new entry. + widened <- list(list( + package = "RcppParallel", + platforms = list("*"), + versions = "*" + )) + expect_identical(new_registry_packages(widened, base), "RcppParallel") +}) + test_that("retirement_candidates flags entries whose package no longer fails", { entries <- list( list(package = "RcppParallel"), diff --git a/local/trial-build-registry.R b/local/trial-build-registry.R new file mode 100644 index 0000000..f649b62 --- /dev/null +++ b/local/trial-build-registry.R @@ -0,0 +1,145 @@ +#!/usr/bin/env Rscript + +# Merge gate for the auto-patch PR (issue #115, step 3): for every registry +# entry the PR ADDS that applies to this platform, trial-build the package with +# the registry applied, in this platform's own build-env image. Nothing is +# uploaded, archived, or written to the metadata DB. +# +# Exit 0 only if every new entry's package builds; exit 1 if any fails, so it +# gates the PR. A platform with no new entries is a fast no-op. +# +# Usage (inside a build-env image): +# PLATFORM=ubuntu-2604 Rscript local/trial-build-registry.R [base_ref] +# base_ref git ref to diff the registry against (default: origin/main) + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(jsonlite, quietly = TRUE) + library(bincraft, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +args <- commandArgs(trailingOnly = TRUE) +base_ref <- if (length(args) >= 1L) { + args[[1L]] +} else { + Sys.getenv("BASE_REF", "origin/main") +} +os <- Sys.getenv("PLATFORM", "") +if (!nzchar(os)) { + stop("PLATFORM env var is not set (e.g. ubuntu-2604).") +} + +patches_dir <- file.path(script_dir, "patches") +registry_file <- file.path(patches_dir, "registry.json") +current <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +# Read the registry at base_ref. Fail loud if the ref or file can't be read: +# silently treating the base as empty would trial-build the WHOLE registry +# instead of just the entries the branch adds. +registry_rel <- "local/patches/registry.json" +ref_ok <- suppressWarnings(system2( + "git", + c("rev-parse", "--verify", "--quiet", sprintf("%s^{commit}", base_ref)), + stdout = TRUE, + stderr = FALSE +)) +if (!is.null(attr(ref_ok, "status"))) { + stop(sprintf("base ref %s does not resolve to a commit.", base_ref)) +} +in_base <- suppressWarnings(system2( + "git", + c("ls-tree", base_ref, "--", registry_rel), + stdout = TRUE, + stderr = FALSE +)) +file_in_base <- length(in_base) > 0L && any(nzchar(in_base)) +base_json <- suppressWarnings(system2( + "git", + c("show", sprintf("%s:%s", base_ref, registry_rel)), + stdout = TRUE, + stderr = FALSE +)) +show_ok <- is.null(attr(base_json, "status")) +if (file_in_base && !show_ok) { + stop(sprintf( + "could not read %s at %s; refusing to build the whole registry.", + registry_rel, + base_ref + )) +} +base <- if (show_ok && length(base_json) > 0L) { + jsonlite::fromJSON(paste(base_json, collapse = "\n"), simplifyVector = FALSE) +} else { + list() # file genuinely absent at base -> every entry is new +} + +pkgs <- new_registry_packages(current, base, os = os) +if (length(pkgs) == 0L) { + cat(sprintf( + "No new registry entries apply to %s; nothing to trial-build.\n", + os + )) + q(status = 0) +} + +cat(sprintf( + "Trial-building %d new registry %s on %s (vs %s):\n %s\n", + length(pkgs), + if (length(pkgs) == 1L) "entry" else "entries", + os, + base_ref, + toString(pkgs) +)) + +results <- vapply( + pkgs, + function(pkg) { + cat(sprintf("\n=== trial build: %s ===\n", pkg)) + tryCatch( + { + bincraft::build_binary_package( + pkg, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ) + TRUE + }, + error = function(e) { + cat(sprintf("FAILED %s: %s\n", pkg, conditionMessage(e))) + FALSE + } + ) + }, + logical(1L) +) + +failed <- pkgs[!results] +cat(sprintf( + "\n%d/%d passed on %s.%s\n", + sum(results), + length(results), + os, + if (length(failed) > 0L) sprintf(" Failed: %s", toString(failed)) else "" +)) +if (length(failed) > 0L) { + q(status = 1) +} +q(status = 0) From 1bca6004fd7a219560e468aa7e7de66bc2aa532b Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 15 Jul 2026 14:50:19 +0000 Subject: [PATCH 010/118] chore(local): reuse FORGEJO_TOKEN for the auto-patch push (#125) Follow-up to #124 (merged): the auto-patch pipeline required a separate write-scoped `REPO_RW_TOKEN` to push the branch. Reuse the existing `FORGEJO_TOKEN` instead. - `propose-patches.R --open-pr` now pushes `auto/registry-patch-proposals` over HTTPS with `FORGEJO_TOKEN` (the same token used for the PR API); the read-only `origin` clone URL can't push, so it builds an authenticated URL explicitly. - Drop `REPO_RW_TOKEN` from `.crow/auto-apply-patches.yaml` and the docs. No new secret needed: the pipeline's secrets are now `PGPASS`, `REPO_RO_TOKEN`, and `FORGEJO_TOKEN` (all existing). `FORGEJO_TOKEN` must have repository write scope for the push to succeed. 105 tests pass; all pre-commit hooks pass. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/125 --- .crow/auto-apply-patches.yaml | 7 +++---- local/patches/README.md | 5 +++-- local/propose-patches.R | 25 +++++++++++-------------- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index 15cc1ee..2ac6886 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -6,8 +6,9 @@ # human reviews the PR. Novel source diffs / unknown signatures are never # proposed. Global across platforms, so a single job -- no matrix. # -# Needs a write token (REPO_RW_TOKEN) to push and FORGEJO_TOKEN to open the PR. -# Register the `auto-apply-patches` cron in the crow UI, or run manually: +# FORGEJO_TOKEN is used for both the branch push and opening the PR (no separate +# write-scoped secret needed). Register the `auto-apply-patches` cron in the crow +# UI, or run manually: # woodpecker-cli pipeline create --var task=auto-apply-patches --branch=main 7 variables: patch_limit: @@ -34,8 +35,6 @@ steps: from_secret: PGPASS REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN - REPO_RW_TOKEN: - from_secret: REPO_RW_TOKEN FORGEJO_TOKEN: from_secret: FORGEJO_TOKEN GIT_USER: devxy-bot diff --git a/local/patches/README.md b/local/patches/README.md index 455a96c..ce60c9a 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -84,11 +84,12 @@ Rscript local/trial-build-patch.R #### Autonomous PR + trial-build gate `--open-pr` closes the loop: it writes the top-N candidates (by failure volume) onto the reused `auto/registry-patch-proposals` branch and opens/updates a single PR. -`.crow/auto-apply-patches.yaml` runs this on a cron (needs `FORGEJO_TOKEN` and a write-scoped `REPO_RW_TOKEN`). +`.crow/auto-apply-patches.yaml` runs this on a cron. +`FORGEJO_TOKEN` is used for both the branch push and the PR (no separate write-scoped secret). ```bash # Bounded batch; opens/updates one PR. -PGPASS=... FORGEJO_TOKEN=... REPO_RW_TOKEN=... Rscript local/propose-patches.R --open-pr --limit 10 +PGPASS=... FORGEJO_TOKEN=... Rscript local/propose-patches.R --open-pr --limit 10 ``` The merge gate is `.crow/trial-build-registry.yaml`: matrixed over the build-env images, each platform trial-builds only the entries the branch **adds** (`local/trial-build-registry.R`, which diffs the registry against `main`) and is green only if every new entry builds. diff --git a/local/propose-patches.R b/local/propose-patches.R index fb45ea9..11085f0 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -21,8 +21,8 @@ # the proposals ledger (commit + open a PR yourself) # --open-issue post/update a Forgejo tracking issue (needs FORGEJO_TOKEN) # --open-pr write the entries, push the `auto/registry-patch-proposals` -# branch, and open/update a PR autonomously (needs -# FORGEJO_TOKEN, and REPO_RW_TOKEN to push in CI). The +# branch, and open/update a PR autonomously. Uses +# FORGEJO_TOKEN for both the push and the PR API. The # `trial-build-registry` pipeline is the merge gate. # --limit N only act on the top-N candidates by failure volume # (bounded batch; the rest are picked up on the next run) @@ -466,18 +466,15 @@ if (do_write) { if (length(candidate_entries) == 1L) "entry" else "entries" ) ) - # Push with a write token when provided (CI); otherwise rely on origin creds. - rw_token <- Sys.getenv("REPO_RW_TOKEN") - push_target <- if (nchar(rw_token) > 0L) { - sprintf( - "https://%s:%s@git.devxy.io/%s.git", - Sys.getenv("GIT_USER", "devxy-bot"), - rw_token, - repo - ) - } else { - "origin" - } + # Push over HTTPS with FORGEJO_TOKEN (same token used for the PR API), so no + # separate write-scoped secret is needed. The read-only `origin` clone URL + # can't push, so build an authenticated URL explicitly. + push_target <- sprintf( + "https://%s:%s@git.devxy.io/%s.git", + Sys.getenv("GIT_REMOTE_USER", "pat-s"), + forgejo_token, + repo + ) git("push", "-f", push_target, sprintf("HEAD:refs/heads/%s", pr_branch)) pr_title <- sprintf( From ed1485c542d4768e75537cf3728423b1c5ee564b Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 15 Jul 2026 16:51:15 +0200 Subject: [PATCH 011/118] ci: remove evaluate conditions --- .crow/auto-apply-patches.yaml | 1 - .crow/trial-build-registry.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index 2ac6886..b9c8697 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -17,7 +17,6 @@ variables: when: - event: manual - evaluate: 'task == "auto-apply-patches"' - event: cron cron: auto-apply-patches diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 1ac71e8..9871238 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -16,7 +16,6 @@ variables: when: - event: manual - evaluate: 'task == "trial-build-registry"' - event: cron cron: trial-build-registry From ff9f5f5177529553d89a289e8682681289b46be9 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 15 Jul 2026 15:43:02 +0000 Subject: [PATCH 012/118] fix(local): shell-quote git args in the auto-patch push and trial-build gate (#126) ## Problem The first real `auto-apply-patches` run classified, limited to the top-10, and validated the candidate registry cleanly, then died at the push step: ``` Validating candidate registry: Patch registry OK (13 entries). sh: syntax error: unexpected "(" Error in git("commit", "-m", ...): git commit -m feat(patches): auto-propose 10 registry entries ... failed ``` Root cause: R's `system2()` with captured output (`stdout=TRUE`) runs the command through `/bin/sh`, and the arguments were passed **unquoted**. The commit message `feat(patches): ...` contains `()`, which the shell tried to interpret. The same class of bug affects the `^{commit}` and `ref:path` git refs in the trial-build gate. ## Fix `shQuote()` every git argument: - `propose-patches.R` -- the `git()` helper used by `--open-pr` (commit, push, checkout). - `trial-build-registry.R` -- the base-ref reads (`rev-parse ... ^{commit}`, `ls-tree`, `show ref:path`). ## Verification - Reproduced against a real git repo: the unquoted call fails with status 2 (the same `unexpected "("`); the `shQuote`d call commits successfully, and `rev-parse HEAD^{commit}` resolves. - 105 tests pass; all pre-commit hooks pass. Everything else in that run was correct: 842,659 failing builds classified, RcppParallel's 895 dependents correctly reported as blocked (not proposed), top-10 tbb-stddef candidates selected, 69 deferred, registry validated. Only the shell quoting was broken. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/126 --- local/propose-patches.R | 4 +++- local/trial-build-registry.R | 13 ++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/local/propose-patches.R b/local/propose-patches.R index 11085f0..7008251 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -435,7 +435,9 @@ if (do_write) { save_ledger(merge_ledger(load_ledger(), new_ledger_records)) git <- function(...) { - st <- system2("git", c(...), stdout = TRUE, stderr = TRUE) + # system2() with captured output runs via /bin/sh, so shell-quote every arg + # (commit messages contain "()", refs contain "^{}", etc.). + st <- system2("git", shQuote(c(...)), stdout = TRUE, stderr = TRUE) if (!identical(attr(st, "status"), NULL)) { stop(sprintf( "git %s failed:\n%s", diff --git a/local/trial-build-registry.R b/local/trial-build-registry.R index f649b62..0749bf7 100644 --- a/local/trial-build-registry.R +++ b/local/trial-build-registry.R @@ -52,9 +52,16 @@ current <- if (file.exists(registry_file)) { # silently treating the base as empty would trial-build the WHOLE registry # instead of just the entries the branch adds. registry_rel <- "local/patches/registry.json" +# system2() with captured output runs via /bin/sh, so shell-quote the git args +# (refs contain "^{}" and ":" that the shell would otherwise mangle). ref_ok <- suppressWarnings(system2( "git", - c("rev-parse", "--verify", "--quiet", sprintf("%s^{commit}", base_ref)), + shQuote(c( + "rev-parse", + "--verify", + "--quiet", + sprintf("%s^{commit}", base_ref) + )), stdout = TRUE, stderr = FALSE )) @@ -63,14 +70,14 @@ if (!is.null(attr(ref_ok, "status"))) { } in_base <- suppressWarnings(system2( "git", - c("ls-tree", base_ref, "--", registry_rel), + shQuote(c("ls-tree", base_ref, "--", registry_rel)), stdout = TRUE, stderr = FALSE )) file_in_base <- length(in_base) > 0L && any(nzchar(in_base)) base_json <- suppressWarnings(system2( "git", - c("show", sprintf("%s:%s", base_ref, registry_rel)), + shQuote(c("show", sprintf("%s:%s", base_ref, registry_rel))), stdout = TRUE, stderr = FALSE )) From 875b0861226d55b1762a1f5a181940b5e9be43d4 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 16 Jul 2026 08:23:44 +0000 Subject: [PATCH 013/118] ci(crow): enable verbose patched builds in the trial-build gate (#129) Companion to bincraft codefloe #65. Sets `BINCRAFT_VERBOSE_PATCH_BUILD=TRUE` in the trial-build gate so a failed isolated patched build prints the real compiler error instead of `System command 'R' failed`. Harmless on bincraft versions without the flag (unknown env var is ignored). Takes effect once bincraft #65 is released and the build-env images pick it up. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/129 --- .crow/trial-build-registry.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 9871238..602837f 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -116,6 +116,10 @@ steps: ARCH: ${ARCH} R_VERSION: ${R_VERSION} R_LIBS_USER: /mnt/cache/R-pkgs + # Surface the real compiler error when an isolated patched build fails, + # instead of bincraft's opaque "System command 'R' failed" (needs bincraft + # with BINCRAFT_VERBOSE_PATCH_BUILD support; harmless on older versions). + BINCRAFT_VERBOSE_PATCH_BUILD: 'TRUE' commands: - git clone -q --branch ${patch_branch} https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git fetch -q origin main From f9d399fac0e60f748eaa2c65d23f2c34c437d3a1 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 16 Jul 2026 08:23:54 +0000 Subject: [PATCH 014/118] feat(local): detect dependency-cascade failures generally, not just RcppParallel (#128) ## Why (from the #127 trial-build gate) The gate did its job: 0/3 passed, merge blocked. The log showed *why* -- BFpack, BayesERtools, GMLTM all fail while building their shared dependency **`rstan`**, not in their own code: ``` Failed to build source package rstan. .../StanHeaders/include/stan/math/prim/core/init_threadpool_tbb.hpp:9:10: fatal error: tbb/tbb_stddef.h: No such file or directory ``` So the per-package `-DTBB_INTERFACE_NEW` makevars entries the classifier proposed are useless for these packages -- they're blocked on `rstan` (which already has a registry entry). This is the **same dependency cascade** the RcppParallel `applies_to` guard catches, but `tbb-stddef-removed` is a generic signature with no such pin, so ~73 Stan packages kept getting proposed. ## What Generalise cascade detection beyond the RcppParallel special case: - `failing_dependency(error_text, package)` -- when the log names a **different** package as the one that failed to compile (`Failed to build source package X`, `compilation failed for package 'X'`, `dependency 'X' ... not available`), that package is the real cause. - `build_triage_report()` now blocks any package whose **every** failing build is such a cascade: reported as `blocked_on` that dependency, never proposed a bogus per-package entry. A package that fails in its **own** compilation is still proposed. - The `applies_to` (RcppParallel) and data-driven (rstan) cases are unified into one `blocked_packages` / `blocked_on` model; the report, proposer, and `blocked_summary` count the actually-blocked packages, and the blocked note shows even when a group also has genuine proposals. ## Effect Next auto-apply run will stop proposing the rstan-blocked Stan packages (and any future dependency cascade) and surface them as "blocked on rstan" instead. Fixing `rstan` once clears the whole cluster. ## Verification - New tests: `failing_dependency` (cascade vs own-compile vs none), and an end-to-end split where BFpack/GMLTM (blocked on rstan) are not proposed while an own-compile package still is. - Full suite: 112 tests pass; all pre-commit hooks pass. Refs #120, #127. (Separate follow-ups: fixing rstan's build itself, and quieting the gate's metadata-DB retry storm -- both root-caused to bincraft.) Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/128 --- local/failing-builds-classify.R | 77 +++++++++++++++++++--- local/failing-builds-report.R | 12 ++-- local/proposal-tracking-lib.R | 19 ++++-- local/propose-patches.R | 7 +- local/tests/test-failing-builds-classify.R | 43 ++++++++++++ 5 files changed, 135 insertions(+), 23 deletions(-) diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index 4ce30cf..716fab2 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -155,6 +155,44 @@ fingerprint_error <- function(error_text, package = NULL, max_chars = 200L) { fp } +# --------------------------------------------------------------------------- +# Dependency-cascade detection +# --------------------------------------------------------------------------- +# If a build's error_text shows the failure was actually in a DIFFERENT package +# (a dependency that would not compile), return that dependency's name; +# otherwise NA. Used to avoid proposing a per-package fix for a package that +# only fails because a shared dependency does not build (e.g. the ~73 Stan +# packages that fail while building `rstan`). Generalises the RcppParallel +# `applies_to` guard to any dependency named in the log. +failing_dependency <- function(error_text, package) { + if (length(error_text) == 0L || is.na(error_text) || !nzchar(error_text)) { + return(NA_character_) + } + x <- as.character(error_text) + # optional opening quote before the package name: apostrophe, double-quote, + # backtick, or curly quotes -- written as \u escapes so the pattern stays + # valid UTF-8 regardless of source encoding. + q <- "[\u0027\u0022\u0060\u2018\u2019]?" + name <- "([A-Za-z][A-Za-z0-9._]+)" + # Markers R/pak emit naming the package that actually failed to compile. + pats <- c( + paste0("compilation failed for package ", q, name), + paste0("Failed to build source package ", q, name), + paste0("Error in building package ", q, name), + paste0("dependenc(?:y|ies) ", q, name, q, "?[^\\n]*not available") + ) + deps <- character(0L) + for (p in pats) { + hits <- regmatches(x, gregexpr(p, x, perl = TRUE))[[1L]] + if (length(hits) > 0L) { + deps <- c(deps, sub(p, "\\1", hits, perl = TRUE)) + } + } + deps <- sub("[._]+$", "", deps) # drop a trailing sentence period (e.g. "rstan.") + deps <- setdiff(unique(deps), package) # a package failing on its OWN code is not a cascade + if (length(deps) == 0L) NA_character_ else deps[[1L]] +} + # --------------------------------------------------------------------------- # Classification # --------------------------------------------------------------------------- @@ -291,18 +329,38 @@ build_triage_report <- function( unregistered <- setdiff(pkgs, registered_pkgs) auto_proposable <- isTRUE(sig$auto) && sig$matched - # A signature whose fix is package-specific (`applies_to`) may only be - # proposed for that package. Everything else matching it is a downstream - # failure blocked on that package (e.g. RcppParallel dependents carrying - # RcppParallel's own error text) -- never propose those a bogus entry. + # Decide, per package, whether it is genuinely fixable or merely blocked on + # a dependency (so a per-package entry would be useless). Two blocking modes: + # 1. `applies_to`: a package-specific patch (e.g. RcppParallel's) is only + # valid for its own package; other matches are downstream of it. + # 2. data-driven cascade: the package's error_text shows a *different* + # package failed to compile (e.g. the ~73 Stan packages blocked on rstan). + # `blocked_map` maps a blocked package -> the dependency it waits on. + g$blocked_dep <- vapply( + seq_len(nrow(g)), + function(i) failing_dependency(g$error_text[[i]], g$name[[i]]), + character(1L) + ) proposable <- unregistered - blocked_on <- NULL + blocked_map <- list() if (!is.null(sig$applies_to)) { - proposable <- intersect(unregistered, sig$applies_to) - downstream <- setdiff(pkgs, sig$applies_to) - if (length(downstream) > 0L) { - blocked_on <- sig$applies_to + for (p in setdiff(pkgs, sig$applies_to)) { + blocked_map[[p]] <- sig$applies_to } + proposable <- intersect(proposable, sig$applies_to) + } + for (p in proposable) { + deps <- g$blocked_dep[g$name == p] + if (!any(is.na(deps))) { + # every failing build of p is a cascade -> blocked, not fixable here + blocked_map[[p]] <- unique(deps) + } + } + proposable <- setdiff(proposable, names(blocked_map)) + blocked_packages <- names(blocked_map) + blocked_on <- unique(unlist(blocked_map, use.names = FALSE)) + if (length(blocked_on) == 0L) { + blocked_on <- NULL } proposed <- list() @@ -326,6 +384,7 @@ build_triage_report <- function( suggested_fix = sig$fix, applies_to = sig$applies_to, blocked_on = blocked_on, + blocked_packages = blocked_packages, fingerprint = names(fp_tab)[[1L]], fingerprint_variants = length(fp_tab), build_count = nrow(g), diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R index 51f5bc7..ce7b683 100644 --- a/local/failing-builds-report.R +++ b/local/failing-builds-report.R @@ -194,15 +194,17 @@ for (r in report) { ) cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "") } - } else if (!is.null(r$blocked_on)) { + } else if (r$auto_proposable && length(r$blocked_packages) == 0L) { + cat(" (all affected packages already have a registry entry)\n") + } + # Blocked packages are shown even when the group also has proposals. + if (length(r$blocked_packages) > 0L) { cat(sprintf( - " (blocked on %s -- these %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", + " (blocked on %s -- %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", toString(r$blocked_on), - length(r$packages), + length(r$blocked_packages), toString(r$blocked_on) )) - } else if (r$auto_proposable) { - cat(" (all affected packages already have a registry entry)\n") } } diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 80c4ab2..9568d78 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -160,17 +160,22 @@ unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) { ) } -# Groups blocked on a dependency (a package-specific fix pinned via `applies_to` -# whose dependents merely carry its error): report the dependency + how many -# dependents wait on it, so fixing it once is recognised as clearing the batch. +# Groups with packages blocked on a dependency (a package-specific fix pinned +# via `applies_to`, or a data-driven cascade where the package fails building a +# dependency): report the dependency + how many dependents wait on it, so fixing +# it once is recognised as clearing the batch. blocked_summary <- function(report, max_pkgs = 15L) { - bl <- Filter(function(g) !is.null(g$blocked_on), report) + bl <- Filter( + function(g) length(g$blocked_packages %||% character(0L)) > 0L, + report + ) lapply(bl, function(g) { + pkgs <- g$blocked_packages list( blocked_on = g$blocked_on, - n_packages = length(g$packages), - packages = utils::head(g$packages, max_pkgs), - packages_truncated = length(g$packages) > max_pkgs + n_packages = length(pkgs), + packages = utils::head(pkgs, max_pkgs), + packages_truncated = length(pkgs) > max_pkgs ) }) } diff --git a/local/propose-patches.R b/local/propose-patches.R index 7008251..cfce338 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -150,14 +150,17 @@ for (r in report) { # Groups blocked on a dependency (e.g. RcppParallel dependents) are reported, # not proposed: fixing the named dependency clears them all at once. -blocked <- Filter(function(r) !is.null(r$blocked_on), report) +blocked <- Filter( + function(r) length(r$blocked_packages) > 0L, + report +) if (length(blocked) > 0L) { cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") for (r in blocked) { cat(sprintf( " %s: %d package(s) fail because %s does not build\n", toString(r$blocked_on), - length(r$packages), + length(r$blocked_packages), toString(r$blocked_on) )) } diff --git a/local/tests/test-failing-builds-classify.R b/local/tests/test-failing-builds-classify.R index a365e14..d4df9f5 100644 --- a/local/tests/test-failing-builds-classify.R +++ b/local/tests/test-failing-builds-classify.R @@ -105,6 +105,49 @@ test_that("RcppParallel dependents are blocked, not proposed a per-package patch expect_identical(grp$blocked_on, "RcppParallel") }) +test_that("failing_dependency names the dependency that actually failed", { + # A leaf package (BFpack) that fails building its rstan dependency. + txt <- paste( + "Error in installing dependencies for package BFpack with tag 1.6.1", + "Failed to build source package rstan.", + "ERROR: compilation failed for package ‘rstan’", + sep = "\n" + ) + expect_identical(failing_dependency(txt, "BFpack"), "rstan") + # A package failing in its OWN compilation is not a cascade. + own <- "ERROR: compilation failed for package ‘BFpack’" + expect_true(is.na(failing_dependency(own, "BFpack"))) + expect_true(is.na(failing_dependency(NA_character_, "x"))) + expect_true(is.na(failing_dependency("some unrelated error", "x"))) +}) + +test_that("Stan packages blocked on rstan are not proposed a per-package entry", { + # BFpack/GMLTM fail building rstan; the tbb error is in rstan's compile. + cascade <- paste( + "Failed to build source package rstan.", + "fatal error: tbb/tbb_stddef.h: No such file or directory", + sep = "\n" + ) + failures <- data.frame( + name = c("BFpack", "GMLTM", "someOwnPkg"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = c( + cascade, + cascade, + # someOwnPkg fails in its OWN compile on the same header -> fixable. + "someOwnPkg.cpp: fatal error: tbb/tbb_stddef.h: No such file or directory" + ), + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + grp <- Filter(function(g) g$signature == "tbb-stddef-removed", report)[[1L]] + # Only the own-compile package is proposed; the rstan cascades are blocked. + expect_identical(names(grp$proposed_entries), "someOwnPkg") + expect_setequal(grp$blocked_packages, c("BFpack", "GMLTM")) + expect_identical(grp$blocked_on, "rstan") +}) + test_that("RcppParallel itself is still proposed when it is the failing package", { failures <- data.frame( name = "RcppParallel", From 118a92889f323f01363c7eed9573e203666ee8a0 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 16 Jul 2026 08:24:03 +0000 Subject: [PATCH 015/118] fix(local): make the trial-build gate detect non-throwing build failures (#130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Critical: the gate was false-green The latest run printed **`3/3 passed on alpine-324`** while all three builds actually **failed** (their `rstan` dependency won't compile). A false-green gate would let broken registry entries merge — worse than no gate. Root cause: `bincraft::build_binary_package()` catches build failures internally and **returns `"error"`** for the failed tag rather than throwing. The gate's `tryCatch` only treated a *thrown* exception as failure, so every non-throwing failure looked like a pass. Fix: inspect the return value. A tag passes only if the flattened result is non-empty and contains no `"error"` sentinel; a thrown error still counts as failure. Verified the verdict against `error`/`skipped`/`TRUE`/`list(success=TRUE)`/`NULL`/mixed inputs. With this, the current rstan-blocked entries will correctly show **0/3 (red)** — which is the right answer until rstan builds. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/130 --- local/trial-build-registry.R | 39 +++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/local/trial-build-registry.R b/local/trial-build-registry.R index 0749bf7..9708408 100644 --- a/local/trial-build-registry.R +++ b/local/trial-build-registry.R @@ -117,23 +117,34 @@ results <- vapply( pkgs, function(pkg) { cat(sprintf("\n=== trial build: %s ===\n", pkg)) - tryCatch( - { - bincraft::build_binary_package( - pkg, - tag_limit = 1L, - patches = patches_dir, - archive = FALSE, - upload = FALSE, - store_build_metadata = FALSE - ) - TRUE - }, + # bincraft::build_binary_package() catches build failures internally and + # RETURNS "error" for the failed tag rather than throwing, so a green gate + # must inspect the return value -- checking only for a thrown exception + # reports a broken build as passing. + res <- tryCatch( + bincraft::build_binary_package( + pkg, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ), error = function(e) { - cat(sprintf("FAILED %s: %s\n", pkg, conditionMessage(e))) - FALSE + cat(sprintf("FAILED %s (threw): %s\n", pkg, conditionMessage(e))) + "error" } ) + flat <- as.character(unlist(res)) + ok <- length(flat) > 0L && !("error" %in% flat) + if (!ok) { + cat(sprintf( + "FAILED %s: build did not succeed (result: %s)\n", + pkg, + if (length(flat) > 0L) toString(flat) else "" + )) + } + ok }, logical(1L) ) From 6c03f278ecac4d3df58d1d8674259eee45dfe6fd Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 16 Jul 2026 21:24:03 +0000 Subject: [PATCH 016/118] feat(local): aggregate blocked-on-dependency reporting by dependency (#131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why With cascade detection (#128) live, the latest `auto-apply-patches` run did exactly the right thing — **proposed nothing** (`No auto-proposable candidates`) because every failure is a dependency cascade, and it surfaced the ~30 root-cause dependencies to fix. But the "Blocked on a dependency" list printed **one line per fingerprint group**, so the same dependency repeated (rstan ×4, lpsymphony ×4, salso ×2, BH ×2, GO.db ×2, RcppCWB ×2, …), burying the priority. ## What Aggregate blocked packages across all groups **by the dependency they wait on**: - Expose `blocked_map` (package → dependency) from `build_triage_report()`. - Add `blocked_by_dependency()` — dedupes dependents (a package in two groups counts once) and ranks dependencies by how many distinct dependents they block. - Proposer and tracker (log + issue) now print one line per dependency, sorted by impact. Replaces the per-group `blocked_summary`. ## Result (same data, aggregated) ``` Blocked on a dependency (3 dependencies block 6 dependents; fix the dependency, not each dependent): RcppParallel 3 dependent(s) rstan 2 dependent(s) sf 1 dependent(s) ``` So the real run becomes a crisp, ranked worklist: RcppParallel (894), sf (128), rstan (~96), Rfast (33), clarabel/DescTools (26), Rglpk (22), xgboost (18), … ## Verified New test covers cross-group aggregation, dedup (a dependent in two groups counted once), the example cap, and ranking. 112 tests pass; hooks pass. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/131 --- local/failing-builds-classify.R | 1 + local/proposal-tracking-lib.R | 38 ++++++++++++++++-------- local/proposal-tracking.R | 8 ++--- local/propose-patches.R | 27 ++++++++--------- local/tests/test-proposal-tracking-lib.R | 33 ++++++++++++++------ 5 files changed, 68 insertions(+), 39 deletions(-) diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index 716fab2..73922b3 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -385,6 +385,7 @@ build_triage_report <- function( applies_to = sig$applies_to, blocked_on = blocked_on, blocked_packages = blocked_packages, + blocked_map = if (length(blocked_map) > 0L) blocked_map else NULL, fingerprint = names(fp_tab)[[1L]], fingerprint_variants = length(fp_tab), build_count = nrow(g), diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 9568d78..28f65db 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -160,24 +160,38 @@ unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) { ) } -# Groups with packages blocked on a dependency (a package-specific fix pinned -# via `applies_to`, or a data-driven cascade where the package fails building a -# dependency): report the dependency + how many dependents wait on it, so fixing -# it once is recognised as clearing the batch. -blocked_summary <- function(report, max_pkgs = 15L) { - bl <- Filter( - function(g) length(g$blocked_packages %||% character(0L)) > 0L, - report - ) - lapply(bl, function(g) { - pkgs <- g$blocked_packages +# Aggregate blocked packages across ALL groups by the dependency they wait on, +# so one dependency (RcppParallel, rstan, sf, ...) is a single line -- deduped +# and ranked by how many distinct dependents it blocks -- instead of repeating +# once per fingerprint group. Reads each group's `blocked_map` (package -> the +# dependency it is blocked on). Returns records sorted by dependent count desc, +# each with up to `max_pkgs` example dependents. +blocked_by_dependency <- function(report, max_pkgs = 15L) { + acc <- list() # dependency -> character vector of dependent packages + for (g in report) { + bm <- g$blocked_map + if (is.null(bm) || length(bm) == 0L) { + next + } + for (pkg in names(bm)) { + for (dep in as.character(unlist(bm[[pkg]]))) { + acc[[dep]] <- unique(c(acc[[dep]], pkg)) + } + } + } + if (length(acc) == 0L) { + return(list()) + } + out <- lapply(names(acc), function(dep) { + pkgs <- acc[[dep]] list( - blocked_on = g$blocked_on, + dependency = dep, n_packages = length(pkgs), packages = utils::head(pkgs, max_pkgs), packages_truncated = length(pkgs) > max_pkgs ) }) + out[order(-vapply(out, function(x) x$n_packages, integer(1L)))] } # Does a registry entry's `platforms` apply to a build on `os` (e.g. diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R index 4d82a11..f8e1284 100644 --- a/local/proposal-tracking.R +++ b/local/proposal-tracking.R @@ -127,13 +127,13 @@ if (length(retire) > 0L) { # --------------------------------------------------------------------------- # Blind spots: failures the classifier could not auto-propose. # --------------------------------------------------------------------------- -blocked <- blocked_summary(report) +blocked <- blocked_by_dependency(report) unmatched <- unclassified_summary(report) cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") if (length(blocked) > 0L) { for (b in blocked) { - cat(sprintf(" %s: %d dependent(s) waiting\n", b$blocked_on, b$n_packages)) + cat(sprintf(" %-20s %5d dependent(s)\n", b$dependency, b$n_packages)) } } else { cat(" (none)\n") @@ -207,8 +207,8 @@ if (do_issue) { body_lines <- c( body_lines, sprintf( - "- **%s**: %d dependent(s) waiting (e.g. %s%s)", - b$blocked_on, + "- **%s**: %d dependent(s) (e.g. %s%s)", + b$dependency, b$n_packages, toString(b$packages), if (isTRUE(b$packages_truncated)) ", ..." else "" diff --git a/local/propose-patches.R b/local/propose-patches.R index cfce338..360df9f 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -148,21 +148,20 @@ for (r in report) { } } -# Groups blocked on a dependency (e.g. RcppParallel dependents) are reported, -# not proposed: fixing the named dependency clears them all at once. -blocked <- Filter( - function(r) length(r$blocked_packages) > 0L, - report -) +# Packages blocked on a dependency are reported (aggregated by dependency, +# ranked by impact), not proposed: fixing the named dependency clears the batch. +blocked <- blocked_by_dependency(report) if (length(blocked) > 0L) { - cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") - for (r in blocked) { - cat(sprintf( - " %s: %d package(s) fail because %s does not build\n", - toString(r$blocked_on), - length(r$blocked_packages), - toString(r$blocked_on) - )) + n_blocked_pkgs <- length(unique(unlist( + lapply(report, function(r) r$blocked_packages) + ))) + cat(sprintf( + "\nBlocked on a dependency (%d dependencies block %d dependents; fix the dependency, not each dependent):\n", + length(blocked), + n_blocked_pkgs + )) + for (b in blocked) { + cat(sprintf(" %-20s %5d dependent(s)\n", b$dependency, b$n_packages)) } } diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index 05cd0e9..869d2b3 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -146,20 +146,35 @@ test_that("unclassified_summary ranks unknown groups and caps output", { expect_identical(capped$dropped_groups, 1L) }) -test_that("blocked_summary lists each dependency and its dependent count", { +test_that("blocked_by_dependency aggregates across groups, deduped and ranked", { + # RcppParallel dependents split across platforms/fingerprints -> separate + # groups, but one aggregated line; a dependent seen twice is counted once. failures <- data.frame( - name = c("ACEsimFit", "AovBay", "AdaptGauss"), - platform = "ubuntu-2604", + name = c("ACEsimFit", "AovBay", "AdaptGauss", "ACEsimFit", "loner"), + platform = c( + "ubuntu-2604", + "ubuntu-2604", + "alpine-324", + "alpine-324", + "ubuntu-2604" + ), arch = "amd64", - error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + error_text = c( + rep("Error: USE_TBB=Linux is not supported on this toolchain", 4L), + "Failed to build source package rstan.\nfatal error: tbb/tbb_stddef.h" + ), stringsAsFactors = FALSE ) report <- build_triage_report(failures, registered_pkgs = character(0L)) - b <- blocked_summary(report, max_pkgs = 2L) - expect_length(b, 1L) - expect_identical(b[[1L]]$blocked_on, "RcppParallel") - expect_identical(b[[1L]]$n_packages, 3L) - expect_true(b[[1L]]$packages_truncated) + agg <- blocked_by_dependency(report, max_pkgs = 2L) + deps <- vapply(agg, function(b) b$dependency, character(1L)) + expect_true("RcppParallel" %in% deps && "rstan" %in% deps) + rcpp <- Filter(function(b) b$dependency == "RcppParallel", agg)[[1L]] + # ACEsimFit appears in two groups -> counted once (3 distinct dependents). + expect_identical(rcpp$n_packages, 3L) + expect_true(rcpp$packages_truncated) # capped at max_pkgs = 2 + # Ranked by dependent count: RcppParallel (3) before rstan (1). + expect_identical(deps[[1L]], "RcppParallel") }) test_that("entry_applies_to_os matches codename, family, and wildcard", { From 13fcad6dc4711c05813132b505c215a986f1b1df Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 17 Jul 2026 10:27:58 +0000 Subject: [PATCH 017/118] fix(crow): make the trial-build gate no-op when the auto-patch branch is absent (#132) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The trial pipeline errored: ``` git clone -q --branch auto/registry-patch-proposals ... fatal: Remote branch auto/registry-patch-proposals not found in upstream origin ``` I deleted the stale `auto/registry-patch-proposals` branch during cleanup, and — now that cascade detection (#128) makes the proposer correctly find **no candidates** — the proposer exits before ever recreating it. So the branch is legitimately absent, and the gate's hard `git clone --branch` fails. ## Fix "Branch absent" = "no pending auto-patch proposals" = **nothing to verify**, which should be a clean no-op, not a failure. Clone `main`, then fetch + checkout the patch branch only if it exists; otherwise log and `exit 0`. A present branch is verified exactly as before (checkout its tip, diff registry vs `main`). This also means the gate now runs `main`'s `trial-build-registry.R` (with the #130 fix) rather than a possibly-stale copy on the branch. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/132 --- .crow/trial-build-registry.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 602837f..3ec0417 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -121,8 +121,13 @@ steps: # with BINCRAFT_VERBOSE_PATCH_BUILD support; harmless on older versions). BINCRAFT_VERBOSE_PATCH_BUILD: 'TRUE' commands: - - git clone -q --branch ${patch_branch} https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + # Clone main, then check out the auto-patch branch if it exists. When the + # proposer had no candidates it never (re)creates that branch, so a missing + # branch means "nothing to verify" -- no-op cleanly instead of failing the + # clone. + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git fetch -q origin main + - 'if git fetch -q origin ${patch_branch}; then git checkout -q FETCH_HEAD; else echo "No ${patch_branch} branch; no pending auto-patch proposals to verify."; exit 0; fi' - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R From a30c6f953273ec6001ed7f1834f4f4f8e7427448 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 17 Jul 2026 11:52:07 +0000 Subject: [PATCH 018/118] fix(crow): probe the auto-patch branch with git ls-remote to avoid a scary fatal (#133) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #132. The no-op path already worked (the `else` branch runs and `exit 0`s), but using `git fetch` as the *existence check* printed `fatal: couldn't find remote ref auto/registry-patch-proposals` to stderr — which looks like a failure even though the step succeeds. Probe with `git ls-remote --exit-code --heads origin ${patch_branch}` instead: verified against the live repo it exits 0 for an existing branch and 2 for a missing one, **silently** (no `fatal`). Only fetch + checkout when the branch actually exists. Net: same behavior, clean log — a run with no pending auto-patch branch prints just `No ... branch; no pending auto-patch proposals to verify.` and exits 0. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/133 --- .crow/trial-build-registry.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 3ec0417..cb8df8b 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -127,7 +127,7 @@ steps: # clone. - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git fetch -q origin main - - 'if git fetch -q origin ${patch_branch}; then git checkout -q FETCH_HEAD; else echo "No ${patch_branch} branch; no pending auto-patch proposals to verify."; exit 0; fi' + - 'if git ls-remote --exit-code --heads origin ${patch_branch} >/dev/null 2>&1; then git fetch -q origin ${patch_branch} && git checkout -q FETCH_HEAD; else echo "No ${patch_branch} branch; no pending auto-patch proposals to verify."; exit 0; fi' - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R From b4b7d53a832fc8d041849169fc94077679c88c1c Mon Sep 17 00:00:00 2001 From: pat-s Date: Sat, 18 Jul 2026 08:37:04 +0000 Subject: [PATCH 019/118] fix(local): make trial-build-patch.R detect non-throwing build failures (#134) `local/trial-build-patch.R` -- the single-package acceptance gate the proposer prints for humans to run (`PGPASS=... Rscript local/trial-build-patch.R `) -- reported **`Trial build OK`** for rstan even though rstan.so failed to load (`symbol not found: tbb::detail::r1::observe`). Same false-green as the registry gate before #130: `build_binary_package()` catches failures internally and returns `"error"` rather than throwing, so a `tryCatch` that only treats a thrown exception as failure passes a broken build. Now inspects the return value (mirrors trial-build-registry.R). Found while verifying the rstan/oneTBB unblock. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/134 --- local/trial-build-patch.R | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/local/trial-build-patch.R b/local/trial-build-patch.R index 4e8030a..2229074 100644 --- a/local/trial-build-patch.R +++ b/local/trial-build-patch.R @@ -40,23 +40,26 @@ cat(sprintf( patches_dir )) -ok <- tryCatch( - { - bincraft::build_binary_package( - package, - tag_limit = 1L, - patches = patches_dir, - archive = FALSE, - upload = FALSE, - store_build_metadata = FALSE - ) - TRUE - }, +# build_binary_package() catches build failures internally and RETURNS "error" +# for the failed tag rather than throwing (bincraft >= v4.4.7), so inspect the +# return value -- checking only for a thrown exception reports a broken build as +# passing (false green). +res <- tryCatch( + bincraft::build_binary_package( + package, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ), error = function(e) { - cat(sprintf("Trial build FAILED: %s\n", conditionMessage(e))) - FALSE + cat(sprintf("Trial build FAILED (threw): %s\n", conditionMessage(e))) + "error" } ) +flat <- as.character(unlist(res)) +ok <- length(flat) > 0L && !("error" %in% flat) if (ok) { cat(sprintf("Trial build OK: %s builds with the proposed patch.\n", package)) From 6df945d1472beb6fd3f0372a31085af38225b119 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 20 Jul 2026 09:56:21 +0000 Subject: [PATCH 020/118] fix(patches): build RcppParallel against system oneTBB instead of disabling TBB (#135) ## What Replace RcppParallel's `disable-tbb` registry patch with `system-tbb`: build RcppParallel against the system oneTBB instead of stripping TBB entirely. ## Why `disable-tbb` skipped the bundled Intel TBB build by forcing the TinyThread backend, which also removed RcppParallel's TBB linkage. That broke every dependent that links TBB through `RcppParallelLibs()` -- `rstan` and the whole Stan cluster -- with `symbol not found: tbb::detail::r1::observe`. This is the root cause behind the large "blocked on RcppParallel" / rstan clusters (issues #115, #120): the packages were not individually broken, they were all waiting on one TBB-linkage regression. ## Change `system-tbb.patch` leaves `USE_TBB` unset (so the bundled build is still skipped on musl and g++ 15) but keeps the TBB backend and links the system oneTBB now shipped in the build-env images: ``` PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=1 -DTBB_SUPPRESS_DEPRECATED_MESSAGES=1 -DTBB_INTERFACE_NEW PKG_LIBS += -ltbb -ltbbmalloc ``` Scoped to `alpine` + `ubuntu-2604`, the only platforms where the bundled build fails; redhat and older ubuntus keep the bundled TBB. ## Requires The companion image change that exports `TBB_INC`/`TBB_LIB` so `RcppParallelLibs()` hands the system-TBB flags to dependents: build-env-images PR #12. Both must ship together. ## Verified In `build-env-alpine:3.24` with the new images: - RcppParallel builds against oneTBB 2022, no ABI errors, patch applies cleanly to the target clone. - `RcppParallelLibs()` returns `-L/usr/lib -Wl,-rpath,/usr/lib -ltbb -ltbbmalloc`. - rstan links (`-ltbb -ltbbmalloc`) and loads with no missing symbol -- `* DONE (rstan)`, trial build exit 0. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/135 --- local/patches/RcppParallel/disable-tbb.patch | 16 ---------------- local/patches/RcppParallel/system-tbb.patch | 19 +++++++++++++++++++ local/patches/registry.json | 17 ++++++++++++----- 3 files changed, 31 insertions(+), 21 deletions(-) delete mode 100644 local/patches/RcppParallel/disable-tbb.patch create mode 100644 local/patches/RcppParallel/system-tbb.patch diff --git a/local/patches/RcppParallel/disable-tbb.patch b/local/patches/RcppParallel/disable-tbb.patch deleted file mode 100644 index 0fe8ad9..0000000 --- a/local/patches/RcppParallel/disable-tbb.patch +++ /dev/null @@ -1,16 +0,0 @@ -diff --git a/src/Makevars.in b/src/Makevars.in -index be8445f..faee771 100644 ---- a/src/Makevars.in -+++ b/src/Makevars.in -@@ -60,7 +60,10 @@ else - endif - - ifeq ($(UNAME), Linux) -- USE_TBB=Linux -+ # bincraft patch: the bundled Intel TBB build hangs/fails on musl (Alpine) -+ # and newer toolchains (g++ 15). Skip it (leave USE_TBB unset) and force the -+ # TinyThread backend so RcppParallel still builds. -+ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=0 - endif - - ifeq ($(UNAME), SunOS) diff --git a/local/patches/RcppParallel/system-tbb.patch b/local/patches/RcppParallel/system-tbb.patch new file mode 100644 index 0000000..263a593 --- /dev/null +++ b/local/patches/RcppParallel/system-tbb.patch @@ -0,0 +1,19 @@ +diff --git a/src/Makevars.in b/src/Makevars.in +index be8445f..7cb6c9e 100644 +--- a/src/Makevars.in ++++ b/src/Makevars.in +@@ -60,7 +60,13 @@ else + endif + + ifeq ($(UNAME), Linux) +- USE_TBB=Linux ++ # bincraft patch: link the system oneTBB (installed in the build-env images) ++ # instead of building the bundled Intel TBB, which fails on musl (Alpine) and ++ # newer toolchains (g++ 15). Leaving USE_TBB unset skips the bundled build; ++ # -DRCPP_PARALLEL_USE_TBB=1 keeps the TBB backend so dependents (rstan, ...) ++ # link TBB, and -ltbb/-ltbbmalloc pull the system library from default paths. ++ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=1 -DTBB_SUPPRESS_DEPRECATED_MESSAGES=1 -DTBB_INTERFACE_NEW ++ PKG_LIBS += -ltbb -ltbbmalloc + endif + + ifeq ($(UNAME), SunOS) diff --git a/local/patches/registry.json b/local/patches/registry.json index e53f1c9..c446fda 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -2,17 +2,22 @@ { "package": "RcppParallel", "versions": "*", - "platforms": ["alpine", "ubuntu-2604"], + "platforms": [ + "alpine", + "ubuntu-2604" + ], "env": {}, "configure_args": [], "makevars": {}, - "patch": "RcppParallel/disable-tbb.patch", - "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); patch unsets USE_TBB and forces -DRCPP_PARALLEL_USE_TBB=0 so RcppParallel skips the bundled build and uses the TinyThread backend" + "patch": "RcppParallel/system-tbb.patch", + "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); link the system oneTBB (now in the build-env images) instead, keeping the TBB backend so dependents (rstan, ...) link TBB" }, { "package": "fs", "versions": "*", - "platforms": ["*"], + "platforms": [ + "*" + ], "env": {}, "configure_args": [], "makevars": {}, @@ -22,7 +27,9 @@ { "package": "rstan", "versions": "*", - "platforms": ["*"], + "platforms": [ + "*" + ], "env": {}, "configure_args": [], "makevars": { From 61e1c0188d79f8dd03b5a523ef5e0a1542a469ef Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 20 Jul 2026 15:24:17 +0200 Subject: [PATCH 021/118] chore: adjust weekly audit runs --- .crow/weekly-audit-missing.yaml | 36 ++++++++++++++++++++----------- .crow/weekly-rebuild-missing.yaml | 26 +++++++++++++++++++--- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 6ff71e3..d66a37b 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -11,12 +11,12 @@ variables: description: "Manual run target: a specific -, or 'all' for every os/arch." options: - all - - alpine-321-amd64 - - alpine-321-arm64 - alpine-322-amd64 - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -27,6 +27,8 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 default: all when: @@ -42,27 +44,27 @@ labels: matrix: include: - - OS: alpine-321 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: alpine-321 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 @@ -106,6 +108,14 @@ matrix: ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute steps: - name: 'Audit missing binaries' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 934c518..53f178f 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -16,6 +16,8 @@ variables: - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -26,6 +28,8 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 default: all when: @@ -44,16 +48,24 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 @@ -97,6 +109,14 @@ matrix: ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute steps: - name: 'Rebuild missing binaries' From 3cc323d99e4de7ad3261f926a9da28d54ed3ff2f Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 20 Jul 2026 20:59:47 +0000 Subject: [PATCH 022/118] fix(patches): widen RcppParallel system-tbb scope to all oneTBB-2021 platforms (#137) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Widen the RcppParallel `system-tbb` registry patch scope from `[alpine, ubuntu-2604]` to `[alpine, ubuntu, redhat-10]`. ## Why #135 fixed the TBB cluster on alpine + ubuntu-2604 only. A per-image audit showed the problem is broader: RcppParallel's bundled Intel TBB build fails on **every** build image with a modern toolchain (alpine/musl, ubuntu jammy/noble/resolute g++ 11/13/15, redhat-10 GCC 14), and only redhat-8/9 (older toolchains + pre-oneTBB system TBB) tolerate the bundled build. The system-TBB fix applies wherever the image ships **oneTBB 2021+** (has the `tbb::detail::r1` namespace rstan needs): | platform | system TBB | in scope? | |---|---|---| | alpine (all) | oneTBB 2022 | ✅ | | ubuntu 2204/2404/2604 | oneTBB 2021.5 / 2021.11 / 2022 | ✅ (via `ubuntu` family token) | | redhat-10 | oneTBB 2021.11 | ✅ | | redhat-8 | TBB 2018 (classic) | ❌ pre-oneTBB → bundled | | redhat-9 | TBB 2020.3 (classic) | ❌ pre-oneTBB, bundled works → bundled | ## Requires The matching build-env-images env (`TBB_INC`/`TBB_LIB`) extended to all ubuntu + el10: build-env-images #13. Ship together. ## Verified RcppParallel + rstan build and link (no `symbol not found`) on alpine, redhat-10, ubuntu noble and ubuntu resolute; RcppParallel builds on ubuntu jammy. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/137 --- local/patches/registry.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/local/patches/registry.json b/local/patches/registry.json index c446fda..b444986 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -4,13 +4,14 @@ "versions": "*", "platforms": [ "alpine", - "ubuntu-2604" + "ubuntu", + "redhat-10" ], "env": {}, "configure_args": [], "makevars": {}, "patch": "RcppParallel/system-tbb.patch", - "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); link the system oneTBB (now in the build-env images) instead, keeping the TBB backend so dependents (rstan, ...) link TBB" + "reason": "RcppParallel bundles an old Intel TBB whose build fails on musl and modern toolchains (g++ 14/15); link the system oneTBB (2021+) shipped in the build-env images instead, keeping the TBB backend so dependents (rstan, ...) link TBB. Scoped to platforms with oneTBB 2021+ (alpine, all ubuntu, el10); el8/el9 ship pre-oneTBB TBB and are left on the bundled build." }, { "package": "fs", From 71f3f0c6011576c751fda00e64fffd059e26ab6e Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 21 Jul 2026 08:51:23 +0000 Subject: [PATCH 023/118] fix(audit): replace arch subsection in place instead of appending duplicates (#138) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem A fresh `weekly-audit-missing` run followed by `weekly-rebuild-missing` reported that almost every package "already exists in the remote bucket", even though the audit had just flagged them as missing. Root cause: the audit's Forgejo-issue update matched the existing `### ` subsection by its **bare** header (`### arm64`) while writing headers with a ` (N missing, M to rebuild)` suffix. The equality test never matched, so every run **appended** a new block instead of replacing the old one. Issue #63 had accumulated 38 arch subsections under `## alpine-323` where 2 are expected (75 total across the alpine platforms; body ~77k lines). `fetch-rebuild-packages-from-issue.R` reads the **first** matching block, which was the oldest snapshot. So the rebuild kept re-checking a months-old list (180 packages, mostly already built), while the genuinely-missing packages, ~1921 for alpine-323/arm64 in the freshest block, were never fed to the rebuild and the backlog grew silently. ## Changes - **`local/weekly-missing-binaries-audit.R`**: match arch subsections by prefix (`^### ( |$)`) and remove **all** blocks for that arch before writing one fresh block. Accumulation now self-heals on every run. - **`local/dedupe-audit-issue.R`** (new): one-off cleanup that collapses each `## platform` section to the freshest block per arch across the three OS-family issues. Supports `DRY_RUN=1`. - **`local/fetch-rebuild-packages-from-issue.R`**: prefer the audit's freshly-written RDS (overwritten each run, immune to issue-body drift), falling back to issue parsing when absent. ## Validation Simulated the dedupe logic against the live #63 body: **75 → 8** arch subsections, body 77k → 28k lines, and the kept alpine-323/arm64 block correctly resolves to `GARCH.X (3.0)` (the stale first block held `2.0`). ## Follow-ups (not in this PR) - `alpine-321` is audited but has no row in the rebuild matrix (7,230 missing, never rebuilt). - An `alpine-324` section exists in the issue but is in neither matrix. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/138 --- local/dedupe-audit-issue.R | 150 ++++++++++++++++++++++ local/fetch-rebuild-packages-from-issue.R | 31 ++++- local/weekly-missing-binaries-audit.R | 54 ++++---- 3 files changed, 211 insertions(+), 24 deletions(-) create mode 100644 local/dedupe-audit-issue.R diff --git a/local/dedupe-audit-issue.R b/local/dedupe-audit-issue.R new file mode 100644 index 0000000..c78015a --- /dev/null +++ b/local/dedupe-audit-issue.R @@ -0,0 +1,150 @@ +# One-off maintenance: collapse the duplicate arch subsections that accumulated +# in the "Missing package binaries for latest version ()" issues. +# +# A bug in weekly-missing-binaries-audit.R matched the existing "### " +# subsection by its bare header while writing headers with a +# " (N missing, M to rebuild)" suffix, so every audit run appended a fresh block +# instead of replacing it. This script rewrites each "## " section to +# keep only the *last* (freshest) block per arch. The audit fix prevents further +# accumulation; this cleans up what is already there. +# +# Env: FORGEJO_TOKEN (required). DRY_RUN=1 to preview counts without patching. + +library(httr2, quietly = TRUE) + +forgejo_base <- "https://git.devxy.io/api/v1" +repo <- "devxy/build-cran-binaries" +token <- Sys.getenv("FORGEJO_TOKEN") +dry_run <- nchar(Sys.getenv("DRY_RUN")) > 0 + +if (nchar(token) == 0) { + stop("FORGEJO_TOKEN env var is not set") +} + +issue_titles <- c( + "Missing package binaries for latest version (Alpine)", + "Missing package binaries for latest version (Ubuntu)", + "Missing package binaries for latest version (Red Hat)" +) + +# Collapse one "## " block: keep only the last block per arch, +# emitted in order of first appearance. `pl[1]` is the "## " header. +dedupe_platform <- function(pl) { + sub_hdr <- which(grepl("^### ", pl)) + if (length(sub_hdr) == 0) { + return(pl) + } + preamble <- pl[seq_len(sub_hdr[1] - 1)] + sub_end <- c(sub_hdr[-1] - 1, length(pl)) + blocks <- lapply(seq_along(sub_hdr), function(k) { + pl[seq(sub_hdr[k], sub_end[k])] + }) + arches <- vapply( + blocks, + function(b) sub("^### (\\S+).*", "\\1", b[1]), + character(1) + ) + # Index of the last block for each arch, kept in first-appearance order. + last_idx <- vapply( + unique(arches), + function(a) max(which(arches == a)), + integer(1) + ) + keep <- sort(last_idx) + out <- preamble + for (i in keep) { + out <- c(out, blocks[[i]]) + } + out +} + +process_issue <- function(title) { + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(title, reserved = TRUE) + ) + resp <- request(search_url) |> + req_headers(Authorization = paste("token", token)) |> + req_perform() + issues <- resp_body_json(resp, simplifyVector = FALSE) + match_idx <- which(vapply(issues, function(x) x$title, character(1)) == title) + if (length(match_idx) == 0) { + cat(sprintf("[skip] No issue found: %s\n", title)) + return(invisible()) + } + + issue_number <- issues[[match_idx[1]]]$number + body <- issues[[match_idx[1]]]$body + if (is.null(body) || nchar(body) == 0) { + cat(sprintf("[skip] Empty body: #%d %s\n", issue_number, title)) + return(invisible()) + } + + lines <- strsplit(body, "\n", fixed = TRUE)[[1]] + before <- sum(grepl("^### ", lines)) + + # Split off the "## Excluded packages" footer so it is preserved verbatim. + excl_idx <- which(lines == "## Excluded packages") + footer <- character(0) + if (length(excl_idx) > 0) { + pre_dash <- which(lines == "---" & seq_along(lines) < excl_idx[1]) + cut <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_idx[1] + footer <- lines[seq(cut, length(lines))] + lines <- lines[seq_len(cut - 1)] + } + + # Platform headers ("## "); everything before the first is preamble. + plat_idx <- which(grepl("^## ", lines)) + if (length(plat_idx) == 0) { + cat(sprintf("[skip] No platform sections: #%d %s\n", issue_number, title)) + return(invisible()) + } + top <- lines[seq_len(plat_idx[1] - 1)] + plat_end <- c(plat_idx[-1] - 1, length(lines)) + + new_lines <- top + for (j in seq_along(plat_idx)) { + pl <- lines[seq(plat_idx[j], plat_end[j])] + new_lines <- c(new_lines, dedupe_platform(pl)) + } + if (length(footer) > 0) { + new_lines <- c(new_lines, footer) + } + + after <- sum(grepl("^### ", new_lines)) + cat(sprintf( + "#%d %s: %d -> %d arch subsections%s\n", + issue_number, + title, + before, + after, + if (dry_run) " (dry run, not patched)" else "" + )) + + if (dry_run) { + return(invisible()) + } + + patch_url <- sprintf( + "%s/repos/%s/issues/%d", + forgejo_base, + repo, + issue_number + ) + request(patch_url) |> + req_headers( + Authorization = paste("token", token), + `Content-Type` = "application/json" + ) |> + req_body_json(list(body = paste(new_lines, collapse = "\n"))) |> + req_method("PATCH") |> + req_perform() + cat(sprintf(" patched #%d\n", issue_number)) +} + +for (t in issue_titles) { + process_issue(t) +} +cat("Done.\n") diff --git a/local/fetch-rebuild-packages-from-issue.R b/local/fetch-rebuild-packages-from-issue.R index 26e544d..9d8043c 100644 --- a/local/fetch-rebuild-packages-from-issue.R +++ b/local/fetch-rebuild-packages-from-issue.R @@ -4,7 +4,6 @@ forgejo_base <- "https://git.devxy.io/api/v1" repo <- "devxy/build-cran-binaries" platform <- Sys.getenv("PLATFORM") arch <- Sys.getenv("ARCH") -token <- Sys.getenv("FORGEJO_TOKEN") output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") if (nchar(platform) == 0) { @@ -13,6 +12,36 @@ if (nchar(platform) == 0) { if (nchar(arch) == 0) { stop("ARCH env var is not set") } + +# Prefer the audit's freshly-written RDS. The audit overwrites it each run +# (saveRDS), so unlike the Forgejo issue body it is never subject to the +# duplicate-subsection accumulation bug. Fall back to parsing the issue when the +# RDS is absent (e.g. a fresh runner with no shared cache). +rds_file <- file.path( + Sys.getenv("REBUILD_PKG_RDS_DIR", "/mnt/cache/packages"), + sprintf("weekly_rebuild_%s_%s.rds", platform, arch) +) +if (file.exists(rds_file)) { + pkgs <- tryCatch(as.character(readRDS(rds_file)), error = function(e) NULL) + if (!is.null(pkgs) && length(pkgs) > 0) { + cat(sprintf( + "Using audit RDS %s: %d rebuildable packages for %s/%s\n", + rds_file, + length(pkgs), + platform, + arch + )) + writeLines(pkgs, output_file) + cat(sprintf("Wrote package list to %s\n", output_file)) + q("no") + } + cat(sprintf( + "RDS %s present but empty/unreadable -- falling back to issue\n", + rds_file + )) +} + +token <- Sys.getenv("FORGEJO_TOKEN") if (nchar(token) == 0) { stop("FORGEJO_TOKEN env var is not set") } diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index 6b32037..7b0b65d 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -328,7 +328,6 @@ if (nchar(forgejo_token) == 0) { } plat_header <- sprintf("## %s", platform) - arch_header <- sprintf("### %s", arch) plat_idx <- which(lines == plat_header) @@ -359,31 +358,40 @@ if (nchar(forgejo_token) == 0) { } plat_lines <- lines[seq(pi, plat_end)] - arch_local_idx <- which(plat_lines == arch_header) - if (length(arch_local_idx) == 0) { - # Append arch subsection at end of platform block - lines <- c( - lines[seq_len(plat_end)], - "", - arch_lines, - lines[seq(plat_end + 1, length(lines))] - ) + # Arch subsection headers within the platform block (### arm64 / ### amd64). + # Match by prefix: headers carry a " (N missing, M to rebuild)" suffix, so + # exact-equality matching never found the existing block and silently + # appended a duplicate on every run. Remove *all* blocks for this arch + # (collapsing any previously accumulated duplicates), then write one fresh + # block, so the issue holds a single current subsection per arch. + sub_hdr <- which(grepl("^### ", plat_lines)) + arch_re <- sprintf("^### %s( |$)", arch) + + if (length(sub_hdr) == 0) { + # No arch subsections yet -- append after the platform header/preamble. + new_plat_lines <- c(plat_lines, "", arch_lines) } else { - ai <- pi + arch_local_idx[1] - 1 # absolute line index - - # End of arch subsection - next_arch <- which( - grepl("^### |^## |^---", lines) & seq_along(lines) > ai - ) - arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end - - lines <- c( - lines[seq_len(ai - 1)], - arch_lines, - lines[seq(arch_end + 1, length(lines))] - ) + preamble <- plat_lines[seq_len(sub_hdr[1] - 1)] + # Each subsection runs from its ### header to the line before the next + # ### header (#### known-failures stays inside its own block). + sub_end <- c(sub_hdr[-1] - 1, length(plat_lines)) + kept <- character(0) + for (k in seq_along(sub_hdr)) { + block <- plat_lines[seq(sub_hdr[k], sub_end[k])] + if (!grepl(arch_re, block[1])) { + kept <- c(kept, block) + } + } + new_plat_lines <- c(preamble, kept, "", arch_lines) } + + tail_lines <- if (plat_end < length(lines)) { + lines[seq(plat_end + 1, length(lines))] + } else { + character(0) + } + lines <- c(lines[seq_len(pi - 1)], new_plat_lines, tail_lines) } # Rebuild excluded footer From 736564a4ff39c7f46b0c72a6bc742062deca9694 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 24 Jul 2026 11:18:28 +0200 Subject: [PATCH 024/118] fix: prevent renovate bumps for fixed matrix assignments --- .crow/process-updates.yaml | 8 ++++---- .crow/trial-build-registry.yaml | 8 ++++---- renovate.json | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 3da41fd..59b1c26 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -47,25 +47,25 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 OS_ID: alpine322 PROCESS_NEW: "FALSE" - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 OS_ID: alpine322 PROCESS_NEW: "FALSE" - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 OS_ID: alpine323 PROCESS_NEW: "FALSE" - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 OS_ID: alpine323 PROCESS_NEW: "FALSE" - OS: alpine-324 diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index cb8df8b..73856a1 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -29,19 +29,19 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/renovate.json b/renovate.json index 53c7306..8c8e208 100644 --- a/renovate.json +++ b/renovate.json @@ -1,7 +1,7 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": ["local>devxy/renovate-config"], - "ignorePaths": ["docker/**"], + "ignorePaths": ["docker/**", ".crow/process-updates.yaml", ".crow/build-all-versions.yaml", ".crow/weekly-rebuild-missing.yaml"], "customManagers": [ { "customType": "regex", From 36c963eac9b13eb525a4aa2a14f5dea2494cfe28 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 28 Jul 2026 00:32:39 +0000 Subject: [PATCH 025/118] chore(deps): update pre-commit hook davidanson/markdownlint-cli2 to v0.23.2 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0f4422..df8af14 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.23.0 + rev: v0.23.2 hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier From adcc74ba4fec1c02d51778c36532cd72058f74ce Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 28 Jul 2026 00:32:55 +0000 Subject: [PATCH 026/118] chore(deps): update pre-commit hook rbubley/mirrors-prettier to v3.9.6 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df8af14..cee748e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.4 + rev: v3.9.6 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit From c7293aefa1bde9ed214eec9702f63c25e4244f2f Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 28 Jul 2026 02:33:47 +0000 Subject: [PATCH 027/118] chore(deps): update terraform bunnynet to v0.15.2 --- .terraform.lock.hcl | 70 ++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 6ef10fa..b18d5af 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.15.1" + version = "0.15.2" constraints = "~> 0.15" hashes = [ - "h1:/2NUpbtjkc+w6n5V3kGP0rSzGjN0K2Wdfe2K+CZdmhU=", - "h1:1TOrpmCR0aT5xX1sjt70zqlYkMJnVe6r0nx2B0DS/mE=", - "h1:4uC8r8ILr+vZJ230uGqIUaWQ7uORCrIzoEuYrAq2JuE=", - "h1:6b+2osJUfwcaYZ6mathLPf/58sr/4XkLXQrcSufnkMk=", - "h1:CPldfjf79QS8mIP+GWoS0FVhrFlyjvN2+39zfyP76Ik=", - "h1:EvXICHyGIKpoYlDeKHOPzfmpvdRdhgsOMcR2nb6+tKY=", - "h1:GZLq+nDxS1CyBH0ELGTSQj9X7ozemJ1jpPA4KwbtR+c=", - "h1:ISNFqL745IQgZ6yMLy8ofV8ixbYqZYa9JKdi2W3pmNk=", - "h1:IrNrEuvFd0nYDGQefwmT8d1CSJb9e8LN5w9vw1ODp7E=", - "h1:Ms79slY9bZ94+n4cwIHvI9+/cvbucwo6S1+z5KAiznw=", - "h1:NWA9XSEBcpSkgwwIvl6tHrxGQY3uYhqNS4Vnb9RLyLQ=", - "h1:UjvxxxggicLtiE3yTe1Gx0oLUTeZpWmgIfXuHzWHn1c=", - "h1:VgJjo14DGkU4Jwo4D3GT4/5sq1tdjiZscKS5l3cb890=", - "h1:dBu3AW5YNLIvbBIMNk3wUHKw4TW+BDbj34a+mCqYhWE=", - "h1:i5oGD06nQ3JRsBIa2u4wCej+ETgp970CFl75dOKkHno=", - "h1:mTqR+vD1AWPx+mu7S0/pzBy71z7WOrjH7arOP77PXh4=", - "h1:nmTM61G8vYjpofeEqspMORpsNvTGCNZySGfjdXardL0=", - "zh:0f9bf5aaa47164a4d6ae4433d5e285a9456a5053401b2bad4ed68622f574ddee", - "zh:3039bee421fb8855a919f449fc731d145254371f5f0c39cc4660f3aeed6a8b10", - "zh:36664b08186e0c194747b18dee24ed97327c6e704133d4cf0df27abef1652f86", - "zh:3c7eae99d8c5ff65dfb99c8b9c1980147282d68971e1ef1ff0f126a6bff59d8f", - "zh:5293cc21abf54f4e5745437ca2d40d206aae323b2e1d41cf45dcc63a8868cbb3", - "zh:5994e5145e616e7e881010717e4c7def2945eb6d933f62db4ec3167732ccac84", - "zh:7994db9ed3fdb6cbf21f2154ea962cf82f04e425988bedc6657de2497d6cb6c3", + "h1:1OktRVcHWTvR/KKK4UWVm49JKibjyKOu0Epew+TawZI=", + "h1:4L1bPF0zW5z7Y6BEuPLXWP1RiOr/V8G5dhaRStV1yQM=", + "h1:4fpJGk4TRnWa0ZsNxBLWLlFgOb5WLyohpJ61Lsk5D0Y=", + "h1:7JTOjvrbATIKRCI+NKXCTMMSmUHYnKgsCsbKV+peAFM=", + "h1:8NDV2elfSxmixLUDqGpCSkKVm1pk7hgToP/L5bmEb4g=", + "h1:CYoaY53mSTjxq8nTMXuu7ET53q4Qe1+M6SfH9UaxE80=", + "h1:JbepdZF6DL97FUxGUI0Fi4sH9txTgzYk5q89Tzltifo=", + "h1:T6u0dqstAgR3L+8oPlNfcmOHVYZjIJ3UNvgdUyucpNA=", + "h1:U0Benusw0c0fJSdFvICBxuBxV1nAVOplY+AfLuda6ds=", + "h1:XZkiPAERPz4PKOMqohfMsA1qlnGhb2++R7rbMYAiDpo=", + "h1:Znc6swnxXdPeMarUgImUtbmw2iP83xVWUb4ShJC+F+U=", + "h1:aTJTUfP8tC71u1qgf4fqKv7RWd5dDTSpRQvD2Co2tWY=", + "h1:fozSGXAjeabAda2lqjXxqZ4haDE/QQAR1T6wV/t/NNw=", + "h1:pVBNrDoiSd5Pmf0lNa0uVFMl2u447kT0iXhDBT67mXk=", + "h1:rLnEQPCr3YbxzUWgmBITbyXzNaraTJGaW0mNEANzOPQ=", + "h1:wNSoVvpO1Y9vFTx3XbWYW5WSb2Go6EoRyci6l/kgBPE=", + "h1:x2gE2pPOUypQL5nCIak13MMqmGIlozwVpaaXfdZakSw=", + "zh:03acea3774550e25e7bf214ea3a8a9163b68d2009eb613c71d7f57c333cd8416", + "zh:13acba069e86d6a5d4d176952950c77fd8bd81629ed6b861349e5f74b0e78cb8", + "zh:275b9567ea05d5a5f0fb4d846032ed01482220a290aeb5d28e0bfa159779e3db", + "zh:444b008de5cbc7275f6d4230219a464e7b97daeea938223d693a3161c1158caf", + "zh:533b4cea6b5aa2351335453f45406eadb6d4485d5c339a0d5d5e0d84df21c22b", + "zh:55f0323631a34d1aa96178d08ec7eb7ec317306886daa00fd3a7dca61b00315b", + "zh:6e077b28e89f4364bbddf4dd783fc657c13df7d1a385f04629ba8deb1eb0a332", + "zh:6e10e279e7c1d44d3fe287c033d10068b5194621f6da03f28fa7b91fa4f2ef19", + "zh:74c1d79356f2c213ef19c7b478162c4f77ab8b840438b83701088554bd4090a1", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:93aa863e536ba9376ccf9e614e9edc9b214a2ce8c4316d416a8e249b436f52d2", - "zh:ab5cb4baeda57559686a0ccf0e09158aa64624ee6ba0ef32b769f13b11a43068", - "zh:ae9388b62eede8fd9272407bf75f8241a965bd489d45ec9dd3f9fac696d500e1", - "zh:caa5befd16960e2f69c7ec483e228e5ff43ab0979c17f1b874c9ffaa1c7c0e43", - "zh:cddd3e1067defa06a4e4ad5cb3940c7943e29c42417de57236aa7d3e2aeaae13", - "zh:cdfa44d591d0805116159556947904d70f534c6816188c45cf3a7544d2722ac9", - "zh:d607e9f1f3e09f13404f219e1893e3b3c77aece4afb54e999f934c021f41f576", - "zh:d8a397aca95125c6a0c0c78d2ded5843b9204effa9f5cf7419f017b500ad9228", - "zh:f5499eaff0d221725ad209d27d87c5b46d5c554caad7dc42947c760377abe3b0", - "zh:fc5f5cf433abc83e5169fa222992ec521c0c802075970251e3dd2c1d50c4f5c1", + "zh:9ebabdd167d1fd13d357517007027ad34acb7db130fdc75fd3b2652560c0399e", + "zh:a185124d2b93dff80203074c0f4a9dea8f21965c3cbc0592929b4ae58692b045", + "zh:a46483a7cd69699488c82406519769361b65eb59c39ab1bfd42f4d84d1bd30fd", + "zh:b29e2807adf9a0965f33246c59eb04bc2ac089fe44f2e439ece7c53bdee033a1", + "zh:b550a4c5eb7b17ca85b2f76c4b10bf0523b9c22f6104fe9fd5dce7b3c3bbbf37", + "zh:b9a9685c9fa99674ddbb11633bb9648152f8fbf92881deae1dd28ebca95b150e", + "zh:e412569cf41ebb5106d034f148df3968fafcf75de804713a20bee7d3d4e74c36", + "zh:fb639b074ee65c08590b971866e10e61aa85c7937a7b7f941899ac8cbcd9157d", ] } From 463054625afaa404a8f5a93e362957785836efab Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 28 Jul 2026 02:34:05 +0000 Subject: [PATCH 028/118] chore(deps): update pre-commit hook posit-dev/air-pre-commit to v0.11.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cee748e..140ae88 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit - rev: 0.10.0 + rev: 0.11.0 hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker From 8d0c4b7ccdca01863be1c6976d154a0074c94a36 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 28 Jul 2026 06:57:08 +0000 Subject: [PATCH 029/118] fix(patches): widen RcppParallel system-tbb scope to the whole redhat family (#139) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends #137. el8/el9 were excluded because they ship only classic Intel TBB. build-env-images now vendors oneTBB 2021 into `/usr/local` on el8/el9, so every platform has oneTBB 2021+. Widen the scope `redhat-10` → `redhat` (family token covers el8/el9/el10) and fix the reason text (no longer 'left on bundled'). **Requires the el8/el9 images rebuilt with the vendored oneTBB (build-env-images PR).** Verified: patched RcppParallel builds + links the vendored oneTBB on both el8 (g++ 8.5) and el9 (g++ 11.5). Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/139 --- local/patches/registry.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/local/patches/registry.json b/local/patches/registry.json index b444986..99b80e3 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -5,13 +5,13 @@ "platforms": [ "alpine", "ubuntu", - "redhat-10" + "redhat" ], "env": {}, "configure_args": [], "makevars": {}, "patch": "RcppParallel/system-tbb.patch", - "reason": "RcppParallel bundles an old Intel TBB whose build fails on musl and modern toolchains (g++ 14/15); link the system oneTBB (2021+) shipped in the build-env images instead, keeping the TBB backend so dependents (rstan, ...) link TBB. Scoped to platforms with oneTBB 2021+ (alpine, all ubuntu, el10); el8/el9 ship pre-oneTBB TBB and are left on the bundled build." + "reason": "RcppParallel bundles an old Intel TBB whose build fails on musl and modern toolchains (g++ 8-15 + modern make); link the system oneTBB (2021+) instead, keeping the TBB backend so dependents (rstan, ...) link TBB. All build-env platforms now provide oneTBB 2021+: native on alpine/ubuntu/el10, built from source into /usr/local on el8/el9 (their stock TBB is classic 2018/2020, too old)." }, { "package": "fs", From 86707e3de4f69e912c2e060386f08dbcd4f25cb9 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Fri, 31 Jul 2026 00:32:20 +0000 Subject: [PATCH 030/118] chore(deps): update terraform bunnynet to ~> 0.16 --- .terraform.lock.hcl | 72 ++++++++++++++++++++++----------------------- provider.tf | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index b18d5af..fdf41e1 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.15.2" - constraints = "~> 0.15" + version = "0.16.0" + constraints = "~> 0.16" hashes = [ - "h1:1OktRVcHWTvR/KKK4UWVm49JKibjyKOu0Epew+TawZI=", - "h1:4L1bPF0zW5z7Y6BEuPLXWP1RiOr/V8G5dhaRStV1yQM=", - "h1:4fpJGk4TRnWa0ZsNxBLWLlFgOb5WLyohpJ61Lsk5D0Y=", - "h1:7JTOjvrbATIKRCI+NKXCTMMSmUHYnKgsCsbKV+peAFM=", - "h1:8NDV2elfSxmixLUDqGpCSkKVm1pk7hgToP/L5bmEb4g=", - "h1:CYoaY53mSTjxq8nTMXuu7ET53q4Qe1+M6SfH9UaxE80=", - "h1:JbepdZF6DL97FUxGUI0Fi4sH9txTgzYk5q89Tzltifo=", - "h1:T6u0dqstAgR3L+8oPlNfcmOHVYZjIJ3UNvgdUyucpNA=", - "h1:U0Benusw0c0fJSdFvICBxuBxV1nAVOplY+AfLuda6ds=", - "h1:XZkiPAERPz4PKOMqohfMsA1qlnGhb2++R7rbMYAiDpo=", - "h1:Znc6swnxXdPeMarUgImUtbmw2iP83xVWUb4ShJC+F+U=", - "h1:aTJTUfP8tC71u1qgf4fqKv7RWd5dDTSpRQvD2Co2tWY=", - "h1:fozSGXAjeabAda2lqjXxqZ4haDE/QQAR1T6wV/t/NNw=", - "h1:pVBNrDoiSd5Pmf0lNa0uVFMl2u447kT0iXhDBT67mXk=", - "h1:rLnEQPCr3YbxzUWgmBITbyXzNaraTJGaW0mNEANzOPQ=", - "h1:wNSoVvpO1Y9vFTx3XbWYW5WSb2Go6EoRyci6l/kgBPE=", - "h1:x2gE2pPOUypQL5nCIak13MMqmGIlozwVpaaXfdZakSw=", - "zh:03acea3774550e25e7bf214ea3a8a9163b68d2009eb613c71d7f57c333cd8416", - "zh:13acba069e86d6a5d4d176952950c77fd8bd81629ed6b861349e5f74b0e78cb8", - "zh:275b9567ea05d5a5f0fb4d846032ed01482220a290aeb5d28e0bfa159779e3db", - "zh:444b008de5cbc7275f6d4230219a464e7b97daeea938223d693a3161c1158caf", - "zh:533b4cea6b5aa2351335453f45406eadb6d4485d5c339a0d5d5e0d84df21c22b", - "zh:55f0323631a34d1aa96178d08ec7eb7ec317306886daa00fd3a7dca61b00315b", - "zh:6e077b28e89f4364bbddf4dd783fc657c13df7d1a385f04629ba8deb1eb0a332", - "zh:6e10e279e7c1d44d3fe287c033d10068b5194621f6da03f28fa7b91fa4f2ef19", - "zh:74c1d79356f2c213ef19c7b478162c4f77ab8b840438b83701088554bd4090a1", + "h1:+7y/HM0f4/cm7dpL5RulSZ3k5EDokVaIutfuanc1hhA=", + "h1:/4n/gUEzVa7LArNrNbZa4RyGC3+MRCCYTuqRqF36Z7U=", + "h1:4cSbrSalUxXnG0RVwrQG51NNleZNFU9GJxWCLK1M9WM=", + "h1:AlidPvq+LqsveQVhc9JYa7OKf4mcR+DjCzMomoBcmkw=", + "h1:Beny1tSl5O/jM7sSP7qYv+NEVTTyS/BGtSxvMFsII/o=", + "h1:E50kVGCtxWYoJgOLtdat1JFc9KCIeMByk65XSIpuaB0=", + "h1:ErNz5hzwf70NouW7a1DAknWI/0/BFUEX53hQ4BnUxWg=", + "h1:GTB8ugjpBEsRZpdVmKJw70IF2B1/fd1Gwven6XKghpk=", + "h1:JEEDgEc2gwjwfvnDwWO2ZHaf9T5WHtb7A6ABx6rQqhE=", + "h1:OVd3EgjJDuyMBFQ4XZYrs09NNEtFw4jGAbwyeos2H3M=", + "h1:U/9QeqZ+Q2MQ7eNfD1eWomLFlbYGwpliHcD/UTFe/D0=", + "h1:VPUe+m5Fx31TAR7udJPGGzk3EpCNizf0TmnKplOJyRM=", + "h1:YkJwUFpUemqi6bwRIEJW4xSpI7CAEXirqhyjl6Rmm68=", + "h1:a/15E8lijUcqvhUGxtqKmJT6jYGmQW9z7yBoOjusDm8=", + "h1:hii960NFcNpr0eZ/dcbnYOf2oZuo2IZl+z1IMTwh5YA=", + "h1:mq/KWNt6PPZLdNpO1ekzrn6EfWz6l8sTHG74dgy07nI=", + "h1:vso6Er2XV/MsUTMtXIlz0AhIFx0TZnK73YNSZfA/9ZY=", + "zh:1f581ed8bc8b676f0127af7c018912ca82972e9348370d9f4dbe5b44005bb0ec", + "zh:1fa5b9c01ffbc7fdc507bbcffc177a1f1deb9f56f3d27183c6d92a11ea3208f8", + "zh:42b6524a9658df7093199b65eb210b3246a9f9822c8778a77e2d00a222441c03", + "zh:4c4f17fe2ae86087626dfd1055b0faea6a23fe3a12aabb2a7de3b9b2df6e3e41", + "zh:5346bb67d7154a8bc8c887b4b86bef95b3e92857d4304f55afea6d56ed152932", + "zh:591df2c3d4552f6f613415c900023d43e73e00e90312b7b1a666fe567219120f", + "zh:7156849aa03348c7556b746d3d9956293678de4a9e30c80480358dcdacb2a7fa", + "zh:75dee86262352f061ad072a5b873d3b3eabbb124ab0f20a803cfd44a87a31067", + "zh:7c33954aff1878bec78a50742897fae78dd2fbd19abeaeaca0c560ccc8f78810", + "zh:7d9cf36633dd991b41891d1c91b45b5c803852a902705453d6be4513524a574c", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9ebabdd167d1fd13d357517007027ad34acb7db130fdc75fd3b2652560c0399e", - "zh:a185124d2b93dff80203074c0f4a9dea8f21965c3cbc0592929b4ae58692b045", - "zh:a46483a7cd69699488c82406519769361b65eb59c39ab1bfd42f4d84d1bd30fd", - "zh:b29e2807adf9a0965f33246c59eb04bc2ac089fe44f2e439ece7c53bdee033a1", - "zh:b550a4c5eb7b17ca85b2f76c4b10bf0523b9c22f6104fe9fd5dce7b3c3bbbf37", - "zh:b9a9685c9fa99674ddbb11633bb9648152f8fbf92881deae1dd28ebca95b150e", - "zh:e412569cf41ebb5106d034f148df3968fafcf75de804713a20bee7d3d4e74c36", - "zh:fb639b074ee65c08590b971866e10e61aa85c7937a7b7f941899ac8cbcd9157d", + "zh:9552ed1be3059b31544632d6293c2bfcd2474983e116c42702a6c408deb8e44d", + "zh:b3cb1f2bde0fd591d18109119f6235ff41c7ea6441777a356a64c348ee4ccdf7", + "zh:b5c1748403bf43274eba2574219cfa082351c8b1c1537a768acebef5cc5b2c0a", + "zh:c1b032e33aee40945770bbdf6002f777ee7b4ca541c00f4cf5eb23c09d32ef54", + "zh:d469680df9845fb506be27b2c838d39ccd8e98e26540b99936775f954299c35c", + "zh:dd59fa97fea4b10b901a6a749ebff81895f54126a2f08b4cd8f5baf6cfbbd854", + "zh:fb142d8ea02909587a4858374fe33d8609347150f4d32b7e15423320ffbd07b1", ] } diff --git a/provider.tf b/provider.tf index b267fcc..c22ce2d 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.15" + version = "~> 0.16" } } } From 98371fb9c585d9e30439c97ad08bec3d61993aff Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 31 Jul 2026 06:41:46 +0000 Subject: [PATCH 031/118] fix(patches): replace the RcppParallel system-TBB patch with a link-order fix (#145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Every build logs: ``` ! Patch for RcppParallel 6.2.0 did not apply cleanly; skipping patched build. ``` RcppParallel 6.x rewrote its build system. `src/Makevars.in` no longer contains the `USE_TBB=Linux` block that `system-tbb.patch` edited (it is now a short `@VAR@` template driven by `tools/config/configure.R`), so the patch can never apply again. The workaround it implemented is obsolete too. 6.x bundles **oneTBB 2022** and builds it with cmake, which works on musl and with g++ 8-15, so the 5.x reason for linking a system TBB is gone. Linking one is now harmful: `install.libs.R` symlinks the system libraries into `RcppParallel/lib`, so the published binary depends on a TBB the consumer does not have — the same failure mode as `fs` and libuv. What is still broken upstream is the **link order**. `configure.R` names the TBB directory with `-Wl,-L`, and gcc expands its own search dirs into `-L` options ahead of anything forwarded verbatim with `-Wl,`: ``` $ gcc -v -o t t.c -Wl,-L/usr/local/lib64 -ltbb -L/usr/lib/gcc/x86_64-redhat-linux/8 -L/lib/../lib64 -L/usr/lib/../lib64 ... -L/usr/local/lib64 ``` So on any build host with a distro TBB installed, `-ltbb` resolves to that library and `RcppParallel.so` records *its* SONAME (`libtbb.so.12`, or `libtbb.so.2` for the classic Intel TBB on el8/el9) instead of the bundled `libtbb.so`. ## Changes - Replace `RcppParallel/system-tbb.patch` with `RcppParallel/bundled-tbb-link-order.patch`, which changes the three `-Wl,-L` occurrences in `tools/config/configure.R` to plain `-L`. - Scope the RcppParallel entry to `>=6.0.0` and widen `platforms` to `*`. - Drop the `rstan` entry: its `-DTBB_INTERFACE_NEW` is already emitted by `RcppParallel::CxxFlags()` once the bundled oneTBB is used, and its `-I/usr/local/include` was an el8/el9 path applied on every platform. - `.pre-commit-config.yaml`: exclude `local/patches/*.patch` from `trailing-whitespace`, `end-of-file-fixer` and `editorconfig-checker`. They rewrite blank context lines (` ` -> ``) in every diff in the registry; `git apply` happens to tolerate it today, but a patch with meaningful trailing whitespace would be silently corrupted. The excludes are per-hook so `validate patch registry` still runs. - README: the RcppParallel example described the 5.x problem. ## Verification Built in the published images: | image | NEEDED | RPATH | loads with system libtbb removed | | --- | --- | --- | --- | | `build-env-alpine:3.24` | `libtbb.so` | `$ORIGIN/../lib` | yes | | `build-env-redhat:8` | `libtbb.so`, `libtbbmalloc.so` | `$ORIGIN/../lib` | yes | | `build-env-ubuntu:noble` | `libtbb.so` | `$ORIGIN/../lib` | yes | Without the patch the same builds record `libtbb.so.12` (alpine, ubuntu) or `libtbb.so.2` (el8, the classic 2018 TBB) and fall back to the system library at load time. rstan 2.32.7 compiles and loads against the patched RcppParallel on ubuntu noble with no makevars override and with the system libtbb moved away. On el8 it also compiles; loading it there is blocked by an unrelated image bug (see below). ## Behaviour change Requires the paired build-env-images PR (drops `TBB_INC`/`TBB_LIB`) and an image rebuild — with those env vars set, configure still takes the system-TBB branch. After that, RcppParallel binaries ship their own oneTBB and are self-contained. ## Also found, not fixed here - The `uvr lock failed ... GLIBC_2.29 not found` errors in the same log are the `-gnu` uvr artifact on el8's glibc 2.28; fixed in build-env-images. - `build-env-redhat:8` R 4.5.3/4.6.0 cannot load `stats` (`libRlapack.so: undefined symbol: dgemmtr_`): the el8 R RPM symlinks `libRblas.so` to openblas 0.3.15, which predates that symbol. R 4.4.3 and el9 are fine. Belongs in the R RPM build. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/145 --- .pre-commit-config.yaml | 9 ++++ README.md | 2 +- .../RcppParallel/bundled-tbb-link-order.patch | 46 +++++++++++++++++++ local/patches/RcppParallel/system-tbb.patch | 19 -------- local/patches/registry.json | 30 ++---------- 5 files changed, 61 insertions(+), 45 deletions(-) create mode 100644 local/patches/RcppParallel/bundled-tbb-link-order.patch delete mode 100644 local/patches/RcppParallel/system-tbb.patch diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 140ae88..80650cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,12 +7,20 @@ exclude: | benchmark/| docker/reprex/ ) +# The `^local/patches/.*\.patch$` excludes below keep unified diffs byte-exact: +# a context line for a blank line is a single space, and stripping it (or +# appending a newline) makes `git apply` reject the patch, which surfaces as +# "patch did not apply cleanly" at build time rather than as a lint failure +# here. The exclusions are per-hook, not global, so `validate patch registry` +# still runs when a patch changes. repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: end-of-file-fixer + exclude: ^local/patches/.*\.patch$ - id: trailing-whitespace + exclude: ^local/patches/.*\.patch$ args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 @@ -31,6 +39,7 @@ repos: rev: v3.8.0 hooks: - id: editorconfig-checker + exclude: ^local/patches/.*\.patch$ - repo: https://github.com/adrienverge/yamllint.git rev: v1.38.0 hooks: diff --git a/README.md b/README.md index 28541a1..538df34 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ For every package+tag combination: ## Patching packages Some CRAN packages fail to compile on specific platforms due to compiler- or OS-specific issues unrelated to the package itself. -The canonical example is `RcppParallel`, whose bundled TBB sources fail on musl (Alpine) and newer compiler/OS combinations. +The canonical example is `RcppParallel`, whose bundled TBB is linked in a way that lets a system TBB on the build host shadow it, so the published binary depends on a library the consumer does not have. Because such packages are often transitive dependencies of many others, a single failure cascades: all dependents fail even though nothing is wrong with the dependent itself. To address this, frequently-failing packages can be "patched" before they are installed — whether as a direct build target or a transitive dependency pulled in by `pak`. diff --git a/local/patches/RcppParallel/bundled-tbb-link-order.patch b/local/patches/RcppParallel/bundled-tbb-link-order.patch new file mode 100644 index 0000000..45114a8 --- /dev/null +++ b/local/patches/RcppParallel/bundled-tbb-link-order.patch @@ -0,0 +1,46 @@ +diff --git a/tools/config/configure.R b/tools/config/configure.R +index 6293fe1..2f84337 100644 +--- a/tools/config/configure.R ++++ b/tools/config/configure.R +@@ -186,12 +186,23 @@ define( + ) + + # set PKG_LIBS ++# ++# bincraft patch: the library directories below are passed as plain '-L', not ++# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) ++# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', ++# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build ++# host with a distro TBB installed, '-ltbb' resolves to that library and ++# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the ++# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads ++# the system TBB rather than the copy shipped in RcppParallel/lib, and fails ++# outright on a machine that has no system TBB. gcc places a plain '-L' before ++# its built-in dirs, so the intended library is found first. + pkgLibs <- if (!is.na(tbbLib)) { + + # a TBB supplied via TBB_LIB / TBB_ROOT. an rpath is meaningless on Windows, + # where the loader has no equivalent -- see R/zzz.R for how we resolve there + c( +- "-Wl,-L\"$(TBB_LIB)\"", ++ "-L\"$(TBB_LIB)\"", + if (.Platform$OS.type != "windows") + sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), + "-l$(TBB_NAME)", +@@ -201,14 +212,14 @@ pkgLibs <- if (!is.na(tbbLib)) { + } else if (R.version$os == "emscripten") { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)" + ) + + } else { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" + ) diff --git a/local/patches/RcppParallel/system-tbb.patch b/local/patches/RcppParallel/system-tbb.patch deleted file mode 100644 index 263a593..0000000 --- a/local/patches/RcppParallel/system-tbb.patch +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/src/Makevars.in b/src/Makevars.in -index be8445f..7cb6c9e 100644 ---- a/src/Makevars.in -+++ b/src/Makevars.in -@@ -60,7 +60,13 @@ else - endif - - ifeq ($(UNAME), Linux) -- USE_TBB=Linux -+ # bincraft patch: link the system oneTBB (installed in the build-env images) -+ # instead of building the bundled Intel TBB, which fails on musl (Alpine) and -+ # newer toolchains (g++ 15). Leaving USE_TBB unset skips the bundled build; -+ # -DRCPP_PARALLEL_USE_TBB=1 keeps the TBB backend so dependents (rstan, ...) -+ # link TBB, and -ltbb/-ltbbmalloc pull the system library from default paths. -+ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=1 -DTBB_SUPPRESS_DEPRECATED_MESSAGES=1 -DTBB_INTERFACE_NEW -+ PKG_LIBS += -ltbb -ltbbmalloc - endif - - ifeq ($(UNAME), SunOS) diff --git a/local/patches/registry.json b/local/patches/registry.json index 99b80e3..fe0d387 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -1,42 +1,22 @@ [ { "package": "RcppParallel", - "versions": "*", - "platforms": [ - "alpine", - "ubuntu", - "redhat" - ], + "versions": ">=6.0.0", + "platforms": ["*"], "env": {}, "configure_args": [], "makevars": {}, - "patch": "RcppParallel/system-tbb.patch", - "reason": "RcppParallel bundles an old Intel TBB whose build fails on musl and modern toolchains (g++ 8-15 + modern make); link the system oneTBB (2021+) instead, keeping the TBB backend so dependents (rstan, ...) link TBB. All build-env platforms now provide oneTBB 2021+: native on alpine/ubuntu/el10, built from source into /usr/local on el8/el9 (their stock TBB is classic 2018/2020, too old)." + "patch": "RcppParallel/bundled-tbb-link-order.patch", + "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake, which works on musl and g++ 8-15, so the system-TBB workaround needed for 5.x is gone. What remains broken is the link order: configure.R passes the TBB directory as '-Wl,-L', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,'. On a build host with a distro TBB installed, '-ltbb' therefore resolves to the system library and RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the classic Intel TBB on el8/el9) instead of the bundled 'libtbb.so'. The published binary then loads the system TBB rather than the copy in RcppParallel/lib and fails to dyn.load on a consumer machine without one, the same way fs did with libuv. Passing a plain '-L' puts the bundled build dir ahead of gcc's defaults; verified on alpine 3.24, el8 and ubuntu noble to produce NEEDED libtbb.so + RPATH $ORIGIN/../lib, loading with every system libtbb removed." }, { "package": "fs", "versions": "*", - "platforms": [ - "*" - ], + "platforms": ["*"], "env": {}, "configure_args": [], "makevars": {}, "patch": "fs/force-vendored-libuv.patch", "reason": "fs 2.x configure links system libuv whenever pkg-config finds libuv-devel (installed as a build-time sysreq), producing an fs.so with NEEDED libuv.so.1. That binary fails to dyn.load on consumer machines lacking runtime libuv, because install.packages/renv do not install SystemRequirements (only pak does, and only in the build container). The patch short-circuits configure to copy src/Makevars.vendor and build the bundled static libuv (needs cmake) so the binary is self-contained on every platform. An env/pkg-config override was tried first but the rebuilt binary still linked libuv.so.1, so a source patch is used instead." - }, - { - "package": "rstan", - "versions": "*", - "platforms": [ - "*" - ], - "env": {}, - "configure_args": [], - "makevars": { - "CPPFLAGS": "-DTBB_INTERFACE_NEW -I/usr/local/include" - }, - "patch": null, - "reason": "StanHeaders' init_threadpool_tbb.hpp unconditionally includes the legacy (removed in oneTBB 2021+) for version detection, breaking compilation of Module.cpp against the bundled oneTBB. Pre-defining TBB_INTERFACE_NEW skips that include and selects the modern tbb/global_control.h + tbb/task_arena.h path that the bundled TBB provides (-I/usr/local/include preserves the default CPPFLAGS the override replaces)" } ] From fe2289af56e4724200cd1ebc41e06e67f5c9cb71 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 31 Jul 2026 09:48:41 +0000 Subject: [PATCH 032/118] fix(patches): force RcppParallel to use its bundled oneTBB (#146) ## Why `RcppParallel` binaries built on ubuntu 22.04 still record `NEEDED libtbb.so.12` and cannot `dyn.load` without the distro TBB, although #145 landed the link-order patch and build-env-images dropped `TBB_INC`/`TBB_LIB` the same morning. The link-order fix does not survive a build image that exports those variables, and the published images still do: `.crow/images.yaml` in build-env-images triggers on `cron`/`manual` only, so every image in the registry is still the 2026-07-21 build. With `TBB_LIB` set, `configure.R` never reaches the bundled-oneTBB branch at all, `install.libs.R` fills `RcppParallel/lib` with absolute symlinks into the image's library dir, and the plain `-L` from #145 just points at the system TBB instead of the bundled one. Nothing on the pipeline side can undo this: R reads `~/.Renviron` **after** the process environment, so neither a Crow `environment:` block nor the registry `env` field wins. ``` # the image, not the Containerfile docker run --rm reg.devxy.io/rpkgs/build-env-ubuntu:jammy sh -lc 'grep TBB ~/.Renviron' TBB_INC=/usr/include/oneapi TBB_LIB=/usr/lib/x86_64-linux-gnu ``` Rather than couple correctness to image freshness for a third round-trip, the patch now forces the bundled TBB itself. ## Changes - `local/patches/RcppParallel/bundled-tbb-link-order.patch` -> `local/patches/RcppParallel/force-bundled-tbb.patch`. Keeps the `-Wl,-L` -> `-L` link-order hunk (the bundled branch has the same problem when a distro libtbb is installed) and adds an opt-out, `BINCRAFT_ALLOW_SYSTEM_TBB=TRUE`, for the ambient `TBB_ROOT`/`TBB_LIB`/`TBB_INC`. - All four read sites have to move together; patching only `configure.R` builds the bundled TBB and then **segfaults R on the post-install load test**: - `tools/config/configure.R`: branch selection. - `src/install.libs.R`: `R CMD INSTALL` runs it outside the `tbb` rule in `src/Makevars` that passes the configured values, so at install time it still sees the image environment. - `R/tbb.R` `tbbLibraryPath()`: run time. `.onLoad()` would `dyn.load` the system TBB into the global scope next to the bundled one that `RcppParallel.so` needs - two copies of the same symbols, hence the segfault. - `R/tbb.R` `tbbCxxFlags()` / `tbbLdFlags()`: otherwise dependents such as `rstan` get the system TBB handed back to them. - `local/patches/registry.json`: new patch path and a `reason` describing both failure modes. ## Verification Built through `bincraft:::prepare_patched_repo()` on the **current, unfixed** images (all three still export `TBB_INC`/`TBB_LIB`), with bincraft v5.0.1, i.e. what CI deploys: | image | `NEEDED` | rpath | `RcppParallel/lib` | load, all system `libtbb*` moved away | | --- | --- | --- | --- | --- | | `build-env-ubuntu:jammy` | `libtbb.so` | `$ORIGIN/../lib` | real `libtbb.so.2`, 5.0 MB | OK, 12 threads | | `build-env-redhat:9` | `libtbb.so` | `$ORIGIN/../lib` | real `libtbb.so.2`, 5.1 MB | OK, 12 threads | | `build-env-alpine:3.24` | `libtbb.so` | `$ORIGIN/../lib` | real `libtbb.so.2`, 4.7 MB | OK, 12 threads | For contrast, the same jammy build without this change: ``` NEEDED libtbb.so.12 RUNPATH /usr/lib/x86_64-linux-gnu:$ORIGIN/../lib lib/ libtbb.so.12.5 -> /usr/lib/x86_64-linux-gnu/libtbb.so.12.5 (dangling off the image) load libtbb.so.12: cannot open shared object file: No such file or directory ``` With `TBB_LIB` still exported, `RcppParallel::tbbLibraryPath()` resolves to the package's own `lib`, and `CxxFlags()` emits the package's own `include` plus `-DTBB_INTERFACE_NEW` - not `/usr/include/oneapi`. ## Behaviour change Published `RcppParallel` binaries carry their own oneTBB on every platform, whichever image version CI pulls. The patch content changed, so the patched-binary cache key changes with it and no stale entry is reused. The build-env-images fix is still worth rolling out (those images also carry the broken el8 `uvr` and uvr 0.4.1), but RcppParallel no longer waits on it. ## Follow-up, not in this PR The binaries already in B2 are the broken ones; they need a rebuild, and a Bunny `/purge` does not evict Perma-Cache. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/146 --- .../RcppParallel/bundled-tbb-link-order.patch | 46 ------ .../RcppParallel/force-bundled-tbb.patch | 154 ++++++++++++++++++ local/patches/registry.json | 4 +- 3 files changed, 156 insertions(+), 48 deletions(-) delete mode 100644 local/patches/RcppParallel/bundled-tbb-link-order.patch create mode 100644 local/patches/RcppParallel/force-bundled-tbb.patch diff --git a/local/patches/RcppParallel/bundled-tbb-link-order.patch b/local/patches/RcppParallel/bundled-tbb-link-order.patch deleted file mode 100644 index 45114a8..0000000 --- a/local/patches/RcppParallel/bundled-tbb-link-order.patch +++ /dev/null @@ -1,46 +0,0 @@ -diff --git a/tools/config/configure.R b/tools/config/configure.R -index 6293fe1..2f84337 100644 ---- a/tools/config/configure.R -+++ b/tools/config/configure.R -@@ -186,12 +186,23 @@ define( - ) - - # set PKG_LIBS -+# -+# bincraft patch: the library directories below are passed as plain '-L', not -+# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) -+# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', -+# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build -+# host with a distro TBB installed, '-ltbb' resolves to that library and -+# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the -+# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads -+# the system TBB rather than the copy shipped in RcppParallel/lib, and fails -+# outright on a machine that has no system TBB. gcc places a plain '-L' before -+# its built-in dirs, so the intended library is found first. - pkgLibs <- if (!is.na(tbbLib)) { - - # a TBB supplied via TBB_LIB / TBB_ROOT. an rpath is meaningless on Windows, - # where the loader has no equivalent -- see R/zzz.R for how we resolve there - c( -- "-Wl,-L\"$(TBB_LIB)\"", -+ "-L\"$(TBB_LIB)\"", - if (.Platform$OS.type != "windows") - sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), - "-l$(TBB_NAME)", -@@ -201,14 +212,14 @@ pkgLibs <- if (!is.na(tbbLib)) { - } else if (R.version$os == "emscripten") { - - c( -- "-Wl,-Ltbb/build/lib_release", -+ "-Ltbb/build/lib_release", - "-l$(TBB_NAME)" - ) - - } else { - - c( -- "-Wl,-Ltbb/build/lib_release", -+ "-Ltbb/build/lib_release", - "-l$(TBB_NAME)", - "-l$(TBB_MALLOC_NAME)" - ) diff --git a/local/patches/RcppParallel/force-bundled-tbb.patch b/local/patches/RcppParallel/force-bundled-tbb.patch new file mode 100644 index 0000000..c46a05e --- /dev/null +++ b/local/patches/RcppParallel/force-bundled-tbb.patch @@ -0,0 +1,154 @@ +diff --git a/R/aaa.R b/R/aaa.R +index 568a2aa..bfdab4c 100644 +--- a/R/aaa.R ++++ b/R/aaa.R +@@ -5,4 +5,21 @@ TBB_LIB <- "" + TBB_INC <- "" + + TBB_NAME <- "tbb" +-TBB_MALLOC_NAME <- "tbbmalloc" +\ No newline at end of file ++TBB_MALLOC_NAME <- "tbbmalloc" ++ ++# bincraft patch: our build images (and plenty of user environments) export ++# TBB_ROOT / TBB_LIB / TBB_INC. A binary we publish always carries its own ++# oneTBB in RcppParallel/lib -- see the companion changes in ++# tools/config/configure.R and src/install.libs.R -- so honouring those ++# variables at run time is actively harmful: .onLoad() would dyn.load() a ++# second, unrelated TBB into the process next to the bundled one (two copies ++# of the same symbols in the global scope, which segfaults R on load), and ++# RcppParallelLibs() / CxxFlags() would hand that system TBB to dependents ++# such as rstan, putting NEEDED libtbb.so.12 back into their binaries. Read ++# the variables only when explicitly opted back in. ++bincraftGetenv <- function(name, unset = "") { ++ if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") ++ Sys.getenv(name, unset = unset) ++ else ++ unset ++} +\ No newline at end of file +diff --git a/R/tbb.R b/R/tbb.R +index 6f6a745..e407986 100644 +--- a/R/tbb.R ++++ b/R/tbb.R +@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { + sysname <- Sys.info()[["sysname"]] + + # find root for TBB install +- tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) ++ tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) + if (is.null(name)) + return(tbbRoot) + +@@ -58,7 +58,7 @@ tbbCxxFlags <- function() { + flags <- c("-DRCPP_PARALLEL_USE_TBB=1") + + # if TBB_INC is set, apply those library paths +- tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) ++ tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) + if (!file.exists(tbbInc)) { + tbbInc <- system.file("include", package = "RcppParallel") + } +@@ -117,7 +117,7 @@ tbbLdFlags <- function() { + } + + # shortcut if TBB_LIB defined +- tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) ++ tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) + if (nzchar(tbbLib)) { + if (R.version$os == "emscripten") { + fmt <- "-L%1$s -l%2$s" +diff --git a/src/install.libs.R b/src/install.libs.R +index 3b3cfda..c0e6f3e 100644 +--- a/src/install.libs.R ++++ b/src/install.libs.R +@@ -477,6 +477,18 @@ prependFlags <- function(prependFlags, toFlags) { + tbbLib <- Sys.getenv("TBB_LIB") + tbbInc <- Sys.getenv("TBB_INC") + ++# bincraft patch: the companion change in tools/config/configure.R stops an ++# ambient TBB_LIB / TBB_INC from selecting a system TBB, but this script is ++# also run directly by `R CMD INSTALL` (not only through the `tbb` rule in ++# src/Makevars, which passes the configured values), so at install time it ++# still sees the image's environment and would symlink the system libraries ++# into RcppParallel/lib. Drop them here for the same reason, under the same ++# opt-out. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbLib <- "" ++ tbbInc <- "" ++} ++ + args <- commandArgs(trailingOnly = TRUE) + if (identical(args, "build")) { + if (nzchar(tbbLib) && nzchar(tbbInc)) { +diff --git a/tools/config/configure.R b/tools/config/configure.R +index 6293fe1..eae4aaa 100644 +--- a/tools/config/configure.R ++++ b/tools/config/configure.R +@@ -40,6 +40,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) + tbbLib <- Sys.getenv("TBB_LIB", unset = NA) + tbbInc <- Sys.getenv("TBB_INC", unset = NA) + ++# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Several of ++# our build images export these (a leftover from RcppParallel 5.x, whose ++# bundled Intel TBB would not build on musl or with modern g++), and any of ++# them switches the branches below to a system TBB. The published binary then ++# records NEEDED libtbb.so.12 (or libtbb.so.2 for the classic Intel TBB) and ++# gets a RcppParallel/lib full of absolute symlinks into the image's library ++# dir, so it cannot dyn.load on a consumer machine without that exact TBB. ++# 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, ++# so the bundled copy is always the right choice here; forcing it in the ++# package rather than relying on the image environment keeps the binary ++# correct whichever image version CI happens to pull. Set ++# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbRoot <- NA ++ tbbLib <- NA ++ tbbInc <- NA ++} ++ + tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") + tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") + +@@ -186,12 +204,23 @@ define( + ) + + # set PKG_LIBS ++# ++# bincraft patch: the library directories below are passed as plain '-L', not ++# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) ++# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', ++# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build ++# host with a distro TBB installed, '-ltbb' resolves to that library and ++# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the ++# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads ++# the system TBB rather than the copy shipped in RcppParallel/lib, and fails ++# outright on a machine that has no system TBB. gcc places a plain '-L' before ++# its built-in dirs, so the intended library is found first. + pkgLibs <- if (!is.na(tbbLib)) { + + # a TBB supplied via TBB_LIB / TBB_ROOT. an rpath is meaningless on Windows, + # where the loader has no equivalent -- see R/zzz.R for how we resolve there + c( +- "-Wl,-L\"$(TBB_LIB)\"", ++ "-L\"$(TBB_LIB)\"", + if (.Platform$OS.type != "windows") + sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), + "-l$(TBB_NAME)", +@@ -201,14 +230,14 @@ pkgLibs <- if (!is.na(tbbLib)) { + } else if (R.version$os == "emscripten") { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)" + ) + + } else { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" + ) diff --git a/local/patches/registry.json b/local/patches/registry.json index fe0d387..a7e29dd 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -6,8 +6,8 @@ "env": {}, "configure_args": [], "makevars": {}, - "patch": "RcppParallel/bundled-tbb-link-order.patch", - "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake, which works on musl and g++ 8-15, so the system-TBB workaround needed for 5.x is gone. What remains broken is the link order: configure.R passes the TBB directory as '-Wl,-L', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,'. On a build host with a distro TBB installed, '-ltbb' therefore resolves to the system library and RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the classic Intel TBB on el8/el9) instead of the bundled 'libtbb.so'. The published binary then loads the system TBB rather than the copy in RcppParallel/lib and fails to dyn.load on a consumer machine without one, the same way fs did with libuv. Passing a plain '-L' puts the bundled build dir ahead of gcc's defaults; verified on alpine 3.24, el8 and ubuntu noble to produce NEEDED libtbb.so + RPATH $ORIGIN/../lib, loading with every system libtbb removed." + "patch": "RcppParallel/force-bundled-tbb.patch", + "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, so the system-TBB workaround needed for 5.x is gone, but two things still steer the build back to a system TBB. (1) Ambient TBB_ROOT/TBB_LIB/TBB_INC: several build images still export these (build-env-images dropped them, but the images are rebuilt only by cron/manual runs, so a stale image keeps them), and R reads ~/.Renviron *after* the process environment, so no pipeline-side env override can undo it. configure.R then takes the system-TBB branch, install.libs.R symlinks the image's libraries into RcppParallel/lib as absolute paths, and the binary records NEEDED libtbb.so.12 (libtbb.so.2 for the classic Intel TBB on el8/el9) -- it cannot dyn.load on a consumer machine without that exact TBB. Both files are patched to ignore those variables (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE); install.libs.R needs it separately because R CMD INSTALL runs it outside the src/Makevars rule that passes the configured values. (2) Link order: the bundled branch passes its build dir as '-Wl,-Ltbb/build/lib_release', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,', so '-ltbb' would still resolve to a distro TBB when one is installed; a plain '-L' puts the bundled dir first. Verified on the current build-env-ubuntu:jammy image (which still exports TBB_INC/TBB_LIB): NEEDED libtbb.so, RUNPATH $ORIGIN/../lib, real libtbb.so.2 in RcppParallel/lib, and the package loads with every system libtbb moved away." }, { "package": "fs", From ee15c50f53996229451a44cfe12b2222853728e2 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 31 Jul 2026 12:19:26 +0000 Subject: [PATCH 033/118] refactor: migrate package installation from pak to uvr (#147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `bincraft` dropped pak in favour of uvr (5.0.x, "Dependencies and their system requirements are now installed with `uvr` instead of pak during `build_binary_package()`"), so the pipelines, helper scripts and images in this repo move with it. ## Approach uvr is project-scoped in a way pak is not: `uvr add` refuses to run outside a project and always installs into `.uvr/library/`, and only `uvr sync` honours `--library`. So there is no one-line `pak::pak(...)` equivalent. `local/uvr-install.sh` encapsulates the dance — bootstrap a pinned uvr, mint a throwaway project under `TMPDIR`, `uvr add --no-install`, then `uvr sync --library `. Keeping the project outside the checkout also keeps `uvr init`'s `.Rprofile` from hijacking `.libPaths()` for every other R call in the pipeline. This matches what bincraft itself does (`uvr sync --install-system-deps --library `), and bincraft requires `uvr` on `PATH`, which the bootstrap provides: every pipeline that calls `bincraft::` runs `install-bincraft.R` (and therefore the bootstrap) first. ## Changes | File | Change | | --- | --- | | `local/uvr-install.sh` | **New.** The single replacement for `pak::pak(...)`. Bootstraps uvr `v0.4.4`, resolves the R interpreter from `UVR_R_BIN`/`R_VERSION`/`PATH`, pins the manifest to that R's exact version, and syncs into `UVR_TARGET_LIB`/`R_LIBS_USER`. | | `local/install-bincraft.R` | Installs `forgejo::codefloe.com/rpkgs/bincraft@` instead of a `git::` URL; keeps the `git ls-remote` tag resolution. Exports `UVR_R_BIN`/`UVR_TARGET_LIB` from `R.home()`/`.libPaths()[1]` so the per-R-minor passes target their own R and library. | | `.crow/auto-apply-patches.yaml`, `.crow/weekly-patch-proposals.yaml`, `.crow/weekly-audit-missing.yaml`, `.crow/weekly-rebuild-missing.yaml`, `.crow/build-all-versions-install-deps.yaml` | `pak::pak(...)` → `UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh ...`. The explicit `UVR_R_BIN` matters in `build-all-versions-install-deps.yaml`, which has no `R_VERSION` in its step environment. | | `.crow/build-all-versions.yaml`, `.crow/process-updates.yaml`, `.crow/weekly-rebuild-missing.yaml` | `R_PKG_CACHE_DIR` → `UVR_CACHE_DIR` + `UVR_PACKAGES_DIR` on the same `/mnt/cache` volume, preserving the amd64-off/arm64-on split. Drops the `rm -rf .../pkgcache/_metadata/...` cleanup. | | `local/r-minor-helpers.R`, `local/build-all.R`, `local/tests/test-trim-pkgcache.R` | Removes `trim_pkgcache_metadata()`, its every-25-packages call and its tests. uvr's cache does not mint a fresh ~70 MB snapshot per `PACKAGES` change. | | `.crow/build-all-versions-install-deps.yaml` | Drops `pak::sysreqs_db_update()`; uvr resolves sysreqs from its vendored `r-system-requirements` rules via `--install-system-deps`. | | `docker/Containerfile-shiny-app` | Bootstraps uvr and drives both dependency installs through one uvr project with `UVR_LIBRARY` pointed at the image's R library. | | `docker/build-one.Dockerfile` | Ships `uvr-install.sh` at `/work/local/` so `install-bincraft.R` finds it. | | `docker/reprex/alpine.sh` | Replaces `pak::local_install_deps()` with DESCRIPTION parsing + `uvr add`. | | `local/test-package-loading.R` | Installs via the helper instead of `pak::pkg_install()`. | | `README.md` | Documents uvr for sysreq inference, archived-version installs and cache clearing. | | `renovate.json` | Tracks the `UVR_PIN` in `uvr-install.sh` via `github-releases`. | ## Behaviour notes - **`weekly-audit-missing` still takes bincraft from the default branch**, not the latest release tag, matching what the `git::` pak call did. Called out in a comment rather than silently changed. - **The uvr pin is repo-wide.** bincraft resolves `uvr` from `PATH` and pins no version of its own, so `UVR_PIN` in `uvr-install.sh` governs the whole pipeline. - **Persistent caches now also benefit bincraft**, which reads `UVR_CACHE_DIR`/`UVR_PACKAGES_DIR` from the inherited pipeline environment. - **`uvr sync` will not prune the shared library.** Pruning is disabled whenever `--library` is passed (`do_prune = prune && library_override.is_none()`), so `/mnt/cache/R-pkgs` keeps bincraft and its dependencies. The wipe-on-ABI-mismatch path is *not* similarly guarded, which is why the helper pins the manifest to the active R's exact version. - **`plans/` and `specs/` are untouched** — they are dated records of decisions made in June 2026 and describe bincraft's then-pak-based internals; rewriting them would misstate history. ## Verification `shellcheck`, all pre-commit hooks (on this commit's file range) and the `local/tests/` suite (100 assertions) pass. Not yet exercised in CI: the build-env images do not ship `uvr`, so the per-step `curl install.sh` bootstrap is untested against a real image. Worth a manual `build-all-versions-install-deps` run before merging. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/147 --- .crow/auto-apply-patches.yaml | 2 +- .crow/build-all-versions-install-deps.yaml | 33 ++++---- .crow/build-all-versions.yaml | 64 ++++++++------- .crow/process-updates.yaml | 46 +++++------ .crow/weekly-audit-missing.yaml | 3 +- .crow/weekly-patch-proposals.yaml | 2 +- .crow/weekly-rebuild-missing.yaml | 7 +- README.md | 19 ++--- build-all-versions-install-deps.yaml | 10 +-- docker/Containerfile-shiny-app | 22 +++++- docker/build-one.Dockerfile | 3 + docker/reprex/alpine.sh | 24 ++++-- local/build-all.R | 31 ++++---- local/install-bincraft.R | 47 +++++++++-- local/packages-to-build.R | 15 ++-- local/r-minor-helpers.R | 39 ---------- local/test-package-loading.R | 32 ++++---- local/tests/test-trim-pkgcache.R | 68 ---------------- local/uvr-install.sh | 90 ++++++++++++++++++++++ renovate.json | 14 +++- 20 files changed, 326 insertions(+), 245 deletions(-) delete mode 100644 local/tests/test-trim-pkgcache.R create mode 100755 local/uvr-install.sh diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index b9c8697..0076353 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -45,7 +45,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT backend_options: kubernetes: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index e0cca86..448cff3 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -10,22 +10,22 @@ variables: - arm64 default: amd64 OS: - description: "Base OS image name." + description: 'Base OS image name.' options: - alpine - redhat - ubuntu default: alpine OS_VERSION: - description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' options: - - "3.24" - - "8" - - "9" - - "10" - - "jammy" - - "noble" - default: "3.24" + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -66,23 +66,24 @@ steps: from_secret: B2_S3_SECRET_KEY PGPASS: from_secret: PGPASS - R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs + # Keep uvr's downloads and extracted-package entries on the persistent + # volume instead of the container-local ~/.uvr default. + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages CCACHE_DIR: /mnt/cache/ccache volumes: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: # one-time full wipe to fix corrupted .so files from previous failed builds # - rm -rf /mnt/cache/R-pkgs - # Clear churny pkgcache metadata left by a prior crashed run (the "patched" - # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). - # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. - - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 3932811..9ab888a 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -11,25 +11,25 @@ variables: - arm64 default: amd64 OS: - description: "Base OS image name." + description: 'Base OS image name.' options: - alpine - redhat - ubuntu default: alpine OS_VERSION: - description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' options: - - "3.22" - - "3.23" - - "3.24" - - "8" - - "9" - - "10" - - "jammy" - - "noble" - - "resolute" - default: "3.24" + - '3.22' + - '3.23' + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + - 'resolute' + default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -47,38 +47,49 @@ labels: platform: linux/${ARCH} group: rpkgs-${ARCH} +# Empty UVR_CACHE_DIR/UVR_PACKAGES_DIR fall back to uvr's container-local +# ~/.uvr defaults; amd64 deliberately does not persist them (as with the +# pkgcache dir it replaces), arm64 does. matrix: include: - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 4 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 4 @@ -107,8 +118,10 @@ steps: from_secret: GITHUB_PAT # normal env vars GIT_USER: pat-s - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR} + # set the location of uvr's caches, which persist the R package + # dependencies needed to install the packages themselves + UVR_CACHE_DIR: ${UVR_CACHE_DIR} + UVR_PACKAGES_DIR: ${UVR_PACKAGES_DIR} R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache NCPUS: 2 @@ -116,12 +129,7 @@ steps: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - # Clear churny pkgcache metadata left by a prior crashed run (the "patched" - # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). - # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. Within-run - # growth is bounded separately by trim_pkgcache_metadata() in build-all.R. - - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # The primary pass must not rely on build-all-versions-install-deps having # run on *this* agent: depends_on only orders the steps, but the cache # volume is per-agent, so a job landing on an agent where install-deps did diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 59b1c26..1e4833c 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -49,109 +49,109 @@ matrix: R_VERSION: 4.5.3 IMG: alpine:3.22 OS_ID: alpine322 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.23 OS_ID: alpine322 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.23 OS_ID: alpine323 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.23 OS_ID: alpine323 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine324 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine324 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-8 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-9 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-9 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-10 ARCH: amd64 R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-10 ARCH: arm64 R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2204 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2204 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2404 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2404 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2604 ARCH: amd64 R_VERSION: 4.5.3 IMG: ubuntu:resolute OS_ID: resolute - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2604 ARCH: arm64 R_VERSION: 4.5.3 IMG: ubuntu:resolute OS_ID: resolute - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' steps: - name: 'Processing Updates' @@ -182,8 +182,10 @@ steps: NTFY_AUTH: TRUE NTFY_PASSWORD: from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + # set the location of uvr's caches, which persist the R package + # dependencies needed to install the packages themselves + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache @@ -194,10 +196,10 @@ steps: INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index d66a37b..4efe409 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -142,7 +142,8 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' backend_options: docker: diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 0284e93..027d41c 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -35,7 +35,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 53f178f..03a13f9 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -141,7 +141,8 @@ steps: FORGEJO_TOKEN: from_secret: FORGEJO_TOKEN GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache @@ -150,12 +151,12 @@ steps: NCPUS: 2 commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: diff --git a/README.md b/README.md index 538df34..989069e 100644 --- a/README.md +++ b/README.md @@ -112,9 +112,9 @@ Processing updates with an existing database file takes around 5 minutes. ### Inferring System Dependencies -R package dependencies and their system dependencies are installed through {pak}. -{pak} allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. -Additionally, it automatically infers package dependencies using JSON rules from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). +R package dependencies and their system dependencies are installed through [uvr](https://github.com/nbafrank/uvr). +`uvr` allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. +Additionally, it automatically infers package dependencies using JSON rules vendored from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). Not all R packages specify required system dependencies in their DESCRIPTION file, and not all listed dependencies have existing rules in `rstudio/r-system-requirements`. For Alpine, no rules existed until recently, establishing a foundation for semi-automated package installation on Alpine Linux. @@ -152,12 +152,13 @@ A Shiny dashboard providing a search functionality of the database and grouped s Is supported by writing `Meta/archive.rds` during each package index update, listing all available archived packages. -### `pak::pak(package@version)` +### `uvr add package@version` -`pak` searches for `Archive/` and can install all versions it finds. +Clients that resolve archived versions search for `Archive/` and can install all versions they find. +This holds for `uvr` as well as for older `pak`-based clients. Ensure to use a clean cache if other repositories have been used previously. -If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. +If in doubt or when testing, call `uvr cache clean`. ## Lessons Learned @@ -180,7 +181,7 @@ If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. ## URL Composition and Platform Identifiers -Platform identifiers have been aligned with those used in to ensure proper recognition by the automatic syslib dependency installer of `pak`, specifically via the environment variable `PKG_SYSREQS_PLATFORM`: +Platform identifiers have been aligned with those used in , which `uvr` vendors and uses for its automatic syslib dependency installer (`uvr sync --install-system-deps`): - redhat-9 - redhat-8 @@ -237,8 +238,8 @@ internal error 1 in memDecompress Solution: ```sh -rm -rf /mnt/cache/R-pkgs/pak /mnt/cache/pkgcache/ /root/.cache/R/ -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +uvr cache clean +rm -rf /mnt/cache/uvr /root/.cache/R/ ``` diff --git a/build-all-versions-install-deps.yaml b/build-all-versions-install-deps.yaml index 926d550..dd02e8a 100644 --- a/build-all-versions-install-deps.yaml +++ b/build-all-versions-install-deps.yaml @@ -30,8 +30,9 @@ # from_secret: HETZNER_S3_SECRET_KEY_K3S # # normal env vars # GIT_USER: pat-s -# # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves -# # R_PKG_CACHE_DIR: /mnt/cache/pkgcache +# # set the location of uvr's caches, which persist the R package dependencies needed to install the packages themselves +# # UVR_CACHE_DIR: /mnt/cache/uvr/cache +# # UVR_PACKAGES_DIR: /mnt/cache/uvr/packages # R_LIBS_USER: /mnt/cache/R-pkgs # CCACHE_DIR: /mnt/cache/ccache # volumes: @@ -39,9 +40,8 @@ # commands: # - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # - mkdir -p /mnt/cache/R-pkgs -# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/pkgcache -# - /opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' -# - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git", dependencies = TRUE)' +# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/uvr +# - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft # - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine322', 'amd64'); saveRDS(pkgs, '/mnt/cache/pkgs_amd64.rds')" # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine323', 'arm64'); saveRDS(pkgs, '/mnt/cache/pkgs_arm64.rds')" diff --git a/docker/Containerfile-shiny-app b/docker/Containerfile-shiny-app index 7c7994f..4b0bd6c 100644 --- a/docker/Containerfile-shiny-app +++ b/docker/Containerfile-shiny-app @@ -1,7 +1,15 @@ FROM devxygmbh/r-alpine:4.4-3.20 AS build # ARG GITHUB_PAT -RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +RUN apk add --no-cache curl ca-certificates \ + && curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh \ + | UVR_INSTALL_DIR=/usr/local/bin sh + +# One uvr project drives both dependency installs below. UVR_LIBRARY points the +# syncs at the image's R library instead of the project-local .uvr/library/, so +# `Rscript app.R` finds the packages without a uvr-aware .Rprofile. +ENV UVR_LIBRARY=/usr/lib/R/library +RUN mkdir -p /uvr && cd /uvr && uvr init --here COPY --link ./DESCRIPTION . COPY --link ./R ./R @@ -11,13 +19,19 @@ COPY --link ./shiny/app.R /app/app.R RUN ls -la -# install R package deps -RUN R -q -e 'pak::pak()' +# install R package deps. uvr has no equivalent of bare `pak::pak()`, which +# reads the DESCRIPTION in the working directory, so extract the dependency +# names and hand them to `uvr add`. +RUN R -q --no-echo -e "d <- read.dcf('DESCRIPTION'); f <- intersect(colnames(d), c('Depends', 'Imports', 'LinkingTo')); p <- trimws(sub('[(].*', '', unlist(strsplit(paste(d[, f], collapse = ','), ',')))); writeLines(setdiff(p[nzchar(p)], c('R', rownames(installed.packages()))), '/tmp/deps.txt')" \ + && cd /uvr && xargs -r uvr add --no-install < /tmp/deps.txt \ + && uvr sync --install-system-deps RUN R CMD INSTALL --no-docs --without-keep.source . # install shiny app deps -RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); pak::pak(pkgs)" +RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); writeLines(pkgs, '/tmp/app-deps.txt')" \ + && cd /uvr && xargs -r uvr add --no-install < /tmp/app-deps.txt \ + && uvr sync --install-system-deps ENV PGPASS="" diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 5bc107a..8e61ebd 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -17,7 +17,10 @@ ARG CACHEBUST WORKDIR /work COPY build-one.R /work/build-one.R # Resolve and install the latest bincraft release dynamically (no hardcoded pin). +# uvr-install.sh lands under /work/local/ because install-bincraft.R looks for it +# there when the working directory is not a repo checkout. COPY install-bincraft.R /work/install-bincraft.R +COPY uvr-install.sh /work/local/uvr-install.sh # Ship the patch registry so build-one.R's `patches = "local/patches"` resolves # (build context is `local/`, CWD is /work). COPY patches /work/local/patches diff --git a/docker/reprex/alpine.sh b/docker/reprex/alpine.sh index 94806e5..851a9cc 100644 --- a/docker/reprex/alpine.sh +++ b/docker/reprex/alpine.sh @@ -1,10 +1,14 @@ docker run --rm -it --platform linux/arm64 alpine sh -apk add --no-cache R R-dev g++ +apk add --no-cache R R-dev g++ curl ca-certificates -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | UVR_INSTALL_DIR=/usr/local/bin sh -R -q -e 'pak::pak(c("gert", "purrr"))' +# `uvr add` always writes to .uvr/library/; only `uvr sync` honours UVR_LIBRARY, +# so add without installing and let the sync place the packages. +export UVR_LIBRARY=/usr/lib/R/library +mkdir -p /uvr && cd /uvr && uvr init --here +uvr add --no-install gert purrr && uvr sync --install-system-deps R @@ -22,14 +26,20 @@ unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE) tag <- all_tags$name package_name <- rep(package_name, length(tag)) +# uvr has no `pak::local_install_deps()`; read the DESCRIPTION of the checkout +# and `uvr add` the dependency names instead. purrr::walk2(package_name[1], tag, \(x, y) { print(y) system("git config --global advice.detachedHead false") + src <- sprintf("/tmp/%s_%s", x[1], y) system2("git", args = c( "clone", "-q", sprintf("--branch=%s", tail(y, 1)), - sprintf("https://github.com/cran/%s", x[1]), sprintf("/tmp/%s_%s", x[1], y) + sprintf("https://github.com/cran/%s", x[1]), src )) - pak::local_install_deps(sprintf("/tmp/%s_%s", x[1], y)) + d <- read.dcf(file.path(src, "DESCRIPTION")) + f <- intersect(colnames(d), c("Depends", "Imports", "LinkingTo")) + deps <- trimws(sub("[(].*", "", unlist(strsplit(paste(d[, f], collapse = ","), ",")))) + deps <- setdiff(deps[nzchar(deps)], "R") + system2("uvr", c("add", "--no-install", deps)) + system2("uvr", c("sync", "--install-system-deps")) }) - - diff --git a/local/build-all.R b/local/build-all.R index 1df1f83..37fc593 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -27,7 +27,9 @@ package_cache_files <- c( "/mnt/cache/packages/s3_cache.rds" ) if (!all(file.exists(package_cache_files))) { - message("Package snapshot missing from cache; recomputing via packages-to-build.R") + message( + "Package snapshot missing from cache; recomputing via packages-to-build.R" + ) dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { tmp <- paste0(path, ".tmp.", Sys.getpid()) @@ -36,7 +38,10 @@ if (!all(file.exists(package_cache_files))) { } source(file.path("local", "packages-to-build.R")) save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds") - save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds") + save_rds_atomic( + pkgs[r_minor_sensitive == TRUE], + "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" + ) message("Package snapshot recomputed.") } @@ -112,20 +117,22 @@ built <- DBI::dbGetQuery( ) DBI::dbDisconnect(con) before <- nrow(chunk) -chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] -sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) +chunk <- chunk[ + !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), +] +sprintf( + "Skipped %d already-attempted package versions; %d remaining for this job", + before - nrow(chunk), + nrow(chunk) +) # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, -# saving significant memory for pak subprocess forks +# saving significant memory for the dependency-installer subprocesses s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") sprintf("S3 cache: %s files", length(s3_cache)) n <- nrow(chunk) -# Every `trim_every` packages, bound the pkgcache _metadata dir so a full-platform -# run does not accumulate thousands of ~70 MB snapshots and fill the host disk. -# No-op on amd64 (R_PKG_CACHE_DIR is empty / cache not persisted). -trim_every <- 25L mapply( function(pkg, ver, sens, i) { cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) @@ -151,12 +158,6 @@ mapply( upload = TRUE, store_build_metadata = TRUE ) - if (i %% trim_every == 0L) { - removed <- trim_pkgcache_metadata() - if (removed > 0L) { - cat(sprintf(" [pkgcache trim] removed %d stale _metadata entries\n", removed)) - } - } }, chunk$Package, chunk$Version, diff --git a/local/install-bincraft.R b/local/install-bincraft.R index 4665b3b..e39cb4f 100644 --- a/local/install-bincraft.R +++ b/local/install-bincraft.R @@ -11,10 +11,11 @@ # # How it works: list the remote tags with `git ls-remote` (no token needed for # the public repo), keep the `vX.Y.Z` release tags, pick the highest version, -# and install it with pak. pak is idempotent on the git ref, so re-running keeps -# the package when it is already current and only updates when a newer tag ships. -# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is -# identical across git versions and `system2()` argument handling. +# and install it with uvr via `local/uvr-install.sh`. uvr is idempotent on the +# git ref, so re-running keeps the package when it is already current and only +# updates when a newer tag ships. Filtering/sorting is done in R (not via git's +# `--sort`/refspec) so behaviour is identical across git versions and +# `system2()` argument handling. repo_url <- Sys.getenv( "BINCRAFT_GIT_URL", @@ -44,7 +45,43 @@ latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][ ] message(sprintf("Installing latest bincraft release: %s", latest)) -pak::pak(sprintf("git::%s@%s", repo_url, latest)) + +# uvr addresses Forgejo repos as `forgejo::host/owner/repo@ref` rather than as a +# git URL, so drop the scheme and the trailing `.git` from `repo_url`. +spec <- sprintf( + "forgejo::%s@%s", + sub("\\.git$", "", sub("^[a-z]+://", "", repo_url)), + latest +) + +# `local/uvr-install.sh` when run from the repo root, `/work/local/` in the +# build-one image, which copies the two scripts into a flatter layout. +helper <- Sys.getenv("UVR_INSTALL_SH", unset = "") +if (!nzchar(helper)) { + candidates <- c("local/uvr-install.sh", "/work/local/uvr-install.sh") + found <- candidates[file.exists(candidates)] + if (length(found) == 0L) { + stop("Could not locate uvr-install.sh; set UVR_INSTALL_SH", call. = FALSE) + } + helper <- found[1L] +} + +# Point uvr at the R running this script and at the library it would install +# into, so the per-R-minor passes in the build pipelines (which call a different +# Rscript with R_LIBS_USER pointed elsewhere) target their own R and library. +Sys.setenv( + UVR_R_BIN = file.path(R.home("bin"), "R"), + UVR_TARGET_LIB = .libPaths()[1L] +) + +status <- system2(helper, shQuote(spec)) +if (!identical(status, 0L)) { + stop( + sprintf("uvr failed to install %s (exit %s)", spec, status), + call. = FALSE + ) +} + message(sprintf( "bincraft %s installed (%s)", as.character(utils::packageVersion("bincraft")), diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 7a3652d..887952d 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -15,15 +15,15 @@ suppressPackageStartupMessages(library(data.table)) # Sys.setenv("OS_VERSION" = "3.22") # Sys.setenv("ARCH" = "arm64") -arch = Sys.getenv("ARCH") +arch <- Sys.getenv("ARCH") # target: alpine-322, ubuntu-2404, redhat-9, etc. -platform = paste( +platform <- paste( Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-" ) # Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10") -codename = bincraft::set_codename(NULL) +codename <- bincraft::set_codename(NULL) con <- DBI::dbConnect( RPostgres::Postgres(), @@ -35,8 +35,8 @@ con <- DBI::dbConnect( sslmode = "require" ) -cran_archive = tools::CRAN_archive_db() -cran_release = tools::CRAN_package_db() +cran_archive <- tools::CRAN_archive_db() +cran_release <- tools::CRAN_package_db() # Subset cran_archive to only those packages cran_archive_in_release <- cran_archive[ names(cran_archive) %in% cran_release$Package @@ -84,13 +84,14 @@ s3fs::s3_file_system( region_name = "eu-central-003", refresh = TRUE ) -s3_pkgs = s3fs::s3_dir_ls( +s3_pkgs <- s3fs::s3_dir_ls( sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename), recurse = TRUE ) # Save the raw S3 file listing for the build step to use as s3_package_cache -# This avoids loading s3fs/reticulate in the build container, saving memory for pak forks +# This avoids loading s3fs/reticulate in the build container, saving memory for +# the dependency-installer subprocesses saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds") file_names <- basename(s3_pkgs) diff --git a/local/r-minor-helpers.R b/local/r-minor-helpers.R index 6f01c29..65aafdf 100644 --- a/local/r-minor-helpers.R +++ b/local/r-minor-helpers.R @@ -31,42 +31,3 @@ parse_build_args <- function(args) { ncpus = as.integer(pos[3L]) ) } - -# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the -# "patched" repo mints a new content hash on every PACKAGES change, so each build -# writes a fresh ~70 MB _metadata/pkgs-.rds (+ patched-/) that is -# never reused. Keep the `keep` newest entries by mtime; only remove entries -# older than `min_age_secs`, so a concurrent split-job's in-flight files are -# never deleted (each build uses a unique hash, so aged entries are -# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are -# not matched and thus preserved. Returns the number of entries removed. -trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"), - keep = 20L, - min_age_secs = 600) { - meta <- file.path(cache_dir, "R", "pkgcache", "_metadata") - if (!nzchar(cache_dir) || !dir.exists(meta)) { - return(0L) - } - entries <- c( - Sys.glob(file.path(meta, "patched-*")), - Sys.glob(file.path(meta, "pkgs-*.rds")) - ) - if (length(entries) == 0L) { - return(0L) - } - info <- file.info(entries) - order_new_first <- order(info$mtime, decreasing = TRUE) - ranked <- entries[order_new_first] - ranked_mtime <- info$mtime[order_new_first] - if (length(ranked) <= keep) { - return(0L) - } - candidates <- ranked[(keep + 1L):length(ranked)] - candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)]) - removable <- candidates[candidate_age >= min_age_secs] - if (length(removable) == 0L) { - return(0L) - } - unlink(removable, recursive = TRUE, force = TRUE) - length(removable) -} diff --git a/local/test-package-loading.R b/local/test-package-loading.R index 26a2e82..a3b7a28 100644 --- a/local/test-package-loading.R +++ b/local/test-package-loading.R @@ -1,22 +1,28 @@ -install.packages( - "pak", - repos = sprintf( - "https://r-lib.github.io/p/pak/stable/%s/%s/%s", - .Platform$pkgType, - R.Version()$os, - R.Version()$arch +# Installs every CRAN package one by one and checks that it loads. Dependencies +# go through uvr via local/uvr-install.sh, which bootstraps the uvr binary on +# first use and installs into .libPaths()[1]. +uvr_install <- function(pkg) { + Sys.setenv( + UVR_R_BIN = file.path(R.home("bin"), "R"), + UVR_TARGET_LIB = .libPaths()[1L] ) -) + status <- system2("local/uvr-install.sh", shQuote(pkg)) + if (!identical(status, 0L)) { + stop( + sprintf("uvr failed to install %s (exit %s)", pkg, status), + call. = FALSE + ) + } +} -Sys.setenv(PKG_SYSREQS = TRUE) all_pkgs <- rownames(available.packages()) -to_skip = c("ABRSQOL", "ACA", "ACE.CoCo") -all_pkgs = setdiff(all_pkgs, to_skip) +to_skip <- c("ABRSQOL", "ACA", "ACE.CoCo") +all_pkgs <- setdiff(all_pkgs, to_skip) for (i in all_pkgs) { message(sprintf("\nInstalling %s", i)) - pak::pkg_install(i) + uvr_install(i) library(i, character.only = TRUE) } @@ -323,7 +329,7 @@ if (length(to_process) == 0) { } else { for (i in to_process) { message(sprintf("\nInstalling %s", i)) - pak::pkg_install(i) + uvr_install(i) library(i, character.only = TRUE) } # Update to_skip to include all up to the last processed diff --git a/local/tests/test-trim-pkgcache.R b/local/tests/test-trim-pkgcache.R deleted file mode 100644 index 1a5b418..0000000 --- a/local/tests/test-trim-pkgcache.R +++ /dev/null @@ -1,68 +0,0 @@ -source(file.path("..", "r-minor-helpers.R")) - -# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is -# set to `age_secs` in the past so we can exercise the age gate deterministically. -make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) { - root <- tempfile("pkgcache-") - meta <- file.path(root, "R", "pkgcache", "_metadata") - dir.create(meta, recursive = TRUE) - old <- Sys.time() - age_secs - mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p } - mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p } - for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i))) - for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i))) - if (keep_repos) { - mk_dir(file.path(meta, "CRAN-075c426938")) - mk_dir(file.path(meta, "BioCsoft-1ac964ed6c")) - mk_file(file.path(meta, "bioc-sysreqs.dcf.gz")) - mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive - } - root -} - -n_churn <- function(root) { - meta <- file.path(root, "R", "pkgcache", "_metadata") - length(Sys.glob(file.path(meta, "patched-*"))) + - length(Sys.glob(file.path(meta, "pkgs-*.rds"))) -} - -test_that("empty cache_dir is a no-op", { - expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L) -}) - -test_that("missing _metadata dir is a no-op", { - expect_identical( - trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0), - 0L - ) -}) - -test_that("fewer than keep entries removes nothing", { - root <- make_meta(patched = 2L, pkgs = 2L) - expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L) - expect_identical(n_churn(root), 4L) -}) - -test_that("trims down to keep newest, leaving churn == keep", { - root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old - removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0) - expect_identical(removed, 40L) - expect_identical(n_churn(root), 20L) -}) - -test_that("entries younger than min_age_secs are protected", { - root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60) - # keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed - expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L) - expect_identical(n_churn(root), 30L) -}) - -test_that("stable repo dirs and pkg downloads are never touched", { - root <- make_meta(patched = 30L, pkgs = 30L) - trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0) - meta <- file.path(root, "R", "pkgcache", "_metadata") - expect_true(dir.exists(file.path(meta, "CRAN-075c426938"))) - expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c"))) - expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz"))) - expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg"))) -}) diff --git a/local/uvr-install.sh b/local/uvr-install.sh new file mode 100755 index 0000000..e1c4d0f --- /dev/null +++ b/local/uvr-install.sh @@ -0,0 +1,90 @@ +#!/bin/sh +# Install R packages into the CI library with uvr (https://github.com/nbafrank/uvr). +# +# Usage: +# local/uvr-install.sh httr2 jsonlite +# local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft@v4.4.3 +# +# Replaces `pak::pak(...)`. uvr is project-scoped: `uvr add` refuses to run +# outside a project and always writes to `.uvr/library/`, and only +# `uvr sync --library` can target an existing library. The project is therefore +# minted in a scratch directory under TMPDIR and thrown away afterwards; that +# also keeps `uvr init`'s `.Rprofile` out of the repo checkout, where it would +# hijack `.libPaths()` for every other R call in the pipeline. +# +# Pruning is a no-op here: uvr disables it whenever `--library` is passed, +# precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds +# bincraft and its dependencies alongside whatever this script installs). +# +# System dependencies come from uvr's vendored r-system-requirements rules, so +# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed. +# +# Environment: +# UVR_R_BIN R interpreter to install for; set by install-bincraft.R so +# the per-R-minor passes target their own R, not the primary +# R_VERSION fallback interpreter selector (/opt/R//bin/R) +# UVR_TARGET_LIB target library; defaults to R_LIBS_USER, then to the +# active R's .libPaths()[1] (which is where pak wrote) +# UVR_INSTALL_DIR where the uvr binary lands (default /usr/local/bin) + +set -eu + +# renovate: datasource=github-releases depName=nbafrank/uvr +UVR_PIN="v0.4.4" + +if [ "$#" -eq 0 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi + +# The build images keep R under /opt/R/ and off PATH. uvr resolves the +# interpreter via PATH and never downloads one unless `uvr r install` is run, so +# put the requested R first. +r_bin="${UVR_R_BIN:-}" +if [ -z "$r_bin" ] && [ -n "${R_VERSION:-}" ] && [ -x "/opt/R/${R_VERSION}/bin/R" ]; then + r_bin="/opt/R/${R_VERSION}/bin/R" +fi +if [ -n "$r_bin" ]; then + PATH="$(dirname "$r_bin"):$PATH" + export PATH +else + r_bin="$(command -v R)" +fi + +target_lib="${UVR_TARGET_LIB:-${R_LIBS_USER:-}}" +if [ -z "$target_lib" ]; then + target_lib="$("$r_bin" --no-echo --no-save -e 'cat(.libPaths()[1])')" +fi +if [ -z "$target_lib" ]; then + echo "error: could not determine a target library; set UVR_TARGET_LIB" >&2 + exit 2 +fi +mkdir -p "$target_lib" + +# Pin the manifest to the active R so the lockfile's R stays in step with the +# library's R sentinel. Without that, uvr can decide the library is ABI-stale +# and wipe it -- and this target is shared with bincraft. uvr only discovers R +# via PATH/R_HOME (it does not scan /opt/R), so the R put on PATH above is the +# only candidate this constraint can resolve to. +# shellcheck disable=SC2016 # $major/$minor are R expressions, not shell vars +r_full="$("$r_bin" --no-echo --no-save -e 'cat(paste(R.version$major, R.version$minor, sep = "."))')" + +install_dir="${UVR_INSTALL_DIR:-/usr/local/bin}" +uvr_bin="${install_dir}/uvr" +if [ ! -x "$uvr_bin" ]; then + echo "Bootstrapping uvr ${UVR_PIN} into ${install_dir}" + UVR_INSTALL_DIR="$install_dir" UVR_VERSION="$UVR_PIN" \ + sh -c 'curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | sh' +fi + +project_dir="${TMPDIR:-/tmp}/uvr-ci-$$" +rm -rf "$project_dir" +mkdir -p "$project_dir" +trap 'rm -rf "$project_dir"' EXIT +cd "$project_dir" + +"$uvr_bin" init --here --r-version "$r_full" +# --no-install: resolve and lock only. The install happens in the sync below, +# which is the only command that honours --library. +"$uvr_bin" add --no-install "$@" +"$uvr_bin" sync --library "$target_lib" --install-system-deps diff --git a/renovate.json b/renovate.json index 8c8e208..7b2c3b3 100644 --- a/renovate.json +++ b/renovate.json @@ -1,7 +1,12 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": ["local>devxy/renovate-config"], - "ignorePaths": ["docker/**", ".crow/process-updates.yaml", ".crow/build-all-versions.yaml", ".crow/weekly-rebuild-missing.yaml"], + "ignorePaths": [ + "docker/**", + ".crow/process-updates.yaml", + ".crow/build-all-versions.yaml", + ".crow/weekly-rebuild-missing.yaml" + ], "customManagers": [ { "customType": "regex", @@ -14,6 +19,13 @@ "packageNameTemplate": "https://codefloe.com/rpkgs/bincraft.git", "datasourceTemplate": "git-tags", "extractVersionTemplate": "^v?(?.+)$" + }, + { + "customType": "regex", + "fileMatch": ["^local/uvr-install\\.sh$"], + "matchStrings": [ + "# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+UVR_PIN=\"(?[^\"]+)\"" + ] } ] } From e02be7518fa4789688a44b5d56280fff5d3fd765 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 4 Aug 2026 02:32:01 +0000 Subject: [PATCH 034/118] chore(deps): update dependency nbafrank/uvr to v0.4.5 --- local/uvr-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local/uvr-install.sh b/local/uvr-install.sh index e1c4d0f..14b7f78 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -30,7 +30,7 @@ set -eu # renovate: datasource=github-releases depName=nbafrank/uvr -UVR_PIN="v0.4.4" +UVR_PIN="v0.4.5" if [ "$#" -eq 0 ]; then echo "usage: $0 ..." >&2 From 02dd5a695dc1a1816a8f815a7f5471deed99d2b3 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Wed, 5 Aug 2026 00:32:29 +0000 Subject: [PATCH 035/118] chore(deps): update terraform bunnynet to ~> 0.17 --- .terraform.lock.hcl | 72 ++++++++++++++++++++++----------------------- provider.tf | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index fdf41e1..aca8ce1 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.16.0" - constraints = "~> 0.16" + version = "0.17.0" + constraints = "~> 0.17" hashes = [ - "h1:+7y/HM0f4/cm7dpL5RulSZ3k5EDokVaIutfuanc1hhA=", - "h1:/4n/gUEzVa7LArNrNbZa4RyGC3+MRCCYTuqRqF36Z7U=", - "h1:4cSbrSalUxXnG0RVwrQG51NNleZNFU9GJxWCLK1M9WM=", - "h1:AlidPvq+LqsveQVhc9JYa7OKf4mcR+DjCzMomoBcmkw=", - "h1:Beny1tSl5O/jM7sSP7qYv+NEVTTyS/BGtSxvMFsII/o=", - "h1:E50kVGCtxWYoJgOLtdat1JFc9KCIeMByk65XSIpuaB0=", - "h1:ErNz5hzwf70NouW7a1DAknWI/0/BFUEX53hQ4BnUxWg=", - "h1:GTB8ugjpBEsRZpdVmKJw70IF2B1/fd1Gwven6XKghpk=", - "h1:JEEDgEc2gwjwfvnDwWO2ZHaf9T5WHtb7A6ABx6rQqhE=", - "h1:OVd3EgjJDuyMBFQ4XZYrs09NNEtFw4jGAbwyeos2H3M=", - "h1:U/9QeqZ+Q2MQ7eNfD1eWomLFlbYGwpliHcD/UTFe/D0=", - "h1:VPUe+m5Fx31TAR7udJPGGzk3EpCNizf0TmnKplOJyRM=", - "h1:YkJwUFpUemqi6bwRIEJW4xSpI7CAEXirqhyjl6Rmm68=", - "h1:a/15E8lijUcqvhUGxtqKmJT6jYGmQW9z7yBoOjusDm8=", - "h1:hii960NFcNpr0eZ/dcbnYOf2oZuo2IZl+z1IMTwh5YA=", - "h1:mq/KWNt6PPZLdNpO1ekzrn6EfWz6l8sTHG74dgy07nI=", - "h1:vso6Er2XV/MsUTMtXIlz0AhIFx0TZnK73YNSZfA/9ZY=", - "zh:1f581ed8bc8b676f0127af7c018912ca82972e9348370d9f4dbe5b44005bb0ec", - "zh:1fa5b9c01ffbc7fdc507bbcffc177a1f1deb9f56f3d27183c6d92a11ea3208f8", - "zh:42b6524a9658df7093199b65eb210b3246a9f9822c8778a77e2d00a222441c03", - "zh:4c4f17fe2ae86087626dfd1055b0faea6a23fe3a12aabb2a7de3b9b2df6e3e41", - "zh:5346bb67d7154a8bc8c887b4b86bef95b3e92857d4304f55afea6d56ed152932", - "zh:591df2c3d4552f6f613415c900023d43e73e00e90312b7b1a666fe567219120f", - "zh:7156849aa03348c7556b746d3d9956293678de4a9e30c80480358dcdacb2a7fa", - "zh:75dee86262352f061ad072a5b873d3b3eabbb124ab0f20a803cfd44a87a31067", - "zh:7c33954aff1878bec78a50742897fae78dd2fbd19abeaeaca0c560ccc8f78810", - "zh:7d9cf36633dd991b41891d1c91b45b5c803852a902705453d6be4513524a574c", + "h1:+qDt35lVSK7acw6a1xHuPYrqmZEcHSmtd+6n1TxNuYw=", + "h1:1dCu2l4DhPBjizVAH/WwAjT1Xbo52K4PMvHoD5zUhuU=", + "h1:Dvn46Auwuel4jqrqZXs2D7kdujNhs17LEmqhuY0k4/4=", + "h1:M5eDL3m2uSEr1XATJW0foHzKl8pFhCtgKuOM24bJRwU=", + "h1:PddaC7nM/gY4x9i3xy6TxOs9MAu2/6g58Xs/gv4DRV8=", + "h1:QVIKiZluI+NQAKu8NpFBl3Nvyx+d81vW9btEUdIQREc=", + "h1:S6TnzXHsRoGYvC1vJBkDiVEc0spceksY4n6x5WN5iYw=", + "h1:VcxZDWqCWMSjcUsC1K4sB6uYEoeoou+BC0ePoJXmf3A=", + "h1:W0y/agBVqls1cJlFGFYMu2VnqoPXFzxVHPIYe3OqfYQ=", + "h1:XmNd5fP9a0O77ve5BMQP2vARExgIa7rYl6KvyUYXPSs=", + "h1:e0EFKrWSQwaa/kGhnha4DXk4T68Av8QxP84mRSdWC9M=", + "h1:eM+/lUiU0pNSgQKoqKPgE3xJrJ0MHIpKG+yhaGB/P0M=", + "h1:fPWWA4T0/y7GX+tCGN23l1jODhZ3uCdR/MKgZDXYpAE=", + "h1:g+r2GVi4gVC4DuQg3PL70gW9BDskgWUzCBIMXTUq63A=", + "h1:gaZ8eALDtVHqykVDHav8004gHiMGaYR/3KwET0FUgao=", + "h1:kbqW25eaiv4N/N/z+sxLdJZ15yh5cgnRD/q6RclPMLc=", + "h1:rGjxue3mXRyQQqpywTXC4zK//JAtf0Cz7RP+uPMMJjw=", + "zh:05943fef14c2028f4722bf078aa1889229e94302f7678cc6f63adb669d8ea612", + "zh:26a163930a92a7408f7bbd0130064b84df8a232b500d8c6c3989952986308539", + "zh:41305feaaade55391447521ec309f3c038b631ca542907ad95132fab71a7e116", + "zh:606919a930f0299948504adbdcd0f239a8af5c418f85741c48f8add370a3d038", + "zh:66963d5b445639511939fc508513fd31da3ee1d4ee1a565ee396c9532897a349", + "zh:6c981ec0c8545556395c43e2511861ab65ee9ecf2a960480e7889c3af0d23af3", + "zh:7334a1bdb726ce1f1bf0a3155f30f84f65206980c229c832ff5f0b0718c44e0b", + "zh:75f6c86bf74511e605423332d113711c76c8028361a32282fb3359d6c7ecae9e", + "zh:7aebb1a01cfe8be54903853202ae06eba14ad99c37d230ed93ce7d6633e05e9b", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9552ed1be3059b31544632d6293c2bfcd2474983e116c42702a6c408deb8e44d", - "zh:b3cb1f2bde0fd591d18109119f6235ff41c7ea6441777a356a64c348ee4ccdf7", - "zh:b5c1748403bf43274eba2574219cfa082351c8b1c1537a768acebef5cc5b2c0a", - "zh:c1b032e33aee40945770bbdf6002f777ee7b4ca541c00f4cf5eb23c09d32ef54", - "zh:d469680df9845fb506be27b2c838d39ccd8e98e26540b99936775f954299c35c", - "zh:dd59fa97fea4b10b901a6a749ebff81895f54126a2f08b4cd8f5baf6cfbbd854", - "zh:fb142d8ea02909587a4858374fe33d8609347150f4d32b7e15423320ffbd07b1", + "zh:9041d0e20c9ceea532de6eebf5cb3a27dad0bb49d3f5b5154be2a08d68fbbf1f", + "zh:a6bbf65431a02be4df0ebb1cbe01185ad357ff6e33c01bd0558f59bed90c8f36", + "zh:c6d075a31096f080c388dfe46036f451c0cc114c3311a4f46ab8dbe1938a202f", + "zh:dd8703f7b55b8bc8e10f8718bea889781100b18e932b04898995b63178c3d36e", + "zh:dd92a5cd4e133a4000e7e5bc8cce876ae0ed803543cedd2f3d590661ba244d04", + "zh:e024fdf121bebc48c1e6debea344c6d4f174117f3ae605fca6e13b9705d92d22", + "zh:ee0e80c31b438e35fa1608f6a2f5824d2806db1e5e8b9f7a90986585c7bcb895", + "zh:fc2d4b705411b48f8c045981f9368a3ea2f74969dd6302008c31ff0bedd51f0a", ] } diff --git a/provider.tf b/provider.tf index c22ce2d..badbbc1 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.16" + version = "~> 0.17" } } } From 0da29ab3f6ae2e5548ce22a6e886b3ec62cd79d0 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 7 Aug 2026 09:15:04 +0000 Subject: [PATCH 036/118] fix(index): add a slot repair for a broken Built stamp and retire alpine 3.21 (#150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `arm64/alpine321` and `arm64/alpine322` advertise a broken stamp: ``` arm64/alpine321 :: 22930 Built: R 4.4.0; NA; 2026-07-31 13:35:52 UTC; unix arm64/alpine322 :: 24696 Built: R 4.5.0; NA; 2026-07-31 13:51:54 UTC; unix ``` The per-minor sub-slots (`contrib/4.4`, `4.5`, `4.6`) are affected too. All 18 other slots are correct. uvr matches the stamp's platform triple plus R minor to pick binary over source, so nothing matches `NA` and both slots silently serve as source-only, which is exactly the regression bincraft#85 added the stamp to prevent. `install.packages()` is unaffected, since it reads `Built:` from each tarball's own `DESCRIPTION`. The tarballs are fine (`arm64/alpine322/.../dress.graph_0.8.3.tar.gz` carries `aarch64-unknown-linux-musl`), and so is the R that built them (`r-4.5.0_1_aarch64.apk` ships `R_PLATFORM='aarch64-unknown-linux-musl'`). Only the index is wrong. bincraft#96 stops a stamp like this being written again, but it cannot repair what is already there. ## Why not just re-run the index update `upload_package_index()` reuses the slot's remote `PACKAGES.db`, and cranlike's `update_db()` only reparses files whose md5 changed, so entries already in the database keep the stamp they were written with. Dropping `PACKAGES.db` to force a full reparse does work, and it is what bincraft#85's rollout note suggested, but for an S3 repo cranlike reads each package's metadata from the CRAN *source* mirror on GitHub. A 25k-entry slot is then 25k requests to raw.githubusercontent.com, with a real risk of being rate-limited part-way through and leaving the slot half-written. Only the `Built` column is wrong, so this corrects it in place instead: patch the column in `PACKAGES.db`, put the database back, and let `upload_package_index()` re-emit `PACKAGES*` from it. `update_db()` always rewrites the index files even when nothing was reparsed, so no tarball is re-read and nothing is fetched from GitHub. ## Change - `local/repair-built-stamp.R` — repairs the generic slot plus every per-minor sub-slot. Dry-run by default; `--apply` writes. The replacement comes from `bincraft::built_stamp()` under the R running the script, so it is exactly what a healthy run would have written, and bincraft#96's guard makes a broken build image fail rather than write a second bad stamp. - `.crow/repair-built-stamp.yaml` — manual pipeline, routed by `target_arch` to the matching agent group and platform image, with `dry_run` defaulting to `true`. - `.crow/archive-missed-packages.yaml` — drop the two `alpine321` matrix entries. Alpine 3.21 is EOL: the website advertises only 3.23/3.24 and `process-updates.yaml` already dropped it, so that slot is retired rather than repaired. ## Verification `crow lint .crow/` passes on all nine pipelines. `air format` and `jarl check` are clean; the script parses, and the `/opt/R` minor-version derivation was checked against `4.4.3 / 4.5.3 / 4.6.0 / current` → `4.4 4.5 4.6`. The repair itself is unrun by design — it needs B2 credentials and an arm64 agent. ## Rollout 1. Cut a bincraft release so `local/install-bincraft.R` picks up #96 (it resolves the latest `vX.Y.Z` tag, and #96 is only on `main`). 2. Run this pipeline with `target_arch=arm64`, `OS=alpine`, `OS_VERSION=3.22`, `R_VERSION=4.5.3`, `dry_run=true` and check the reported counts. 3. Re-run with `dry_run=false`. 4. Confirm: `curl -sS https://cran.devxy.io/arm64/alpine322/latest/src/contrib/PACKAGES | grep '^Built:' | sort | uniq -c` `arm64/alpine321` is deliberately left alone. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/150 --- .crow/archive-missed-packages.yaml | 4 - .crow/repair-built-stamp.yaml | 97 ++++++++++++++++ local/repair-built-stamp.R | 175 +++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+), 4 deletions(-) create mode 100644 .crow/repair-built-stamp.yaml create mode 100644 local/repair-built-stamp.R diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 71f808b..bd127d6 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -28,10 +28,6 @@ matrix: ARCH: amd64 - CODENAME: redhat-10 ARCH: arm64 - - CODENAME: alpine321 - ARCH: amd64 - - CODENAME: alpine321 - ARCH: arm64 - CODENAME: alpine322 ARCH: amd64 - CODENAME: alpine322 diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml new file mode 100644 index 0000000..12e2644 --- /dev/null +++ b/.crow/repair-built-stamp.yaml @@ -0,0 +1,97 @@ +### Manual repair of a slot whose PACKAGES index advertises a broken `Built` +### stamp (e.g. `Built: R 4.5.0; NA; ...`). +# +# uvr matches the stamp's platform triple plus R minor to decide binary vs +# source, so an unusable triple turns a whole slot source-only. See +# local/repair-built-stamp.R for why this patches PACKAGES.db in place instead +# of forcing a full reparse. +# +# Run with `dry_run: true` first: it reports how many entries are broken per +# slot and changes nothing. Pick the R version the slot should advertise, which +# is the R_VERSION its entry in .crow/process-updates.yaml uses. +variables: + target_arch: + description: 'Architecture of the slot to repair.' + options: + - amd64 + - arm64 + default: arm64 + OS: + description: 'Base OS image name.' + options: + - alpine + - redhat + - ubuntu + default: alpine + OS_VERSION: + description: 'OS image tag. Must match OS (alpine: 3.22/3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).' + options: + - '3.22' + - '3.23' + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + - 'resolute' + default: '3.22' + R_VERSION: + description: 'R version whose stamp the slot should advertise.' + options: + - 4.5.3 + - 4.4.3 + default: 4.5.3 + dry_run: + description: 'Report what would change without writing anything.' + options: + - 'true' + - 'false' + default: 'true' + +when: + - event: manual + evaluate: 'target_arch == "${ARCH}"' + +skip_clone: true + +labels: + platform: linux/${ARCH} + group: rpkgs-${ARCH} + +matrix: + include: + - ARCH: amd64 + - ARCH: arm64 + +steps: + - name: 'Repair Built stamp' + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - | + if [ "$dry_run" = "false" ]; then + /opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" --apply + else + /opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" + fi + backend_options: + docker: + resources: + requests: + memory: 2Gi + cpu: 1000m + limits: + memory: 8Gi + cpu: 2000m diff --git a/local/repair-built-stamp.R b/local/repair-built-stamp.R new file mode 100644 index 0000000..06f73be --- /dev/null +++ b/local/repair-built-stamp.R @@ -0,0 +1,175 @@ +#!/usr/bin/env Rscript + +### Rewrite a broken `Built` stamp across one arch/codename slot. +### +### A slot's index can end up advertising a stamp whose platform triple is +### unusable, e.g. `Built: R 4.5.0; NA; ...`. uvr picks binary vs source by +### matching that triple plus the R minor, so no client matches it and the whole +### slot silently reverts to source-only, which makes uvr compile everything and +### fail wherever a system `-dev` library is missing. +### +### Rewriting it is not a matter of re-running the normal index update. +### `upload_package_index()` reuses the slot's remote `PACKAGES.db`, and +### cranlike's `update_db()` only reparses files whose md5 changed, so entries +### already in the database keep the stamp they were written with. Dropping +### `PACKAGES.db` to force a full reparse does work, but for an S3 repo cranlike +### reads each package's metadata from the CRAN *source* mirror on GitHub, so a +### 25k-entry slot means 25k requests to raw.githubusercontent.com and a real +### risk of being rate-limited part-way through. +### +### Only the `Built` column is wrong, so correct it in place instead: patch the +### column in `PACKAGES.db`, put the database back, and let +### `upload_package_index()` re-emit `PACKAGES*` from it. `update_db()` always +### rewrites the index files even when nothing was reparsed, so no tarball is +### re-read and nothing is fetched from GitHub. +### +### The replacement comes from `bincraft::built_stamp()` under the R running +### this script, so run it under the R version the slot should advertise (the +### `R_VERSION` its entry in `.crow/process-updates.yaml` uses). That is what a +### healthy `upload_package_index()` run would have written. +### +### Usage, inside the platform's build image: +### Rscript local/repair-built-stamp.R [--apply] +### +### Without `--apply` it reports what it would change and touches nothing. + +suppressPackageStartupMessages({ + library(bincraft) +}) + +args <- commandArgs(trailingOnly = TRUE) +arch <- args[1L] +apply_changes <- "--apply" %in% args + +if (is.na(arch) || !nzchar(arch)) { + stop( + "Usage: Rscript local/repair-built-stamp.R [--apply]", + call. = FALSE + ) +} + +bucket <- "devxy-rpkgs-binaries" +endpoint <- "https://s3.eu-central-003.backblazeb2.com" +region <- "eu-central-003" + +codename <- bincraft::set_codename(NULL) +if (is.null(codename) || is.na(codename) || !nzchar(codename)) { + stop( + "Could not detect a codename from /etc/os-release; run this in a build image.", + call. = FALSE + ) +} + +# built_stamp() refuses an unusable platform, so a broken build image fails here +# rather than writing a second bad stamp over the first one. +stamp <- bincraft::built_stamp() +message(sprintf("Slot: %s/%s", arch, codename)) +message(sprintf("New stamp: %s", stamp)) + +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = endpoint, + region_name = region, + refresh = TRUE +) + +base_dir <- file.path(bucket, arch, codename, "latest", "src", "contrib") + +# A stamp is broken when its platform component is absent or literally "NA". +broken_stamp_where <- paste( + "Built IS NULL", + "OR Built LIKE '%; NA;%'", + "OR Built LIKE '%; ;%'" +) + +# The generic slot plus every per-minor sub-slot, which carry the same stamp and +# are poisoned by the same run. +r_minors <- sub( + "^.*/R/([0-9]+\\.[0-9]+)\\.[0-9]+$", + "\\1", + list.dirs("/opt/R", recursive = FALSE) +) +r_minors <- unique(grep("^[0-9]+\\.[0-9]+$", r_minors, value = TRUE)) +slots <- c(base_dir, file.path(base_dir, r_minors)) + +repair_slot <- function(slot) { + db_remote <- file.path(slot, "PACKAGES.db") + if (!s3fs::s3_file_exists(db_remote)) { + message(sprintf(" %s: no PACKAGES.db, skipping", slot)) + return(invisible(NULL)) + } + + db_local <- tempfile(fileext = ".db") + s3fs::s3_file_download(db_remote, db_local, overwrite = TRUE) + + con <- DBI::dbConnect(RSQLite::SQLite(), db_local) + on.exit(DBI::dbDisconnect(con), add = TRUE) + + total <- DBI::dbGetQuery(con, "SELECT COUNT(*) AS n FROM packages")$n + broken <- DBI::dbGetQuery( + con, + sprintf("SELECT COUNT(*) AS n FROM packages WHERE %s", broken_stamp_where) + )$n + + message(sprintf(" %s: %s entries, %s broken", slot, total, broken)) + if (broken == 0L) { + return(invisible(NULL)) + } + if (!apply_changes) { + message(" (dry run, pass --apply to rewrite)") + return(invisible(NULL)) + } + + DBI::dbExecute( + con, + sprintf("UPDATE packages SET Built = ? WHERE %s", broken_stamp_where), + params = list(stamp) + ) + DBI::dbDisconnect(con) + on.exit() + + s3fs::s3_file_upload(db_local, db_remote, overwrite = TRUE) + message(sprintf(" rewrote %s entries and uploaded PACKAGES.db", broken)) + invisible(NULL) +} + +invisible(lapply(slots, repair_slot)) + +if (!apply_changes) { + message("Dry run complete; nothing was changed.") + quit(save = "no") +} + +# Re-emit PACKAGES/PACKAGES.gz/PACKAGES.rds from the corrected database. Nothing +# is reparsed, because no tarball's md5 changed. +message("Re-emitting index files from the corrected database...") +bincraft::upload_package_index( + codename = codename, + s3_endpoint = endpoint, + s3_region = region, + s3_bucket = bucket, + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") +) +for (minor in r_minors) { + try( + bincraft::upload_package_index( + codename = codename, + r_minor = minor, + s3_endpoint = endpoint, + s3_region = region, + s3_bucket = bucket, + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") + ), + silent = FALSE + ) +} + +message("Done. Verify with:") +message(sprintf( + " curl -sS https://cran.devxy.io/%s/%s/latest/src/contrib/PACKAGES | grep '^Built:' | sort | uniq -c", + arch, + codename +)) From 77ce1d212dfdf791be19c768931b75ddd3991e09 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 7 Aug 2026 09:41:53 +0000 Subject: [PATCH 037/118] fix(ci): gate the Built-stamp repair on its own variable (#151) ## Problem `repair-built-stamp.yaml` gated on `target_arch`: ```yaml when: - event: manual evaluate: 'target_arch == "${ARCH}"' ``` That is the same variable `build-all-versions.yaml` and `build-all-versions-install-deps.yaml` gate on. A manual run passing `target_arch=arm64` to reach the repair therefore matched all three, so triggering a dry-run repair also queued a full arm64 rebuild. I hit this triggering the alpine 3.22 dry run (pipeline 10706), which I killed. `crow pipeline ps` renders empty states on this version, so I could not confirm from the CLI whether the rebuild workflows started before the kill or only sat queued behind the running cron jobs; no output was attributable to them. ## Change Gate on `repair_built_stamp` instead. Every other pipeline here already gates on a variable named after itself (`process_cran_updates`, `weekly_audit_missing`, `weekly_rebuild_missing`), and `target_arch` was the odd one out being shared by two. The header now records the collision and the exact invocation, so the next pipeline added here does not repeat it. ## Verification `crow lint` passes; `prek` clean. Grep of every trigger condition in `.crow/` confirms `repair_built_stamp` is unique and that no other pipeline gates on `OS`, `OS_VERSION`, `R_VERSION` or `dry_run` alone: ``` archive-missed-packages.yaml: task == "archive-missed-packages" build-all-versions.yaml: target_arch == "${ARCH}" build-all-versions-install-deps.yaml: target_arch == "${ARCH}" weekly-audit-missing.yaml: weekly_audit_missing == ... repair-built-stamp.yaml: repair_built_stamp == "${ARCH}" process-updates.yaml: process_cran_updates == ... weekly-rebuild-missing.yaml: weekly_rebuild_missing == ... ``` ## Note for whoever runs it `crow pipeline create` against this instance returned HTTP 504 while still creating the pipeline. Retrying on that error duplicates the run: I created four before noticing. Verify with `pipeline ls` rather than trusting the exit status. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/151 --- .crow/repair-built-stamp.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml index 12e2644..83821e9 100644 --- a/.crow/repair-built-stamp.yaml +++ b/.crow/repair-built-stamp.yaml @@ -9,9 +9,18 @@ # Run with `dry_run: true` first: it reports how many entries are broken per # slot and changes nothing. Pick the R version the slot should advertise, which # is the R_VERSION its entry in .crow/process-updates.yaml uses. +# +# The gate variable is `repair_built_stamp`, not `target_arch`: `target_arch` is +# what build-all-versions and build-all-versions-install-deps gate on, so a +# manual run passing it would start a full rebuild alongside this repair. Every +# pipeline here gates on a variable named after itself for exactly that reason. +# +# crow pipeline create --branch main \ +# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \ +# --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries variables: - target_arch: - description: 'Architecture of the slot to repair.' + repair_built_stamp: + description: 'Architecture of the slot to repair. Also gates this pipeline.' options: - amd64 - arm64 @@ -51,7 +60,7 @@ variables: when: - event: manual - evaluate: 'target_arch == "${ARCH}"' + evaluate: 'repair_built_stamp == "${ARCH}"' skip_clone: true From 2f732457d2af605415b1768a0992db7d9a17c1cf Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 7 Aug 2026 14:12:22 +0000 Subject: [PATCH 038/118] feat(edge): route PACKAGES requests to the per-R-minor slot (#152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `install.packages("curl")` fails in `reg.devxy.io/r/r-alpine:4.5-3.24` with "package 'curl' is not available for this version of R", on both arches. `curl` is not missing from the repo: it is in `…/latest/src/contrib/4.5/` and `…/4.6/`, the per-minor slots that base R cannot address. The image's repo URL resolves to `…/latest/src/contrib`, whose index does not list it. On `amd64/alpine324` that is 2 886 packages invisible to `install.packages()` (23 on `amd64/noble`) — what issue #63 records as "missing binaries". Two further findings while investigating: - The middleware only ever rewrote the bare `cran.rpkgs.com/src/contrib/…` form, and that form was broken for every Linux client on a stock R user agent: `ALPINE_REGEX`/`UBUNTU_REGEX`/`RHEL_REGEX` need a Posit-style UA that carries the distro, so stock R fell through to `extractOs()` and got redirected to `/amd64/linux-musl/latest/…`, a slot that does not exist. - `PACKAGES*` is served `cdn-cache: BYPASS` (bincraft uploads it `no-store`), so the middleware sees every index request and no purge is needed for routing changes to take effect. ## What this changes **`edge/rpkgs-router.ts`** — the middleware, now a reviewed file in this repo rather than dashboard state. It routes `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib//` for slots listed in `UNION_SLOTS`, and nothing else. Tarballs are deliberately left alone. R keeps the `contriburl` it *asked for*, not the one the redirect served it, so every tarball URL is resolved against the flat directory and the union index steers the per-minor ones with a `Path: ` field. Rewriting a tarball request here would send flat-slot packages into a directory that does not hold them. Also in the script: the phantom `linux-gnu`/`linux-musl` fallback is gone (an unidentifiable distro goes to CRAN, as an unparseable UA already did), and every redirect carries `Cache-Control: no-store` since its target depends on the User-Agent. The macOS branches are unchanged. **`cdn.tf`** — `bunnynet_compute_script.rpkgs_router` with `content = file("edge/rpkgs-router.ts")`, the `UNION_SLOTS` variable, and `middleware_script` pointing at the resource instead of the literal `29277`. `UNION_SLOTS` is empty, so merging and applying this changes no client's behaviour. A slot is added only once bincraft has republished its per-minor index as a union (rpkgs/bincraft#97); routing to a raw per-minor index would hide every package it does not carry. Rolling back is a variable edit, not a deploy. **`specs/`, `plans/`** — the design and the implementation plan, including the two approaches that were rejected (edge-side merge, moving the minor up the path) and why. ## Verification `just edge-test` runs 13 routing cases against the SDK's local server, so what is tested is the artifact that gets deployed; pass-through cases proxy to the real origin. All pass. End to end, with the middleware in front of a locally built union index for `amd64/alpine324` (31 507 records), inside the runtime image: ``` curl: 7.1.0 -> …/latest/src/contrib/4.5 -> curl_7.1.0.tar.gz 717 725 B jsonlite: 2.0.0 -> …/latest/src/contrib -> jsonlite_2.0.0.tar.gz 1 055 849 B ``` `tofu validate` passes. `tofu plan` has not been run: no `BUNNYNET_API_KEY` available in this environment. ## Before applying The script pre-dates this configuration, so it must be adopted, not created: ```sh tofu import bunnynet_compute_script.rpkgs_router 29277 tofu plan ``` The plan should show an in-place `content` update and no replacement of the pull zone. Without the import, tofu creates a second script and repoints the zone at it. Note that `name = "rpkgs-router"` will rename the existing script on apply. ## Not fixed here `install.packages("curl")` on `alpine324` will now *resolve*, then fail to build: that slot's tarballs are byte-identical CRAN **source** tarballs (no `Meta/`, no `Built:` in DESCRIPTION) which the index nevertheless stamps `Built: R 4.5.3; …-linux-musl`. Sampled: `amd64/alpine324` 3/12 binary, `arm64/alpine324` 13/30, `amd64/noble` 12/12, `amd64/alpine323` 17/20. That slot needs a rebuild, tracked separately. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/152 --- cdn.tf | 25 +- edge/rpkgs-router.test.ts | 160 ++++++++++++ edge/rpkgs-router.ts | 235 ++++++++++++++++++ justfile | 14 ++ plans/2026-08-07-per-minor-edge-routing.md | 166 +++++++++++++ ...026-08-07-per-minor-edge-routing-design.md | 174 +++++++++++++ 6 files changed, 773 insertions(+), 1 deletion(-) create mode 100644 edge/rpkgs-router.test.ts create mode 100644 edge/rpkgs-router.ts create mode 100644 plans/2026-08-07-per-minor-edge-routing.md create mode 100644 specs/2026-08-07-per-minor-edge-routing-design.md diff --git a/cdn.tf b/cdn.tf index 8a950f5..fe1b25f 100644 --- a/cdn.tf +++ b/cdn.tf @@ -52,6 +52,29 @@ ### cran.rpkgs.com +# The edge middleware that resolves the bare cran.rpkgs.com form to an +# / slot and routes PACKAGES* to the per-R-minor slot. The source of +# truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release. +# +# The script pre-dates this configuration, so it is adopted rather than created: +# tofu import bunnynet_compute_script.rpkgs_router 29277 +resource "bunnynet_compute_script" "rpkgs_router" { + type = "middleware" + name = "rpkgs-router" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +# Slots ("/", comma separated) whose per-minor index bincraft has +# already republished as a union of the per-minor and flat slots. Routing to a +# slot that is not listed here would hide every package the per-minor index does +# not carry, so this stays empty until a slot has been backfilled. +resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + default_value = "" + required = false +} + resource "bunnynet_pullzone" "cran_rpkgs_com" { name = "cran-rpkgs" @@ -64,7 +87,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { origin { type = "OriginUrl" url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" - middleware_script = 29277 + middleware_script = bunnynet_compute_script.rpkgs_router.id } routing { diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts new file mode 100644 index 0000000..553185d --- /dev/null +++ b/edge/rpkgs-router.test.ts @@ -0,0 +1,160 @@ +/** + * Routing matrix for `edge/rpkgs-router.ts`. + * + * The script is exercised through the SDK's local server rather than by + * importing its internals, so what is tested is the artifact that gets + * deployed. Requests that the script passes through are proxied to the real + * origin, which keeps the "no redirect" cases honest: they assert that the + * client reached the flat slot, not merely that no `Location` was set. + * + * Run with `just edge-test`. + */ +import { assertEquals } from 'jsr:@std/assert@1'; + +const SCRIPT = new URL('./rpkgs-router.ts', import.meta.url).pathname; +const BASE = 'http://127.0.0.1:8080'; +const UNION_SLOTS = 'amd64/alpine324'; + +const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; +const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)'; +const UA_CURL = 'curl/8.0.1'; + +const SLOT = '/amd64/alpine324/latest/src/contrib'; +const OTHER_SLOT = '/amd64/noble/latest/src/contrib'; + +interface Probe { + status: number; + location: string | null; + cacheControl: string | null; +} + +async function probe(path: string, userAgent: string): Promise { + const res = await fetch(BASE + path, { + headers: { 'User-Agent': userAgent }, + redirect: 'manual', + }); + await res.body?.cancel(); + return { + status: res.status, + location: res.headers.get('location'), + cacheControl: res.headers.get('cache-control'), + }; +} + +/** Kill tolerantly: the child has already exited if the script failed to load. */ +async function stopServer(child: Deno.ChildProcess): Promise { + try { + child.kill(); + } catch { + // already gone + } + await child.status; +} + +async function startServer(): Promise { + const child = new Deno.Command(Deno.execPath(), { + args: ['run', '-A', SCRIPT], + env: { UNION_SLOTS }, + stdout: 'null', + stderr: 'inherit', + }).spawn(); + + for (let attempt = 0; attempt < 150; attempt++) { + try { + const res = await fetch(`${BASE}/`, { + headers: { 'User-Agent': UA_CURL }, + redirect: 'manual', + }); + await res.body?.cancel(); + return child; + } catch { + await new Promise((resolve) => setTimeout(resolve, 200)); + } + } + + await stopServer(child); + throw new Error('edge script did not start listening on ' + BASE); +} + +Deno.test('rpkgs-router', async (t) => { + const server = await startServer(); + + try { + await t.step("routes an index request to the client's R minor", async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.status, 302); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + + await t.step('routes R 4.6 to its own slot', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R46_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`); + }); + + await t.step('routes PACKAGES and PACKAGES.rds too', async () => { + for (const file of ['PACKAGES', 'PACKAGES.rds']) { + const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/${file}`, `expected ${file} to be routed`); + } + }); + + await t.step('marks the redirect uncacheable', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.cacheControl, 'no-store'); + }); + + await t.step('leaves a slot outside UNION_SLOTS alone', async () => { + const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('never routes a tarball', async () => { + const res = await probe(`${SLOT}/jsonlite_2.0.0.tar.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('does not redirect a path already under a minor', async () => { + const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('leaves a client without an R version alone', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_CURL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('resolves the bare root to slot and minor', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_ALPINE); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + + await t.step('sends an unidentifiable distro to CRAN', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); + }); + + await t.step('keeps the macOS rewrite', async () => { + const res = await probe('/src/contrib/foo_1.0.tar.gz', UA_R45_DARWIN); + assertEquals(res.location, 'https://cran.rpkgs.com/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz'); + }); + + await t.step('keeps the macOS binary passthrough to CRAN', async () => { + const path = '/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz'; + const res = await probe(path, UA_R45_DARWIN); + assertEquals(res.location, `https://cran.r-project.org${path}`); + }); + + await t.step('collapses duplicate slashes before matching', async () => { + const res = await probe(`/amd64/alpine324//latest/src/contrib//PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + } finally { + await stopServer(server); + } +}); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts new file mode 100644 index 0000000..9cd410b --- /dev/null +++ b/edge/rpkgs-router.ts @@ -0,0 +1,235 @@ +/** + * Edge middleware for cran.rpkgs.com. + * + * Two jobs: + * + * 1. Resolve the bare `https://cran.rpkgs.com` form to a concrete + * `/` slot from the User-Agent, or send the client to CRAN when + * the distro cannot be identified. + * 2. Route `PACKAGES*` requests to the per-R-minor slot + * (`…/latest/src/contrib//`), so a stock `install.packages()` sees the + * packages that only exist there. + * + * Only index files are routed. Tarballs are deliberately left alone: R keeps + * the contrib URL it asked for, not the one it was redirected to, so every + * tarball URL is resolved against the flat directory and the union index steers + * the per-minor ones with a `Path: ` field. Rewriting a tarball request + * here would send flat-slot packages into a directory that does not hold them. + * + * Routing is gated on UNION_SLOTS. The raw per-minor index holds only the + * ABI-sensitive subset of a slot; it is safe to route to it only once bincraft + * has republished it as a union of the per-minor and flat slots. + * + * Deployed by OpenTofu from this file (`bunnynet_compute_script.rpkgs_router`). + * Test with `just edge-test`. + */ +import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; + +const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; +const CRAN_ORIGIN = 'https://cran.r-project.org'; + +/** Slots ("/", comma separated) whose per-minor index is a union. */ +const UNION_SLOTS = new Set( + (Deno.env.get('UNION_SLOTS') ?? '') + .split(',') + .map((slot) => slot.trim()) + .filter((slot) => slot.length > 0), +); + +/** `///latest/src/contrib[/]` */ +const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/; + +/** A path that already sits in a per-minor slot, e.g. `4.5/PACKAGES.gz`. */ +const MINOR_DIR_REGEX = /^\d+\.\d+\//; + +/** The only files this script routes. */ +const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; + +const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; + +const MACOS_BIN_REGEX = + /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; + +const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i; + +const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i; +const UBUNTU_CODENAMES: Record = { + '24.04': 'noble', + '22.04': 'jammy', +}; + +const ALPINE_REGEX = /(?:Alpine Linux(?:\s+VERSION_ID=)?|alpine-)\s*(\d+)\.(\d+)/i; + +/** + * R's own User-Agent is `R (4.5.3 x86_64-pc-linux-musl …)`; the Posit-style one + * some sites configure is `R/4.5.3 R (…)`. Both carry the minor, which is why + * per-minor routing works without the distro being identifiable. + */ +const R_MINOR_REGEXES = [/\bR\/(\d+)\.(\d+)/, /\bR \((\d+)\.(\d+)/]; + +function normalizePathname(pathname: string): string { + return pathname.replace(/\/{2,}/g, '/'); +} + +function redirectTo(location: string, status = 302): Response { + return new Response(null, { + status, + headers: { + Location: location, + // The target depends on the User-Agent, so the redirect itself must + // never be cached; only its target is a cacheable, UA-independent URL. + 'Cache-Control': 'no-store', + 'X-Via': 'MyMiddleware', + 'X-Rewritten-By': 'rpkgs-edge-middleware', + }, + }); +} + +function extractRMinor(userAgent: string): string | null { + for (const regex of R_MINOR_REGEXES) { + const match = userAgent.match(regex); + if (match) { + return `${match[1]}.${match[2]}`; + } + } + return null; +} + +function mapArch(arch: string): string { + if (arch === 'aarch64') return 'arm64'; + if (arch === 'x86_64') return 'amd64'; + return arch; +} + +function extractArch(userAgent: string): string { + const match = userAgent.match(/(x86_64|aarch64|arm64|i386|i686)/); + return match ? mapArch(match[1]) : ''; +} + +/** + * Identify the `/` slot from the User-Agent, or null. + * + * A stock R User-Agent carries only `linux-gnu` / `linux-musl`, which are not + * slot names: returning them produced redirects into slots that do not exist + * (`/amd64/linux-musl/latest/…`, a guaranteed 404). An unidentifiable distro + * is reported as such so the caller can fall back to CRAN. + */ +function parseSlot(userAgent: string): string | null { + const arch = extractArch(userAgent); + if (!arch) { + return null; + } + + const rhel = userAgent.match(RHEL_REGEX); + if (rhel) { + return `${arch}/rhel${rhel[2]}`; + } + + const ubuntu = userAgent.match(UBUNTU_REGEX); + if (ubuntu) { + const codename = UBUNTU_CODENAMES[ubuntu[1]]; + if (codename) { + return `${arch}/${codename}`; + } + } + + const alpine = userAgent.match(ALPINE_REGEX); + if (alpine) { + return `${arch}/alpine${alpine[1]}${alpine[2]}`; + } + + return null; +} + +function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: string } | null { + const rverMatch = userAgent.match(/R \((\d+)\.(\d+)/); + const archMatch = userAgent.match(/(aarch64|arm64|x86_64)/); + const osMatch = userAgent.match(/darwin(\d+)/); + + if (!rverMatch || !archMatch || !osMatch) { + return null; + } + + const arch = archMatch[1] === 'aarch64' ? 'arm64' : archMatch[1]; + const darwinVer = parseInt(osMatch[1], 10); + const os = darwinVer >= 21 && darwinVer < 22 ? `monterey-${arch}` : `big-sur-${arch}`; + + return { os, arch, rver: `${rverMatch[1]}.${rverMatch[2]}` }; +} + +/** + * The contrib path a request should be served from, relative to the slot. + * + * Returns the per-minor path for an index file when the slot is known to carry + * a union index and the client's R minor is known; otherwise the flat path, + * which is what every client sees today. + */ +function contribPath(slot: string, rest: string, userAgent: string): string { + const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`; + + if (!INDEX_FILE_REGEX.test(rest) || !UNION_SLOTS.has(slot)) { + return flat; + } + + const rMinor = extractRMinor(userAgent); + return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; +} + +BunnySDK.net.http + .servePullZone({ url: 'https://cran.rpkgs.com/' }) + .onOriginRequest((ctx) => { + const url = new URL(ctx.request.url); + const path = normalizePathname(url.pathname); + const userAgent = ctx.request.headers.get('User-Agent') || ''; + + // macOS clients are served from CRAN's own binary tree. + const srcContrib = path.match(SRC_CONTRIB_REGEX); + if (srcContrib && /darwin/.test(userAgent)) { + const mac = parseMacUserAgent(userAgent); + if (mac) { + return Promise.resolve( + redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`), + ); + } + } + + if (MACOS_BIN_REGEX.test(path)) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + + // Already-qualified slot URLs: what the runtime images have baked in. + const slotPath = path.match(SLOT_PATH_REGEX); + if (slotPath) { + const slot = `${slotPath[1]}/${slotPath[2]}`; + const rest = slotPath[3]; + + // Never rewrite a request that is already in a per-minor slot, or the + // redirect would chase its own tail. + if (MINOR_DIR_REGEX.test(rest)) { + return Promise.resolve(ctx.request); + } + + const target = contribPath(slot, rest, userAgent); + if (target === path) { + return Promise.resolve(ctx.request); + } + return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`)); + } + + // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. + if (path === '/' || path === '/src/contrib' || path.startsWith('/src/contrib/')) { + const slot = parseSlot(userAgent); + if (!slot) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + + const rest = srcContrib ? srcContrib[1] : ''; + return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`)); + } + + return Promise.resolve(ctx.request); + }) + .onOriginResponse((ctx) => { + ctx.response.headers.append('X-Via', 'MyMiddleware'); + return Promise.resolve(ctx.response); + }); diff --git a/justfile b/justfile index a9a705e..6c75b13 100644 --- a/justfile +++ b/justfile @@ -73,3 +73,17 @@ rebuild os tag arch package *versions: --build-arg CACHEBUST="$(date +%s)" \ -f docker/build-one.Dockerfile \ local + +# run the edge middleware routing matrix (uses a local deno, else the deno image) +edge-test: + #!/usr/bin/env bash + set -euo pipefail + if command -v deno >/dev/null 2>&1; then + deno test -A edge/rpkgs-router.test.ts + else + docker run --rm \ + -v "$PWD:/w" -w /w \ + -v deno-cache:/deno-dir \ + denoland/deno:latest \ + deno test -A edge/rpkgs-router.test.ts + fi diff --git a/plans/2026-08-07-per-minor-edge-routing.md b/plans/2026-08-07-per-minor-edge-routing.md new file mode 100644 index 0000000..3c3e320 --- /dev/null +++ b/plans/2026-08-07-per-minor-edge-routing.md @@ -0,0 +1,166 @@ +# Per-R-minor edge routing implementation plan + +Spec: `specs/2026-08-07-per-minor-edge-routing-design.md` + +**Goal:** let a stock `install.packages()` see the per-minor packages by routing `PACKAGES*` requests to `…/src/contrib//`, where `bincraft` publishes a union index. + +**Architecture:** the union is built in `bincraft`; the edge script only redirects index requests, gated on a `UNION_SLOTS` script variable; the script lives in this repo and is applied by OpenTofu. + +**Tech stack:** Deno / TypeScript (Bunny Edge Scripting, SDK 0.12), OpenTofu with `BunnyWay/bunnynet` 0.17, R (bincraft). + +## Global constraints + +- Redirect only `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds`; never a tarball, because the union index already carries the correct tarball URL for both classes of package. +- Every redirect carries `Cache-Control: no-store`; redirect targets stay UA-independent. +- `UNION_SLOTS` is empty by default, so deploying the script is a no-op until a slot is backfilled. +- A slot is `/`, e.g. `amd64/alpine324`. +- Verified prerequisites: `PACKAGES*` is served `cdn-cache: BYPASS`, so the script sees every index request; `Deno.env.get()` reads script variables; the SDK local server listens on `127.0.0.1:8080`. + +--- + +## Task 1: Edge script and its test matrix + +**Files:** + +- Create: `edge/rpkgs-router.ts` +- Create: `edge/rpkgs-router.test.ts` +- Modify: `justfile` (add `edge-test`) + +**Produces:** a single-file script deployable as `bunnynet_compute_script.content`, reading `UNION_SLOTS` from the environment. + +- [ ] **Step 1: write the test matrix first** + +`edge/rpkgs-router.test.ts` spawns `deno run -A edge/rpkgs-router.ts` with `UNION_SLOTS=amd64/alpine324`, waits for `127.0.0.1:8080`, and issues requests with `redirect: "manual"`. + +Cases, asserted on the `location` header (or its absence): + +| # | path | User-Agent | expectation | +| --- | ------------------------------------------------------ | --------------------------------------------- | ------------------------------------------------------------ | +| 1 | `/amd64/alpine324/latest/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)` | 302 → `…/src/contrib/4.5/PACKAGES.gz` | +| 2 | same | `R (4.6.0 …)` | 302 → `…/src/contrib/4.6/PACKAGES.gz` | +| 3 | same, but slot `amd64/noble` | `R (4.5.3 …)` | no redirect (slot not in `UNION_SLOTS`) | +| 4 | `…/src/contrib/curl_7.1.0.tar.gz` | `R (4.5.3 …)` | no redirect | +| 5 | `…/src/contrib/4.5/PACKAGES.gz` | `R (4.5.3 …)` | no redirect (loop guard) | +| 6 | `…/src/contrib/PACKAGES.gz` | `curl/8.0` | no redirect (no R minor) | +| 7 | `/src/contrib/PACKAGES.gz` | alpine UA with `Alpine Linux … 3.24` | 302 → `/amd64/alpine324/latest/src/contrib/4.5/PACKAGES.gz` | +| 8 | `/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)`, no distro | 302 → `cran.r-project.org`, **not** a `linux-musl` slot | +| 9 | `/src/contrib/foo_1.0.tar.gz` | `R (4.5.1 aarch64-apple-darwin20 …)` | 302 → `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | +| 10 | `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | any | 302 → `cran.r-project.org` | +| 11 | any redirect above | — | `cache-control: no-store` | + +- [ ] **Step 2: run the tests and watch them fail** + +`just edge-test` → every case fails, because `edge/rpkgs-router.ts` does not exist. + +- [ ] **Step 3: write `edge/rpkgs-router.ts`** + +Order of evaluation in `onOriginRequest`: + +1. normalise `//` runs in the path +2. darwin `/src/contrib/*` → `/bin/macosx//contrib//` +3. `/bin/macosx/**` → CRAN +4. `/{arch}/{os}/latest/src/contrib/`: pass through if `rest` already starts with `/`, or is not an index file, or the slot is not in `UNION_SLOTS`, or the UA has no R minor; otherwise redirect into `/` +5. `/`, `/src/contrib`, `/src/contrib/**`: resolve arch+os from the UA, redirect to CRAN when the distro is unidentifiable, otherwise redirect to the qualified path, adding `/` under the same index-file rule +6. anything else: pass through + +The R minor comes from either `R/4.5.3` or `R (4.5.3 …)`, so a stock UA is enough. The `linux-gnu` / `linux-musl` fallback in `parseUserAgent` is deleted: those are not slot names. + +- [ ] **Step 4: run the tests until they pass** + +`just edge-test` + +- [ ] **Step 5: commit** + +```bash +git add edge/rpkgs-router.ts edge/rpkgs-router.test.ts justfile +git commit -m "feat(edge): route PACKAGES requests to the per-R-minor slot" +``` + +--- + +## Task 2: Manage the script from OpenTofu + +**Files:** + +- Modify: `cdn.tf` + +**Consumes:** `edge/rpkgs-router.ts` from Task 1. + +- [ ] **Step 1: add the resources** + +```terraform +resource "bunnynet_compute_script" "rpkgs_router" { + type = "middleware" + name = "rpkgs-router" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + default_value = "" + required = false +} +``` + +and replace `middleware_script = 29277` with `middleware_script = bunnynet_compute_script.rpkgs_router.id`. + +- [ ] **Step 2: validate** + +`tofu init -backend=false && tofu validate` + +- [ ] **Step 3: import the existing script (needs `BUNNYNET_API_KEY`)** + +```bash +tofu import bunnynet_compute_script.rpkgs_router 29277 +tofu plan +``` + +The plan must show an in-place `content` update and **no** replacement of the pull zone. A replacement means the import did not take. + +- [ ] **Step 4: commit** + +```bash +git add cdn.tf +git commit -m "feat(cdn): manage the edge middleware script from this repo" +``` + +--- + +## Task 3: Union index writer in bincraft + +**Files (repo `codefloe.com/rpkgs/bincraft`):** + +- Modify: `R/package_index.R` +- Test: `tests/testthat/test-package_index.R` + +**Produces:** `write_union_index(flat_records, minor_records)` returning the merged records, called from `upload_package_index()` when `r_minor` is set. + +- [ ] **Step 1: write the failing tests** + +- a package present in both slots keeps the per-minor record, with `Path = "4.5"` +- a package only in the flat slot survives with no `Path` +- a package only in the per-minor slot survives with `Path = "4.5"` +- a union smaller than the flat input raises an error rather than returning + +- [ ] **Step 2: run them and watch them fail** + +`Rscript -e 'testthat::test_file("tests/testthat/test-package_index.R")'` + +- [ ] **Step 3: implement `write_union_index()` and call it from `upload_package_index()`** + +After `update_PACKAGES()` has written the per-minor index, read the flat slot's `PACKAGES.rds`, set `Path = ` on the per-minor records, drop the flat records for packages the per-minor slot already has, and rewrite `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` in the per-minor slot. + +- [ ] **Step 4: run the tests until they pass** + +- [ ] **Step 5: commit and open the PR against bincraft** + +--- + +## Task 4: Roll out slot by slot + +- [ ] Re-index one slot (`amd64/alpine324`, R 4.5) and confirm the union index lists both `curl` (per-minor, `Path: 4.5`) and `jsonlite` (flat, no `Path`). +- [ ] Set `UNION_SLOTS = "amd64/alpine324"` and confirm in `reg.devxy.io/r/r-alpine:4.5-3.24` that `available.packages()` returns the union count and `"curl" %in% rownames(...)`. +- [ ] Add `arm64/alpine324`, then the remaining slots. + +`install.packages("curl")` will still fail to build on `alpine324` until that slot's source tarballs are replaced with real binaries. That is tracked separately. diff --git a/specs/2026-08-07-per-minor-edge-routing-design.md b/specs/2026-08-07-per-minor-edge-routing-design.md new file mode 100644 index 0000000..b55812c --- /dev/null +++ b/specs/2026-08-07-per-minor-edge-routing-design.md @@ -0,0 +1,174 @@ +# Design: Routing clients to per-R-minor binary slots + +Date: 2026-08-07 +Status: Approved (pending spec review) + +## Problem + +`bincraft` routes ABI-"risky" packages to a per-minor slot `…/latest/src/contrib//` and indexes every directory independently (`upload_package_index()` calls `cranlike::update_PACKAGES()` on one prefix at a time). +Nothing unions those indices, and `contrib.url()` only ever yields `/src/contrib`, so no value of `options(repos)` can address a per-minor slot. +Only `uvr` resolves per-minor URLs, which means the per-minor slots are invisible to `install.packages()` by construction. + +Measured on 2026-08-07: + +| slot | flat `src/contrib` | `src/contrib/4.5` | unique packages only in the per-minor slot | +| ----------------- | ------------------ | ----------------- | ------------------------------------------ | +| `amd64/alpine324` | 21 640 | 3 310 | 2 886 | +| `amd64/noble` | 24 495 | 398 | 23 | + +This is what issue #63 records as "missing binaries" on `alpine324`. +The packages are not missing; they are in a directory base R cannot reach. +The user-visible symptom in `reg.devxy.io/r/r-alpine:4.5-3.24` is: + +``` +> install.packages("curl") +Warning message: +package 'curl' is not available for this version of R +``` + +A second, unrelated defect exists on the same slot and is **out of scope here**: many `alpine324` tarballs are byte-identical CRAN _source_ tarballs that the index nevertheless stamps `Built: R 4.5.3; …-linux-musl`. +Routing exposes `curl`; only a rebuild of that slot makes it install. + +## Goal + +Let a stock `install.packages()` see one complete package list for its own R minor, without duplicating tarballs and without an R-version-varying cache key anywhere in the CDN. + +## Key constraint that drives the design + +R resolves a package's download URL from the index, not from the request path, and it keeps the `contriburl` it _asked for_ rather than the one it was redirected to. +Measured with `options(repos = …/latest)` against a middleware that redirects the index into `4.5/`: + +``` +curl available: TRUE +curl repo: …/latest/src/contrib # the flat URL, not the 4.5 one it was served from +``` + +So the union index is always addressed relative to the **flat** directory, whatever path it was fetched from. +`available.packages()` honours a `Path:` field and folds it into the `Repository` column, which gives the whole routing for free: + +- a per-minor record carries `Path: `, so its tarball is fetched from `…/src/contrib//` +- a flat record carries no `Path`, so its tarball is fetched from `…/src/contrib/` + +Verified end to end against the live CDN with a locally built union index for `amd64/alpine324` (31 507 records): + +``` +curl: 7.1.0 -> …/latest/src/contrib/4.5 -> curl_7.1.0.tar.gz 717 725 B +jsonlite: 2.0.0 -> …/latest/src/contrib -> jsonlite_2.0.0.tar.gz 1 055 849 B +``` + +The corollary is that the edge script must **not** rewrite tarball requests: every tarball URL is already correct when it leaves the client, and redirecting one into `/` would break exactly the flat packages the union is meant to preserve. + +The complementary trick does not work: R's `gzcon()` reads only the first member of a concatenated gzip stream (10 291 of an expected 31 931 records), so an edge-side merge would have to fully decompress and recompress both indices and additionally 404 `PACKAGES.rds` to stop R preferring it. +That is why the union is produced in `bincraft`, not at the edge. + +## Approaches considered + +| Approach | Where the union lives | Verdict | +| ----------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- | +| **A. Union index written by `bincraft` (chosen)** | per-minor `PACKAGES*`, per-minor entries carry `Path: ` | Edge does one redirect; `PACKAGES.rds` stays correct; no duplication | +| B. Merge at the edge | middleware fetches both indices, recompresses | ~2 MB decompress/recompress per cache fill, cache key must include the R minor, breaks R's `.rds` fast path | +| C. Move the minor up the path (`latest//src/contrib/`) | addressable by `options(repos)` directly | No edge logic at all, but a full layout migration and breaks the published URL contract | + +Chosen: **A**. + +## Architecture + +### bincraft: union index (separate PR) + +After writing a per-minor index, republish it as a union of that slot and the flat slot: + +1. Read the flat slot's `PACKAGES.rds` and the per-minor slot's own records. +2. Set `Path: ` on every per-minor record, so its tarball resolves into the per-minor directory. +3. Drop every flat record whose package is already present in the per-minor slot, so the per-minor build always wins, and leave the survivors without a `Path`. +4. Write the merged `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib//`. + +Guard: refuse to publish a union with fewer records than the flat index it was built from. +A truncated union is worse than no union, because it silently removes packages from every client on that minor. + +### Edge script (this repo) + +The script routes `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` requests, and nothing else. + +``` +normalize path +parseClient(UA) -> { rMinor, arch, os } # rMinor from "R (4.5.3 …)" or "R/4.5.3" +darwin branches # unchanged +if path is /{arch}/{os}/latest/src/contrib/PACKAGES* + already under // ? pass through # loop guard + rMinor known && slot in UNION_SLOTS ? 302 -> …/src/contrib//PACKAGES* + else pass through # flat slot, today's behaviour +if path is /src/contrib/… # bare root + resolve arch + os; unknown -> 302 to CRAN + then apply the same PACKAGES* rule +else pass through +``` + +Redirects carry `Cache-Control: no-store`. +Every cacheable URL is therefore UA-independent, and no cache key has to vary by R version. + +### Repaired bare-root detection + +The bare `https://cran.rpkgs.com` form is currently broken for every Linux client that uses a stock R user agent. +`ALPINE_REGEX`, `UBUNTU_REGEX` and `RHEL_REGEX` only match a Posit-style user agent the user has to set by hand; stock R never carries the distro, so the script falls through to `extractOs()` and redirects to a slot that does not exist: + +``` +UA: R (4.5.3 x86_64-pc-linux-musl …) -> 302 /amd64/linux-musl/latest/… (404) +UA: R (4.5.3 x86_64-pc-linux-gnu …) -> 302 /amd64/linux-gnu/latest/… (404) +``` + +The fallback to a phantom `linux-musl` / `linux-gnu` slot is removed. +An unidentifiable distro redirects to CRAN, which is the existing behaviour for an unparseable user agent. +The R _minor_ is always present in a stock user agent, so per-minor routing itself does not depend on distro detection. + +### Rollout gate + +`UNION_SLOTS` is a `bunnynet_compute_script_variable` listing the slots whose per-minor index is already a union. +It is empty by default, so deploying the script changes nothing until `bincraft` has backfilled a slot, and a rollback is a variable edit rather than a code deploy. +All slots currently carry `4.4`, `4.5` and `4.6`; a client on any other minor falls through to the flat slot. + +### Deployment from this repo + +The script is a file in the repo, applied by the existing OpenTofu configuration: + +``` +edge/rpkgs-router.ts # the script +edge/rpkgs-router.test.ts # UA x path -> expected Location matrix +cdn.tf # bunnynet_compute_script + _variable +``` + +Provider `BunnyWay/bunnynet` v0.17.0 (already pinned) ships `bunnynet_compute_script` with `content` loadable via `file()`, plus `bunnynet_compute_script_variable`. +`middleware_script = bunnynet_compute_script.rpkgs_router.id` replaces the hard-coded `29277`, after a one-time `tofu import` of the existing script. + +## Error handling + +- Unknown R minor, or a slot not listed in `UNION_SLOTS`: pass through to the flat slot. + The client sees exactly today's behaviour. +- Unparseable distro on the bare-root form: redirect to CRAN. +- A request already under `…/src/contrib//`: pass through, so a redirect can never loop. +- A per-minor slot that does not exist for a listed minor: the client gets the origin's 404. + `UNION_SLOTS` is the operator's assertion that the slot is ready, so this is a configuration error, not a runtime condition to paper over. + +## Testing + +Local, before any apply: `deno run -A edge/rpkgs-router.ts` serves the middleware against the real origin, so `edge/rpkgs-router.test.ts` drives the whole matrix against that local server. + +- User agent matrix: R 4.4 / 4.5 / 4.6 on musl and gnu, both arches, Posit-style and stock forms, plus a darwin UA and a non-R UA. +- Path matrix: `PACKAGES`, `PACKAGES.gz`, `PACKAGES.rds`, a tarball, a path already under `4.5/`, and `/src/contrib/…` on the bare root. +- Assertion is the `Location` header (or its absence), not the body. + +After apply, a smoke test against `cran.rpkgs.com`: + +- `available.packages()` inside `reg.devxy.io/r/r-alpine:4.5-3.24` returns the union count, and `"curl" %in% rownames(...)` is `TRUE`. +- A flat-slot package still downloads from `…/src/contrib/`, and a per-minor package downloads from `…/src/contrib//`. + +## Out of scope + +- `Meta/archive.rds` stays flat-only, so `remotes::install_version()` does not see per-minor archives. +- The `alpine324` source-tarball defect: that slot serves CRAN sources stamped as binaries, and needs a rebuild independent of this work. +- Any change to how `uvr` resolves per-minor URLs; it already addresses the slots directly. + +## Split of work + +1. `bincraft`: union index writer plus its guard, and a re-index of one slot to validate. +2. This repo: `edge/rpkgs-router.ts`, its test matrix, and the `cdn.tf` resources with `UNION_SLOTS` empty. +3. Enable `UNION_SLOTS` slot by slot as `bincraft` backfills them. From 6bc8213a2f7e229e1d953e170c987023a04dfcd3 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sat, 8 Aug 2026 00:32:07 +0000 Subject: [PATCH 039/118] chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker to v3.10.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80650cc..2334bd5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.8.0 + rev: v3.10.0 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ From 4cb8b692ab4d38b10e785a0f947e83dd3c098bd7 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sun, 9 Aug 2026 00:32:28 +0000 Subject: [PATCH 040/118] chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker to v3.11.1 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2334bd5..82461cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.10.0 + rev: v3.11.1 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ From 1e843523a0312f3943fd11aca8f64e0117f7eca9 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 10:02:05 +0000 Subject: [PATCH 041/118] fix(ci): gate the three ungated pipelines on their own variable (#155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem A manual `crow pipeline create` instantiates **every** pipeline in `.crow/`, and each one decides for itself whether to run. Three had nothing to decide with — their only manual condition was a bare `event: manual`: - `auto-apply-patches` — pushes to `auto/registry-patch-proposals` and opens/updates a PR - `weekly-patch-proposals` — posts and edits two Forgejo issues - `trial-build-registry` — starts a build per matrix row, on both arches So they fired on *any* manual trigger in this repo, whatever it was for. That is how they came to run alongside a manual `process-updates` run for `alpine-324-arm64` (#10723), which is also why that pipeline is marked failure. `repair-built-stamp.yaml` already documents the rule this breaks: > The gate variable is `repair_built_stamp`, not `target_arch` … Every pipeline here gates on a variable named after itself for exactly that reason. ## What this changes Each of the three gets a gate variable named after the pipeline, `evaluate`d on the manual event, defaulting to off: ```yaml variables: auto_apply_patches: description: 'Run the auto-patch proposer. Also gates this pipeline.' options: ['true', 'false'] default: 'false' when: - event: manual evaluate: 'auto_apply_patches == "true"' - event: cron cron: auto-apply-patches ``` Cron triggers are untouched, so the scheduled runs behave exactly as before. The run-manually comments in all three headers were also stale: they documented `--var task=` with `woodpecker-cli`, and no pipeline evaluates a `task` variable. They now show the real invocation. ## Note on the sibling pipelines The already-gated pipelines use `default: all` (e.g. `weekly_rebuild_missing`). If Crow applies a declared default to a variable that an API-created pipeline never passed, those would match on an unrelated manual run too — `weekly-rebuild-missing` would be an expensive way to find out. I could not settle that from #10723 because its step logs have since expired, so I left them alone rather than guess. The three fixed here default to `'false'`, which is safe under either semantics. ## Verification `crow lint .crow/` passes. Auditing every pipeline that accepts a manual event now reports a gate on all ten: ``` build-all-versions-install-deps.yaml: gated auto-apply-patches.yaml: gated weekly-patch-proposals.yaml: gated weekly-audit-missing.yaml: gated repair-built-stamp.yaml: gated archive-missed-packages.yaml: gated weekly-rebuild-missing.yaml: gated trial-build-registry.yaml: gated build-all-versions.yaml: gated process-updates.yaml: gated ``` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/155 --- .crow/auto-apply-patches.yaml | 15 ++++++++++++++- .crow/trial-build-registry.yaml | 16 ++++++++++++++-- .crow/weekly-patch-proposals.yaml | 18 ++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index 0076353..7c33c5b 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -9,14 +9,27 @@ # FORGEJO_TOKEN is used for both the branch push and opening the PR (no separate # write-scoped secret needed). Register the `auto-apply-patches` cron in the crow # UI, or run manually: -# woodpecker-cli pipeline create --var task=auto-apply-patches --branch=main 7 +# crow pipeline create --branch main \ +# --var auto_apply_patches=true devxy/build-cran-binaries +# +# The gate variable is `auto_apply_patches`, named after the pipeline: a manual +# run instantiates every pipeline in `.crow/`, so one without its own gate runs +# on *any* manual trigger in this repo. This one pushes a branch and opens a PR, +# so it must stay off unless it is what was asked for. variables: + auto_apply_patches: + description: 'Run the auto-patch proposer. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' patch_limit: description: 'Max candidates to propose per run (top by failure volume).' default: '10' when: - event: manual + evaluate: 'auto_apply_patches == "true"' - event: cron cron: auto-apply-patches diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 73856a1..2195423 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -7,15 +7,27 @@ # # The repo uses no `pull_request` triggers, so this runs manually against the # branch (or on a cron); point it at the auto-patch branch via `patch_branch`: -# woodpecker-cli pipeline create --var task=trial-build-registry \ -# --var patch_branch=auto/registry-patch-proposals --branch=main 7 +# crow pipeline create --branch main --var trial_build_registry=true \ +# --var patch_branch=auto/registry-patch-proposals devxy/build-cran-binaries +# +# The gate variable is `trial_build_registry`, named after the pipeline: a +# manual run instantiates every pipeline in `.crow/`, so one without its own +# gate runs on *any* manual trigger in this repo. This one starts a build per +# matrix row on both arches, which is far too expensive to fire by accident. variables: + trial_build_registry: + description: 'Trial-build the branch new registry entries. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' patch_branch: description: 'Branch whose new registry entries to trial-build.' default: auto/registry-patch-proposals when: - event: manual + evaluate: 'trial_build_registry == "true"' - event: cron cron: trial-build-registry diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 027d41c..712e87d 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -8,8 +8,26 @@ # Global across platforms (the classifier groups over all of single_builds), so # a single job -- no matrix. Clones read-only; the only writes are the two # Forgejo issues via FORGEJO_TOKEN. +# +# Run manually with: +# crow pipeline create --branch main \ +# --var weekly_patch_proposals=true devxy/build-cran-binaries +# +# The gate variable is `weekly_patch_proposals`, named after the pipeline: a +# manual run instantiates every pipeline in `.crow/`, so one without its own +# gate runs on *any* manual trigger in this repo. This one posts and edits +# Forgejo issues, so an unrelated manual run must not fire it. +variables: + weekly_patch_proposals: + description: 'Run the weekly failure triage. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' + when: - event: manual + evaluate: 'weekly_patch_proposals == "true"' - event: cron cron: weekly-patch-proposals From 0c86b2692d07e18decc55b4738a3cfa2562ab892 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 10:08:40 +0000 Subject: [PATCH 042/118] fix(cdn): declare the User-Agent cache vary instead of dropping it (#156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `tofu plan` after importing the middleware script shows: ``` ~ resource "bunnynet_pullzone" "cran_rpkgs_com" { ~ cache_vary_headers = [ - "User-Agent", ] ``` The zone carries `cache_vary_headers = ["User-Agent"]`, set before this configuration existed. `cdn.tf` never declared it, so the first apply of the managed middleware would remove it — as a side effect of an unrelated change, with no decision recorded anywhere. ## What this changes Declares the attribute with its current value, so the pull zone is a no-op in that plan. ## Why keep it rather than let it go On paper the router makes it redundant. The only UA-dependent responses it produces are redirects, and those carry `Cache-Control: no-store`; their targets are concrete per-slot, per-minor URLs whose content depends only on the path. Dropping the vary would also be a genuine win, since otherwise every distinct R version string (`R (4.5.3 x86_64-pc-linux-musl …)`) keys its own copy of every tarball. It stays anyway, for now: - it is the second line of defence against the one failure mode that would be quiet and confusing — an R 4.6 client served the 4.5 index - Bunny honouring `no-store` on an edge-script response has been confirmed for today's redirects (`cdn-cache: BYPASS` on `max-age=0`), but not for the new script in production - keeping it is the status quo, so it cannot regress anything Removing it is worth doing on its own, once per-minor routing is confirmed live and the redirects can be observed bypassing cache — not as a side effect of enabling that routing. ## Verification `tofu validate` passes. Re-planning after this merges should leave `bunnynet_pullzone.cran_rpkgs_com` unchanged, reducing the plan to the script `content` update and the new `UNION_SLOTS` variable. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/156 --- cdn.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cdn.tf b/cdn.tf index fe1b25f..5e8899d 100644 --- a/cdn.tf +++ b/cdn.tf @@ -105,6 +105,21 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { request_coalescing_enabled = true block_post_requests = true + # Set on the zone since before this configuration existed; declared here so + # `tofu apply` stops silently removing it. + # + # The router makes it redundant on paper: the only UA-dependent responses it + # produces are redirects, and those carry `Cache-Control: no-store`, while + # their targets are concrete per-slot, per-minor URLs whose content depends + # only on the path. Dropping it would also be a real win, because otherwise + # every distinct R version string keys its own copy of every tarball. + # + # It stays for now anyway: it is the second line of defence against the one + # failure that would be quiet and confusing (an R 4.6 client served the 4.5 + # index), and removing it is worth doing on its own once per-minor routing is + # confirmed live, not as a side effect of enabling that routing. + cache_vary_headers = ["User-Agent"] + limit_requests = 5000 limit_connections = 1000 From 070396908819b9365e5cf824e3f387ee875993bf Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 10:37:41 +0000 Subject: [PATCH 043/118] fix(audit): count a source fallback as a missing binary (#157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The audit's definition of "missing" was `missing_dt <- cran_dt[!s3_dt]`, where `s3_dt` came from listing the bucket. But an object being present does not mean a binary was built: when a build fails, bincraft publishes the CRAN source tarball in its place so the package stays installable, and a listing cannot tell the two apart. So every failed build looked done. On `amd64/alpine324` that is 13 547 of 24 134 comparable objects (56%) — all CRAN sources from the June 2026 bootstrap, none ever reported here, so `weekly-rebuild-missing` never retried any of them. `amd64/noble` sits at 4.6% for comparison. ## What this changes Reads the slot's own `PACKAGES` index instead of listing the bucket, and counts only records carrying a `Built` stamp. bincraft writes that stamp only for what it actually built (rpkgs/bincraft#105), so the index answers the question a listing cannot. Side effects of reading the index rather than the bucket: - the audit reports how many records the index holds and how many are served as source, so the gap is visible in the log - `s3fs` is no longer used, so the audit needs no B2 credentials - an object present but absent from the index is now counted as missing, which is correct: clients only see the index ## Sequencing This is inert until the matching bincraft change ships and a slot is re-indexed. A slot last indexed by an older bincraft carries `Built` on every record, including the fallbacks, so the audit reports exactly what it does today. Verified against the live indices: ``` amd64/alpine324: records=24152 built=24152 source-served=0 amd64/noble: records=24681 built=24681 source-served=0 ``` After a re-index those `source-served` counts become the real ones, and the packages behind them start appearing in issue #63 and getting rebuilt. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/157 --- local/weekly-missing-binaries-audit.R | 78 +++++++++++++++------------ 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index 7b0b65d..e7ca776 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -7,7 +7,6 @@ options(error = function() { suppressPackageStartupMessages(library(data.table)) library(DBI, quietly = TRUE) library(RPostgres, quietly = TRUE) -library(s3fs, quietly = TRUE) library(jsonlite, quietly = TRUE) library(httr2, quietly = TRUE) @@ -50,8 +49,8 @@ cat(sprintf( )) # --------------------------------------------------------------------------- -# 1. Query PostgreSQL for known build failures (before s3fs init to avoid -# C++ pointer conflicts between s3fs/curl and RPostgres/libpq) +# 1. Query PostgreSQL for known build failures (before anything that uses curl, +# to avoid C++ pointer conflicts between curl and RPostgres/libpq) # --------------------------------------------------------------------------- cat("Connecting to PostgreSQL...\n") con <- DBI::dbConnect( @@ -97,62 +96,71 @@ cran_dt <- data.table( ) # --------------------------------------------------------------------------- -# 3. S3 tarballs +# 3. Published binaries # --------------------------------------------------------------------------- -cat("Connecting to S3...\n") -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE -) - -s3_path <- sprintf( - "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", +# Read the slot's own index rather than listing the bucket. An object being +# present does not mean a binary was built: when a build fails, bincraft +# publishes the CRAN source tarball in its place so the package stays +# installable, and a bucket listing cannot tell the two apart. That is how +# amd64/alpine324 came to hold 13,547 CRAN sources that this audit never +# reported. bincraft stamps `Built` only on records it actually built, so the +# index answers the question a listing cannot. +# +# A slot last indexed by a bincraft that predates the source-fallback fix +# stamps `Built` on every record, including the fallbacks, so this reports +# exactly what it used to until that slot is re-indexed. +index_url <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", arch, s3_codename ) -cat(sprintf("Listing S3 path: %s\n", s3_path)) +cat(sprintf("Reading package index: %s\n", index_url)) -s3_pkgs <- tryCatch( - s3fs::s3_dir_ls(s3_path, recurse = FALSE), +index <- tryCatch( + { + con <- gzcon(url(index_url, open = "rb")) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "Built")) + }, error = function(e) { cat(sprintf( - "WARNING: Could not list S3 path %s: %s\n", - s3_path, + "WARNING: Could not read %s: %s\n", + index_url, conditionMessage(e) )) - character(0) + NULL } ) -file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) -parts <- parts[sapply(parts, length) == 3] -if (length(parts) == 0) { - s3_dt <- data.table(Package = character(0), Version = character(0)) +if (is.null(index) || nrow(index) == 0) { + binary_dt <- data.table(Package = character(0), Version = character(0)) } else { - s3_dt <- data.table( - Package = sapply(parts, `[`, 2), - Version = sapply(parts, `[`, 3) + built <- !is.na(index[, "Built"]) + binary_dt <- data.table( + Package = as.character(index[built, "Package"]), + Version = as.character(index[built, "Version"]) ) + cat(sprintf( + "Index holds %d records, %d of them built binaries (%d served as CRAN source)\n", + nrow(index), + sum(built), + sum(!built) + )) } cat(sprintf( - "S3 contains %d tarballs for %s/%s\n", - nrow(s3_dt), + "S3 contains %d binaries for %s/%s\n", + nrow(binary_dt), arch, s3_codename )) # --------------------------------------------------------------------------- -# 4. Find missing packages (CRAN release version not in S3) +# 4. Find missing packages (CRAN release version without a binary in S3) # --------------------------------------------------------------------------- setkey(cran_dt, Package, Version) -setkey(s3_dt, Package, Version) -missing_dt <- cran_dt[!s3_dt] +setkey(binary_dt, Package, Version) +missing_dt <- cran_dt[!binary_dt] cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) # --------------------------------------------------------------------------- From f8e31af75b3ae418b858dca3d6baeda73e651834 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 15:31:14 +0000 Subject: [PATCH 044/118] fix(ci): make every manual gate default to a value that matches nothing (#158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem A manual `crow pipeline create` instantiates **every** file in `.crow/`, and a declared variable default is applied even when the run never passed that variable. A gate is therefore only a gate if its default matches nothing. #155 fixed the three pipelines that had no manual gate at all. It missed that a *permissive default* leaves a pipeline just as exposed. Demonstrated the expensive way: creating a pipeline with only ``` --var weekly_audit_missing=alpine-324-amd64 ``` also started `build-all-versions` — because its gate `target_arch` defaults to `amd64`, which matches its own amd64 matrix rows — and `process-updates` across every row, because that gate defaults to `all`. The run was killed before any `Upload package indexes` step produced output and both alpine324 indices were verified unchanged, but `build-all-versions` uploads binaries and rewrites indexes, so the next one might not be caught in time. Before: | pipeline | gate | default | fired on an unrelated manual run | | --- | --- | --- | --- | | `build-all-versions` | `target_arch` | `amd64` | amd64 rows — builds and uploads | | `build-all-versions-install-deps` | `target_arch` | `amd64` | amd64 rows | | `weekly-rebuild-missing` | `weekly_rebuild_missing` | `all` | every row | | `weekly-audit-missing` | `weekly_audit_missing` | `all` | every row | | `process-updates` | `process_cran_updates` | `all` | every row | | `repair-built-stamp` | `repair_built_stamp` | `arm64` | arm64 rows | `archive-missed-packages` was the one that behaved, because its gate variable is never declared and so matches nothing. That is the property this restores everywhere. ## What this changes Each of the six gets a `none` option on its gate variable and defaults to it, so a manual run has to name its target explicitly. The reason is recorded next to the default, where someone would go to change it. `none` is used rather than dropping the default so the expression always has a defined value to compare, instead of relying on undefined-variable semantics. Cron triggers are untouched — they match on the `cron:` name, not the variable. ## Verification `crow lint .crow/` reports all ten configs valid. Auditing every pipeline that accepts a manual event: ``` archive-missed-packages.yaml: gate=task default= auto-apply-patches.yaml: gate=auto_apply_patches default='false' build-all-versions-install-deps.yaml gate=target_arch default=none build-all-versions.yaml: gate=target_arch default=none process-updates.yaml: gate=process_cran_updates default=none repair-built-stamp.yaml: gate=repair_built_stamp default=none trial-build-registry.yaml: gate=trial_build_registry default='false' weekly-audit-missing.yaml: gate=weekly_audit_missing default=none weekly-patch-proposals.yaml: gate=weekly_patch_proposals default='false' weekly-rebuild-missing.yaml: gate=weekly_rebuild_missing default=none ``` Every gate now defaults to something that matches no matrix row. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/158 --- .crow/build-all-versions-install-deps.yaml | 8 ++++++-- .crow/build-all-versions.yaml | 8 ++++++-- .crow/process-updates.yaml | 8 ++++++-- .crow/repair-built-stamp.yaml | 8 ++++++-- .crow/weekly-audit-missing.yaml | 8 ++++++-- .crow/weekly-rebuild-missing.yaml | 8 ++++++-- 6 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 448cff3..2a6d9c2 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -3,12 +3,16 @@ # Variables are declared so the manual-run form exposes them (crow #1165); # they are merged with build-all-versions' identical declarations. variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. target_arch: - description: 'Architecture to build.' + description: 'Architecture to build, or "none" to run nothing.' options: + - none - amd64 - arm64 - default: amd64 + default: none OS: description: 'Base OS image name.' options: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 9ab888a..fb1229d 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -4,12 +4,16 @@ # image and cache volume. Placement is via the group label (rpkgs-amd64/rpkgs-arm64). # Skip list lives in local/excluded-packages.json (read by local/build-all.R). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. target_arch: - description: 'Architecture to build.' + description: 'Architecture to build, or "none" to run nothing.' options: + - none - amd64 - arm64 - default: amd64 + default: none OS: description: 'Base OS image name.' options: diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 1e4833c..7600876 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -7,9 +7,13 @@ # ("all" = every os/arch). # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. process_cran_updates: - description: "Manual run target: a specific -, or 'all' for every os/arch." + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." options: + - none - all - alpine-322-amd64 - alpine-322-arm64 @@ -29,7 +33,7 @@ variables: - ubuntu-2404-arm64 - ubuntu-2604-amd64 - ubuntu-2604-arm64 - default: all + default: none when: - event: cron diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml index 83821e9..e56e1e9 100644 --- a/.crow/repair-built-stamp.yaml +++ b/.crow/repair-built-stamp.yaml @@ -19,12 +19,16 @@ # --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \ # --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. repair_built_stamp: - description: 'Architecture of the slot to repair. Also gates this pipeline.' + description: 'Architecture of the slot to repair, or "none" to run nothing.' options: + - none - amd64 - arm64 - default: arm64 + default: none OS: description: 'Base OS image name.' options: diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 4efe409..3a9d98f 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -7,9 +7,13 @@ # ("all" = every os/arch). # Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. weekly_audit_missing: - description: "Manual run target: a specific -, or 'all' for every os/arch." + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." options: + - none - all - alpine-322-amd64 - alpine-322-arm64 @@ -29,7 +33,7 @@ variables: - ubuntu-2404-arm64 - ubuntu-2604-amd64 - ubuntu-2604-arm64 - default: all + default: none when: - event: cron diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 03a13f9..0353901 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -8,9 +8,13 @@ # single - to run just one. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, or 'all' for every os/arch." + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." options: + - none - all - alpine-322-amd64 - alpine-322-arm64 @@ -30,7 +34,7 @@ variables: - ubuntu-2404-arm64 - ubuntu-2604-amd64 - ubuntu-2604-arm64 - default: all + default: none when: - event: cron From a126d74cd37422d2c0a9e8dec782173f945585eb Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 16:27:51 +0000 Subject: [PATCH 045/118] fix(build): keep source fallbacks out of the S3 package cache (#159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem This is the gap flagged in rpkgs/bincraft#106. `build_binary_package()` has a fast path that compares against `s3_package_cache` instead of querying S3 per package, and that cache is produced here: ```r s3_pkgs <- s3fs::s3_dir_ls(".../latest/src/contrib", recurse = TRUE) saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds") ``` A raw bucket listing cannot tell a binary from a package whose build failed and was published as its CRAN source — the two occupy the same key. So every source fallback reads as "already built" and is skipped for good. That is how `alpine324` accumulated ~13.5k of them. The same listing feeds `s3_dt`, which is subtracted from the build list at line 194 (`pkgs <- pkgs_no_error[!s3_dt]`). That one matters more: it excludes the very packages that need building, before `build_binary_package()` is even called. ## What this changes Drops from the listing every object the slot's own index reports as served from source. bincraft leaves the `Built` stamp off exactly those records (rpkgs/bincraft#105), so the index already carries the answer and no credentials, downloads or extra API calls are needed. Both consumers are fixed: the saved cache and `s3_dt`. The cache stays a plain filename vector, so the build container still needs no `s3fs`/reticulate — that was the point of saving it in the first place. Two deliberately conservative edges: - archived objects have no index record, so they are kept. Unknown means binary, never "rebuild it". - if the index cannot be read, the full listing is kept and a warning is printed, so a CDN blip cannot mass-schedule a rebuild. ## Verification The script parses, and the new block run against the live indices: ``` amd64/alpine324: index=24235 source-served=13542 e.g. AATtools_0.0.3.tar.gz, ABCDscores_7.0.0.tar.gz amd64/noble: index=24681 source-served=0 ``` `alpine324` is re-indexed by bincraft 5.1.1, so 13 542 objects drop out and those packages become buildable. `noble` has not been re-indexed yet, so every record still carries `Built`, nothing is dropped, and its behaviour is exactly what it is today — the safe failure mode this relies on. The new log line makes it visible per run: ``` S3 cache: N objects, M served as CRAN source, K usable binaries ``` ## Sequencing Needs rpkgs/bincraft#106 (and a release) before a rebuild actually builds: this fixes the bulk build path's list, #106 fixes the per-package pre-build skip. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/159 --- local/packages-to-build.R | 59 +++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 887952d..3b508fb 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -89,14 +89,61 @@ s3_pkgs <- s3fs::s3_dir_ls( recurse = TRUE ) -# Save the raw S3 file listing for the build step to use as s3_package_cache +file_names <- basename(s3_pkgs) + +# An object occupying a key is not proof a binary was built: a package whose +# build failed has its CRAN source published under exactly that name. Left in +# the cache, `build_binary_package()` reads it as "already built" and skips the +# package forever, which is how alpine324 accumulated ~13.5k source tarballs. +# +# bincraft stamps `Built` only on records it actually built, so the slot's own +# index distinguishes them. A slot last indexed by a bincraft that predates that +# fix stamps `Built` on everything, so the cache is then unchanged from before. +# Archived objects have no index record and are kept: unknown means binary, +# never "rebuild it". +index_url <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", + arch, + codename +) +source_served <- tryCatch( + { + con_idx <- gzcon(url(index_url, open = "rb")) + on.exit(close(con_idx), add = TRUE) + idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) + sprintf( + "%s_%s.tar.gz", + idx[is.na(idx[, "Built"]), "Package"], + idx[is.na(idx[, "Built"]), "Version"] + ) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read %s (%s); keeping the full S3 cache\n", + index_url, + conditionMessage(e) + )) + character(0) + } +) + +binary_cache <- setdiff(file_names, source_served) +cat(sprintf( + "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", + length(file_names), + length(file_names) - length(binary_cache), + length(binary_cache) +)) + +# Save the S3 file listing for the build step to use as s3_package_cache. # This avoids loading s3fs/reticulate in the build container, saving memory for # the dependency-installer subprocesses -saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds") - -file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) +saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") +# Built from the filtered listing, not the raw one: `s3_dt` is subtracted from +# the build list below, so a source fallback left in here would exclude the very +# package that needs building. +matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", binary_cache) +parts <- regmatches(binary_cache, matches) parts <- parts[sapply(parts, length) == 3] s3_dt <- data.table( Package = sapply(parts, `[`, 2), From 6372f0f928f81c76aaca3c3b6f71541a04a976d4 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 10 Aug 2026 06:30:25 +0000 Subject: [PATCH 046/118] fix(ci): refresh the apt index before uvr installs system dependencies (#161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unblocks the ubuntu 26.04 builds without waiting for an image rebuild. ## What broke ``` i Installing R package dependencies ! Error in installing dependencies for package huge with tag 2.0.1: uvr sync --install-system-deps --library /mnt/cache/R-pkgs-4.4 failed (exit 1) ! WARN Missing system dependencies for 1 package(s) igraph needs: libglpk-dev > Running: apt-get install -y libglpk-dev E: Unable to locate package libglpk-dev ``` Neither the package name nor the sysreq mapping is wrong. `libglpk-dev` is `5.0-2build1` in `universe`, which `ubuntu:resolute` enables by default. What is missing is the apt index: uvr v0.4.5 runs `apt-get install` with no `apt-get update` before it, and every ubuntu build image ends its apt layers with `rm -rf /var/lib/apt/lists/*`, so the shipped image has no index at all. Not specific to 26.04 — jammy and noble fail identically for any sysreq the image's preinstall line doesn't already cover. 26.04 surfaced it first, through `igraph` and everything depending on it (`huge` was the first to fail). ## Fix `apt-get update` before the sync in `local/uvr-install.sh`. That one site covers every pipeline: `build-all-versions`, `build-all-versions-install-deps`, `weekly-rebuild-missing`, `weekly-audit-missing`, `trial-build-registry`, `auto-apply-patches` and `weekly-patch-proposals` all reach it, either directly or through `install-bincraft.R`. The index it populates persists for the rest of the step, so the bincraft-driven `uvr sync` calls that follow are covered too. Guarded on `apt-get` and non-fatal: `apk add` fetches its index implicitly and dnf refreshes expired metadata on its own, so alpine and redhat are unaffected and skip it. ## Why here as well as in the image The real fix is upstream `e491b2e`, which refreshes the index before the auto-install. It landed one day after v0.4.5 was tagged, so no release carries it ([nbafrank/uvr#250](https://github.com/nbafrank/uvr/issues/250)); [build-env-images#23](https://codefloe.com/rpkgs/build-env-images/pulls/23) pins the ubuntu image to a commit that has it. That only reaches CI after an image rebuild is triggered. This change takes effect on the next pipeline run. It also stays useful afterwards: a baked index goes stale within weeks, and a stale index turns the same install into a 404 on the `.deb`. ## Verified In `ubuntu:resolute`, with the image state reproduced exactly (R installed, then `rm -rf /var/lib/apt/lists/*` to empty the index again): | Check | Result | |---|---| | `apt-get install -y libglpk-dev`, empty index | `E: Unable to locate package libglpk-dev` — the reported failure, reproduced | | Same container after `apt-get update` | `Candidate: 5.0-2build1`, `Components: main universe restricted multiverse` | | `igraph` + `UVR_INSTALL_SYSREQS=1 uvr sync`, empty index, uvr with the refresh | `Setting up libglpk-dev:amd64 (5.0-2build1)`, `System dependencies installed.`, `Installed 11 package(s)` | | `sh -n` and `shellcheck local/uvr-install.sh` | pass | ## Reverting Delete the guarded block. It is marked TEMPORARY with the upstream reference, and can go once the images ship a uvr above v0.4.5. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/161 --- local/uvr-install.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/local/uvr-install.sh b/local/uvr-install.sh index 14b7f78..c5d37d3 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -87,4 +87,23 @@ cd "$project_dir" # --no-install: resolve and lock only. The install happens in the sync below, # which is the only command that honours --library. "$uvr_bin" add --no-install "$@" + +# TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs +# `apt-get install` for every resolved system dependency without refreshing the +# index first, and the ubuntu build images end their apt layers with +# `rm -rf /var/lib/apt/lists/*`. With no index apt cannot resolve a package that +# exists and is enabled, so `igraph needs: libglpk-dev` fails the whole build +# with `E: Unable to locate package libglpk-dev` on ubuntu 26.04. +# +# Fixed upstream in `e491b2e`, tagged one day after v0.4.5 (nbafrank/uvr#250), +# and the images pick it up via build-env-images#23 — but only after an image +# rebuild is triggered, which is why this runs here too. +# +# apt only: `apk add` fetches its index implicitly and dnf refreshes expired +# metadata on its own. Non-fatal, since a refresh failure still leaves whatever +# index is already there, and the install's own error is the more actionable one. +if command -v apt-get >/dev/null 2>&1; then + apt-get update -qq || echo "warning: apt-get update failed; continuing" >&2 +fi + "$uvr_bin" sync --library "$target_lib" --install-system-deps From 01b8ab43dfffee0e71bc63617275d5cffe5b2d88 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 10 Aug 2026 06:30:35 +0000 Subject: [PATCH 047/118] fix(rebuild): re-index and purge the CDN after a rebuild (#160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The rebuild now works — `AATtools 0.0.3` was detected as a source fallback, built, and published: ``` ℹ `upload_single_binary()`: Replacing the CRAN source published for AATtools 0.0.3 … with the binary. ✔ Successfully uploaded package AATtools with tag 0.0.3. ``` But clients still get the source, and will for about a year: ``` $ curl -sI .../src/contrib/AATtools_0.0.3.tar.gz etag: "ea8127d953ca6a2f118ea49441772af6" # CRAN's source MD5 cdn-cache: HIT cdn-cachedat: 08/09/2026 16:43:32 # predates the 18:12 upload ``` Two causes, both specific to a rebuild: 1. **The slot is never re-indexed.** `weekly-rebuild-missing` has no `upload_package_index` step, so the index keeps the old MD5 and — for anything that had been served from source — no `Built` stamp. This one self-heals at the next `process-updates` run. 2. **The tarball URL is never purged.** A normal update publishes new packages at *new* URLs, so `purge_cdn_cache.sh` only needs the five index files. A rebuild replaces an object *in place*, and the zone caches tarballs for `cache_expiration_time = 31919000` (~370 days). This does not self-heal. Nothing about a stale package looks wrong from the outside, which is what makes it worth fixing rather than documenting. ## What this changes **Re-index at the end of a rebuild**, flat and per-minor, mirroring the tail of `process-updates`. The codename is detected from the image's `/etc/os-release` (as `local/packages-to-build.R` already does) rather than adding `OS_ID` to all 18 matrix rows. **Purge the zone afterwards**, via a new `scripts/purge_cdn_zone.sh`. One call to `POST /pullzone/{id}/purgeCache` covers every replaced object, and all three hostnames — `cran.devxy.io`, `cran.allianceswisspass.devxy.io`, `cran.rpkgs.com` — share pull zone `3857050`, confirmed from the `cdn-pullzone` response header. Purging per URL was the alternative and is worse here: ~13.5k rate-limited calls per arch, where a single missed call leaves a package silently stale. The cost of the zone purge is a cold cache for everything else, which is why it stays out of the daily update path — `purge_cdn_cache.sh` is untouched. The purge runs on failure too (`when: status: [success, failure]`): a rebuild that died part-way still replaced objects, and those are exactly the ones a stale edge keeps hiding. ## Verification `crow lint .crow/` reports all ten configs valid; `bash -n` on the new script passes; prek hooks pass. Not yet exercised against Bunny — it needs `BUNNYNET_API_KEY`, which is a CI secret. The failure mode is explicit rather than silent: any status other than 200/204 prints the response body and exits non-zero. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/160 --- .crow/weekly-rebuild-missing.yaml | 33 ++++++++++++++++++++ scripts/purge_cdn_zone.sh | 52 +++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100755 scripts/purge_cdn_zone.sh diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 0353901..c86fe5a 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -163,6 +163,17 @@ steps: - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + # A rebuild replaces objects in place, so the slot's index still advertises + # the old MD5 and, for anything that had been served from source, no Built + # stamp. Re-index here rather than waiting for the next process-updates + # run, or the rebuilt binaries stay invisible to clients until then. + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done backend_options: docker: resources: @@ -172,3 +183,25 @@ steps: limits: memory: 18Gi cpu: 3000m + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + # All hostnames on the zone share this id, so one purge covers + # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. + BUNNY_PULLZONE: '3857050' + commands: + - apk add --no-cache -q bash curl git + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" + # A rebuild that died part-way still replaced objects, and those are exactly + # the ones a stale edge would keep hiding, so purge either way. + when: + - status: [success, failure] diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh new file mode 100755 index 0000000..391a814 --- /dev/null +++ b/scripts/purge_cdn_zone.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Purge the entire BunnyCDN pull zone. +# +# `purge_cdn_cache.sh` purges the five index files by URL, which is right after +# a normal update: new packages arrive at new URLs, so only the index is stale. +# +# A rebuild is different. It replaces an object *in place*: a package whose +# build failed was published as its CRAN source, and the rebuilt binary takes +# exactly the same URL. The zone caches tarballs for ~370 days +# (`cache_expiration_time` in cdn.tf), so without a purge every client keeps +# receiving the source tarball for up to a year, and nothing about it looks +# wrong from the outside. +# +# Purging per URL would mean one API call per replaced package -- ~13.5k per +# arch against a rate-limited endpoint, where a single missed call leaves a +# silently stale package. One zone purge is a single call regardless of how many +# objects were replaced. The cost is a cold cache for everything else, which is +# why this is not used by the daily update path. +# +# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io, +# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them. +# +# Usage: +# purge_cdn_zone.sh +# +set -euo pipefail + +if (($# < 2)); then + echo "usage: $0 " >&2 + exit 2 +fi + +api_key="$1" +zone_id="$2" + +echo "Purging BunnyCDN pull zone ${zone_id}" + +status=$( + curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \ + -H "AccessKey: ${api_key}" \ + -H "Content-Length: 0" \ + "https://api.bunny.net/pullzone/${zone_id}/purgeCache" +) + +if [[ "${status}" != "200" && "${status}" != "204" ]]; then + echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 + cat /tmp/purge_zone_response.txt >&2 + exit 1 +fi + +echo "Purged pull zone ${zone_id} (HTTP ${status})" From caf3276fb968c732f0a80792435e63973bf80c45 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 11 Aug 2026 00:31:56 +0000 Subject: [PATCH 048/118] chore(deps): update dependency nbafrank/uvr to v0.4.6 --- local/uvr-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local/uvr-install.sh b/local/uvr-install.sh index c5d37d3..1e25e47 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -30,7 +30,7 @@ set -eu # renovate: datasource=github-releases depName=nbafrank/uvr -UVR_PIN="v0.4.5" +UVR_PIN="v0.4.6" if [ "$#" -eq 0 ]; then echo "usage: $0 ..." >&2 From 4b7dc28cc83784fd350731b87f0d515df7c21299 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 12 Aug 2026 08:30:29 +0000 Subject: [PATCH 049/118] feat(rebuild): shard the weekly rebuild and make each shard resumable (#163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `weekly-rebuild-missing` runs one job per `-` and walks that slot's list serially in a single `R -q -e` argument. That was cheap while every source fallback was skipped as "already built". Since bincraft #105/#106/#107 and #159 the gate works, and the lists are large: 8 917 source-served records on `amd64/alpine324`, 15 023 on `amd64/resolute`. Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) ran for two days, reached `[8692/23885] cholera`, and was killed there. Two failures follow from that shape: - **No parallelism.** The work is embarrassingly parallel across packages; one job does all of it. - **No resumability and no clean stopping point.** The loop ends only by exhausting the list, so the only way to stop it is a kill. A restart re-walks from the first entry, paying a CRAN version resolution and an S3 `HEAD` per package before reaching new work. And a kill matches neither `success` nor `failure`, so the `Purge CDN cache` step never ran: the ~4 600 binaries 10910 did publish stayed hidden behind stale edge copies. ## What this changes **Three shards per slot.** Each of the 18 `OS`/`ARCH` rows gains `SPLIT_INTO`/`SPLIT_INDEX`, mirroring `build-all-versions.yaml`. Cron and manual routing are unchanged: both filters already match on `${OS}-${ARCH}`, so they now match all three shards of a slot. **`local/rebuild-missing.R`** replaces the ~1 500-character inline one-liner. The slice is interleaved rather than contiguous, because the list is alphabetical and cost clusters by name (`Rcpp*`, `Bioc*`, `rstan*`). **Resume by re-deriving state from the bucket.** One `s3_dir_info()` listing gives ETags for the slot; a package is outstanding iff its object's ETag equals CRAN's published `MD5sum`, i.e. it is still byte-identical to CRAN's source. That is `check_s3_root_package()` evaluated in bulk. No progress file, no volume, no DB cursor, and correct when a sibling shard or a `process-updates` run completes something concurrently. It reads ETags rather than the index's `Built` field the way `packages-to-build.R` does, because the index is no longer rewritten until the dependent pipeline runs and so cannot reflect the current run's progress. Unknown always means "already a binary", never "rebuild it": a multipart ETag, an unreadable CRAN index or an empty listing can never mass-schedule work. **A 20 h wall-clock budget** per shard. It exits 0, so the re-index and purge always fire and the remainder is picked up next run with no bookkeeping. **`.crow/weekly-rebuild-reindex.yaml`** takes over re-indexing and the purge, with `depends_on: [weekly-rebuild-missing]` and `runs_on: [success, failure]`. Three shards writing one slot's `PACKAGES` concurrently would race: `update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work. ## Verification `crow lint .crow/` passes on all 11 pipelines. `prek run` passes. 19 assertions in `local/tests/test-rebuild-missing.R`, 0 failures, covering the partition (disjoint, covering, deterministic, short lists, out-of-range index) and the outstanding filter (source ETag kept, binary ETag dropped, absent object kept, multipart and missing-from-CRAN treated as built). One of those tests caught a real bug before it shipped: an empty ETag table indexed to zero length rather than to `NA`, which recycled the result away and reported "nothing to build" — the dangerous direction. Fixed with an explicit `lookup()`. The filter run against the live `amd64/alpine324` index, using its `MD5sum` column as the ETag (established to match the objects): ``` index packages: 24343 outstanding (filter): 8950 no Built stamp: 8917 filter vs no-Built agreement: 8917 of 8917 outstanding but stamped Built: 33 (version drift vs CRAN) shard sizes: 2984/2983/2983 (sum 8950, unique 8950) ``` It reproduces the source-served set exactly. The extra 33 are packages whose slot version differs from CRAN's current one, so no object exists at the CRAN version key: correctly outstanding. ## Notes for review - The 20 h budget is a chosen default, exposed as `REBUILD_BUDGET_HOURS` in the pipeline. - `depends_on` is file-level, not row-level, so on a full cron run no slot is re-indexed until the slowest of all 54 jobs finishes. The budget bounds that at roughly a day. - An explicit cancel still skips the re-index. Recovery is to trigger `weekly-rebuild-reindex` on its own. - The purge runs on every re-index row rather than one designated slot: a cron fires only its own slot's row, so gating on a named slot would leave every other slot unpurged. - Out of scope: `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row, which is precisely the source-fallback set. Design: `specs/2026-08-12-shard-weekly-rebuild-design.md` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/163 --- .crow/weekly-rebuild-missing.yaml | 306 +++++++++++++++--- .crow/weekly-rebuild-reindex.yaml | 193 +++++++++++ local/rebuild-missing-helpers.R | 87 +++++ local/rebuild-missing.R | 194 +++++++++++ local/tests/test-rebuild-missing.R | 94 ++++++ .../2026-08-12-shard-weekly-rebuild-design.md | 167 ++++++++++ 6 files changed, 1005 insertions(+), 36 deletions(-) create mode 100644 .crow/weekly-rebuild-reindex.yaml create mode 100644 local/rebuild-missing-helpers.R create mode 100644 local/rebuild-missing.R create mode 100644 local/tests/test-rebuild-missing.R create mode 100644 specs/2026-08-12-shard-weekly-rebuild-design.md diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index c86fe5a..5afb3c3 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -1,12 +1,21 @@ # Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). -# One matrix row per OS/arch replaces the former per-platform files. +# Three matrix rows per OS/arch, one per shard of that slot's rebuild list. # Routing is preserved 1:1: # - cron: each existing `weekly-rebuild-missing--` cron fires only -# its matching matrix row (via the per-row `cron:` name filter). +# its matching matrix rows (via the per-row `cron:` name filter), +# which is now all three shards of that slot. # - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the # previous bare manual trigger that ran every os/arch); pick a # single - to run just one. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). +# +# The shard picks up its own slice and re-derives what is still outstanding +# from the bucket, so a restart resumes rather than replaying; see +# local/rebuild-missing.R. +# +# Re-indexing and the CDN purge deliberately do NOT live here. Three shards +# writing one slot's PACKAGES concurrently would race, so they moved to +# .crow/weekly-rebuild-reindex.yaml, which depends on this pipeline. variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed @@ -53,74 +62,326 @@ matrix: ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-8 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-9 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-9 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-10 ARCH: amd64 R_VERSION: 4.5.3 IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-10 ARCH: arm64 R_VERSION: 4.5.3 IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2204 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2204 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2404 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2404 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2604 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2604 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 steps: - name: 'Rebuild missing binaries' @@ -153,6 +414,12 @@ steps: PLATFORM: ${OS} ARCH: ${ARCH} NCPUS: 2 + SPLIT_INTO: ${SPLIT_INTO} + SPLIT_INDEX: ${SPLIT_INDEX} + # Wall clock after which the shard stops cleanly instead of having to be + # killed. A kill matches neither `success` nor `failure`, so it would skip + # the dependent re-index and leave rebuilt binaries behind a stale edge. + REBUILD_BUDGET_HOURS: 20 commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -162,18 +429,7 @@ steps: - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - # A rebuild replaces objects in place, so the slot's index still advertises - # the old MD5 and, for anything that had been served from source, no Built - # stamp. Re-index here rather than waiting for the next process-updates - # run, or the rebuilt binaries stay invisible to clients until then. - # The codename is detected from the image's /etc/os-release. - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 backend_options: docker: resources: @@ -183,25 +439,3 @@ steps: limits: memory: 18Gi cpu: 3000m - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - # All hostnames on the zone share this id, so one purge covers - # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. - BUNNY_PULLZONE: '3857050' - commands: - - apk add --no-cache -q bash curl git - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" - # A rebuild that died part-way still replaced objects, and those are exactly - # the ones a stale edge would keep hiding, so purge either way. - when: - - status: [success, failure] diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml new file mode 100644 index 0000000..224596b --- /dev/null +++ b/.crow/weekly-rebuild-reindex.yaml @@ -0,0 +1,193 @@ +# Re-index and purge after weekly-rebuild-missing. +# +# weekly-rebuild-missing runs three shards per slot. Each of them replaces +# objects in place, so the slot's index still advertises the old MD5 and, for +# anything that had been served from source, no Built stamp. Re-indexing from +# inside a shard would mean three concurrent `upload_package_index()` calls on +# one prefix: `cranlike::update_PACKAGES()` lists the live bucket, so an early +# lister that uploads last publishes an index missing its siblings' work. +# +# So it happens exactly once per slot, here, after every shard has finished. +# `runs_on: [success, failure]` keeps that true when a shard fails; only an +# explicit cancel skips it, and this pipeline can then be triggered on its own. + +variables: + # Mirrors the gate on weekly-rebuild-missing so a manual run re-indexes + # exactly the slots it rebuilt. A manual pipeline creation instantiates every + # file in .crow/, so the default must match no matrix row. + weekly_rebuild_missing: + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-rebuild-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' + +depends_on: + - weekly-rebuild-missing + +runs_on: [success, failure] + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + PLATFORM: ${OS} + ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + # All hostnames on the zone share this id, so one purge covers + # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. + BUNNY_PULLZONE: '3857050' + commands: + - apk add --no-cache -q bash curl git + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" + # Runs on every row rather than on one designated slot: a cron fires only + # its own slot's row, so gating on a named slot would leave every other + # slot unpurged. A manual "all" run therefore purges the zone 18 times, + # which is a cheap API call and rare. + # + # Run it even when the re-index above failed: the objects were still + # replaced, and a stale edge is exactly what keeps them hidden. + when: + - status: [success, failure] diff --git a/local/rebuild-missing-helpers.R b/local/rebuild-missing-helpers.R new file mode 100644 index 0000000..16a588e --- /dev/null +++ b/local/rebuild-missing-helpers.R @@ -0,0 +1,87 @@ +# Pure helpers for local/rebuild-missing.R, kept separate so local/tests can +# source them without executing a rebuild. + +# Interleaved slice of the rebuild list. +# +# The list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, +# rstan*), so contiguous thirds would be badly unbalanced. Interleaving also +# makes each shard's progress counter representative of the slot as a whole. +shard_slice <- function(pkgs, split_into, split_index) { + split_into <- as.integer(split_into) + split_index <- as.integer(split_index) + if (is.na(split_into) || is.na(split_index)) { + stop("shard_slice(): split_into and split_index must be integers") + } + if (split_into < 1L || split_index < 1L || split_index > split_into) { + stop(sprintf( + "shard_slice(): need 1 <= split_index <= split_into, got %s of %s", + split_index, + split_into + )) + } + # seq() errors on a descending range, which is what an empty list or a shard + # index past the end would produce. + if (length(pkgs) < split_index) { + return(pkgs[0L]) + } + pkgs[seq.int(split_index, length(pkgs), by = split_into)] +} + +# Packages that still need building, decided from the bucket rather than from +# remembered progress. +# +# This is bincraft's `check_s3_root_package()` evaluated in bulk: an object +# whose ETag equals CRAN's published MD5sum is byte-identical to CRAN's source, +# so the build that was supposed to replace it has not happened yet. +# +# `etag_by_file` named by `_.tar.gz`, values are unquoted ETags +# `cran_version` named by package +# `cran_md5` named by `_` +# +# Unknown always means "already a binary", never "rebuild it", so an unreadable +# CRAN index or a multipart ETag can never mass-schedule work. +outstanding_packages <- function(pkgs, etag_by_file, cran_version, cran_md5) { + if (length(pkgs) == 0L) { + return(pkgs) + } + + # An empty table indexes to zero length rather than to NA, which would + # recycle the whole result away and silently report "nothing to build". + lookup <- function(table, key) { + if (length(table) == 0L) { + return(rep(NA_character_, length(key))) + } + unname(as.character(table[key])) + } + + version <- lookup(cran_version, pkgs) + file <- sprintf("%s_%s.tar.gz", pkgs, version) + etag <- lookup(etag_by_file, file) + md5 <- lookup(cran_md5, paste(pkgs, version, sep = "_")) + + # No CRAN version means the package cannot be resolved to a tarball at all; + # leave it in and let bincraft report why. + unresolved <- is.na(version) + # No object at the key: never built, so it is outstanding by definition. + absent <- !unresolved & is.na(etag) + # A multipart upload carries a compound ETag rather than an MD5. + unknown <- !is.na(etag) & grepl("-", etag, fixed = TRUE) + + is_source <- !unresolved & + !is.na(etag) & + !unknown & + !is.na(md5) & + etag == md5 + + pkgs[unresolved | absent | is_source] +} + +parse_rebuild_args <- function(args) { + pos <- args[!startsWith(args, "--")] + budget <- as.numeric(pos[3L]) + list( + split_into = as.integer(pos[1L]), + split_index = as.integer(pos[2L]), + budget_hours = if (is.na(budget)) 20 else budget + ) +} diff --git a/local/rebuild-missing.R b/local/rebuild-missing.R new file mode 100644 index 0000000..369024e --- /dev/null +++ b/local/rebuild-missing.R @@ -0,0 +1,194 @@ +### Rebuild one shard of a slot's missing-binary list. +# +# Usage: Rscript local/rebuild-missing.R [budget_hours] +# +# The list itself comes from local/fetch-rebuild-packages-from-issue.R, which +# writes $REBUILD_PKG_LIST (default /tmp/rebuild_pkgs.txt). +# +# Two properties matter here and are the reason this is a script rather than an +# `R -q -e` argument in the pipeline: +# +# * it is restartable. The outstanding set is re-derived from the bucket on +# every start, so a shard that died resumes where it stopped without any +# progress file, and without replaying thousands of per-package HEADs. +# * it terminates. A wall-clock budget stops the loop cleanly instead of the +# run having to be killed, which is what previously skipped the re-index and +# CDN purge and left rebuilt binaries hidden behind stale edge copies. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +source(file.path("local", "rebuild-missing-helpers.R")) + +args <- parse_rebuild_args(commandArgs(trailingOnly = TRUE)) +if (is.na(args$split_into) || is.na(args$split_index)) { + stop("usage: rebuild-missing.R [budget_hours]") +} + +list_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") +pkgs <- if (file.exists(list_file)) readLines(list_file) else character(0) +pkgs <- pkgs[nzchar(pkgs)] +if (length(pkgs) == 0L) { + cat("Nothing to rebuild\n") + q("no") +} + +excluded <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] +pkgs <- setdiff(pkgs, excluded) + +mine <- shard_slice(pkgs, args$split_into, args$split_index) +cat(sprintf( + "Shard %s/%s: %s of %s listed packages\n", + args$split_index, + args$split_into, + length(mine), + length(pkgs) +)) + +### Resume: ask the bucket what is still outstanding + +codename <- bincraft::set_codename(NULL) +local_machine <- Sys.info()[["machine"]] +arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" +slot_dir <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", + arch, + codename +) + +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) + +# One paginated listing instead of a HEAD per package. Not recursed: the +# rebuild passes no `is_r_minor_sensitive`, so it only ever targets the flat +# path, and the resume filter matches that scope deliberately. +info <- tryCatch(s3fs::s3_dir_info(slot_dir), error = function(e) NULL) +etag_by_file <- if (is.null(info) || nrow(info) == 0L) { + cat(sprintf( + "WARNING: could not list %s; building the whole shard\n", + slot_dir + )) + stats::setNames(character(), character()) +} else { + stats::setNames( + gsub('^"|"$', "", as.character(info$etag)), + basename(as.character(info$uri)) + ) +} + +cran <- tryCatch( + { + con <- gzcon(url( + "https://cloud.r-project.org/src/contrib/PACKAGES.gz", + open = "rb" + )) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "MD5sum")) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read CRAN's index (%s)\n", + conditionMessage(e) + )) + NULL + } +) +cran_version <- stats::setNames(character(), character()) +cran_md5 <- stats::setNames(character(), character()) +if (!is.null(cran)) { + cran_version <- stats::setNames( + as.character(cran[, "Version"]), + as.character(cran[, "Package"]) + ) + keep <- !is.na(cran[, "MD5sum"]) + cran_md5 <- stats::setNames( + as.character(cran[keep, "MD5sum"]), + paste(cran[keep, "Package"], cran[keep, "Version"], sep = "_") + ) +} + +before <- length(mine) +mine <- outstanding_packages(mine, etag_by_file, cran_version, cran_md5) +cat(sprintf( + "Resume: %s of %s already carry a binary; %s outstanding\n", + before - length(mine), + before, + length(mine) +)) + +if (length(mine) == 0L) { + cat("Nothing outstanding for this shard\n") + q("no") +} + +### Build + +options( + crayon.enabled = TRUE, + Ncpus = as.integer(Sys.getenv("NCPUS", "2")), + future.globals.onReference = NULL +) + +started <- Sys.time() +n <- length(mine) +completed <- 0L +for (i in seq_along(mine)) { + elapsed <- as.numeric(difftime(Sys.time(), started, units = "hours")) + if (elapsed > args$budget_hours) { + cat(sprintf( + "Budget of %sh reached after %d/%d packages; stopping cleanly. The next run resumes from the bucket.\n", + args$budget_hours, + completed, + n + )) + break + } + + x <- mine[i] + cat(sprintf("[%d/%d] %s\n", i, n, x)) + tryCatch( + bincraft::build_binary_package( + x, + tag_limit = 1L, + patches = "local/patches", + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + upload = TRUE, + store_build_metadata = TRUE + ), + error = function(e) { + cat(sprintf("ERROR building %s - %s\n", x, conditionMessage(e))) + } + ) + completed <- completed + 1L +} + +cat(sprintf( + "Shard %s/%s finished: %d/%d packages processed in %.1fh\n", + args$split_index, + args$split_into, + completed, + n, + as.numeric(difftime(Sys.time(), started, units = "hours")) +)) diff --git a/local/tests/test-rebuild-missing.R b/local/tests/test-rebuild-missing.R new file mode 100644 index 0000000..a5e70b9 --- /dev/null +++ b/local/tests/test-rebuild-missing.R @@ -0,0 +1,94 @@ +source(file.path("..", "rebuild-missing-helpers.R")) + +test_that("shard_slice partitions the list without gaps or overlap", { + pkgs <- letters[1:10] + parts <- lapply(1:3, function(i) shard_slice(pkgs, 3, i)) + + expect_identical(parts[[1]], c("a", "d", "g", "j")) + expect_identical(parts[[2]], c("b", "e", "h")) + expect_identical(parts[[3]], c("c", "f", "i")) + + expect_identical(sort(unlist(parts)), sort(pkgs)) + expect_identical(anyDuplicated(unlist(parts)), 0L) +}) + +test_that("shard_slice is deterministic and survives short lists", { + expect_identical( + shard_slice(letters[1:10], 3, 2), + shard_slice(letters[1:10], 3, 2) + ) + expect_identical(shard_slice(character(0), 3, 1), character(0)) + # more shards than packages: the tail shards get nothing rather than erroring + expect_identical(shard_slice(c("a"), 3, 1), "a") + expect_identical(shard_slice(c("a"), 3, 2), character(0)) +}) + +test_that("shard_slice rejects an out-of-range index", { + expect_error(shard_slice(letters, 3, 4), "split_index") + expect_error(shard_slice(letters, 3, 0), "split_index") +}) + +test_that("outstanding_packages keeps source fallbacks and drops real binaries", { + cran_version <- c(httr = "1.4.8", R6 = "2.6.1", curl = "7.1.0") + cran_md5 <- c( + httr_1.4.8 = "8756015b94a9cff6f410ca4de8557f12", + R6_2.6.1 = "f01b1787f12797c29194d63c9afd5d70", + curl_7.1.0 = "8af2ccbf5d85dc18866f45f1f26f348d" + ) + etag <- c( + # byte-identical to CRAN: the build never happened + "httr_1.4.8.tar.gz" = "8756015b94a9cff6f410ca4de8557f12", + # a real binary was published + "R6_2.6.1.tar.gz" = "9d6087ee9adda3f0a3b8067cfc652c05" + # curl has no object at all + ) + + out <- outstanding_packages( + c("httr", "R6", "curl"), + etag, + cran_version, + cran_md5 + ) + expect_identical(out, c("httr", "curl")) +}) + +test_that("outstanding_packages treats unknowns as already built", { + cran_version <- c(a = "1.0", b = "1.0") + cran_md5 <- c(a_1.0 = "aaaa") + + # a multipart ETag carries no MD5, and `b` is missing from CRAN's index: + # neither may schedule a rebuild + etag <- c("a_1.0.tar.gz" = "abc-3", "b_1.0.tar.gz" = "bbbb") + + expect_identical( + outstanding_packages(c("a", "b"), etag, cran_version, cran_md5), + character(0) + ) +}) + +test_that("outstanding_packages keeps a package CRAN has no version for", { + out <- outstanding_packages( + "ghost", + c(), + c(other = "1.0"), + c(other_1.0 = "aaaa") + ) + expect_identical(out, "ghost") +}) + +test_that("outstanding_packages handles an empty list", { + expect_identical( + outstanding_packages(character(0), c(), c(), c()), + character(0) + ) +}) + +test_that("parse_rebuild_args defaults the budget", { + a <- parse_rebuild_args(c("3", "2")) + expect_identical(a$split_into, 3L) + expect_identical(a$split_index, 2L) + expect_identical(a$budget_hours, 20) + + b <- parse_rebuild_args(c("3", "2", "1.5")) + expect_identical(b$budget_hours, 1.5) +}) diff --git a/specs/2026-08-12-shard-weekly-rebuild-design.md b/specs/2026-08-12-shard-weekly-rebuild-design.md new file mode 100644 index 0000000..f4ccbaa --- /dev/null +++ b/specs/2026-08-12-shard-weekly-rebuild-design.md @@ -0,0 +1,167 @@ +# Design: Sharding and resuming the weekly rebuild + +Date: 2026-08-12 +Status: Approved (pending spec review) + +## Problem + +`weekly-rebuild-missing` runs one job per `-` and walks that slot's rebuild list serially in a single `R -q -e` invocation (`.crow/weekly-rebuild-missing.yaml:165`). +Until 2026-08-09 that was cheap, because every source fallback was skipped as "already built" and the list was effectively empty. +Since bincraft #105/#106/#107 and build-cran-binaries #159 the gate works, and the lists are now large. + +Share of records whose object is byte-identical to CRAN's source, measured against `cran.r-project.org` MD5s on 2026-08-12: + +| slot | records | source-served | share | +| ------------------ | ------: | ------------: | -----------: | +| `amd64/resolute` | 24 212 | 15 023 | 62.1% | +| `arm64/resolute` | 24 291 | 13 670 | 56.3% | +| `arm64/alpine324` | 24 328 | 9 514 | 39.2% | +| `amd64/alpine324` | 24 343 | 8 917 | 36.7% | +| `arm64/rhel10` | 24 695 | 5 384 | 21.9% | +| `amd64/rhel10` | 24 881 | 4 712 | 19.2% | +| 12 remaining slots | ~24 700 | 850 to 2 130 | 3.5% to 8.7% | + +A single serial job cannot absorb that. +Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) started on 2026-08-09, ran for roughly two days, reached `[8692/23885] cholera`, and was killed there. + +Two distinct failures follow from that shape. + +**No parallelism.** The work is embarrassingly parallel across packages, but one job does all of it. + +**No resumability, and no clean stopping point.** The loop has no terminating condition other than exhausting the list, so the only way to stop it is a kill. +A restarted run re-reads the same list and walks it from the first entry. +It skips completed packages via `check_s3_root_package()`, but that costs a CRAN version resolution and an S3 `HEAD` per package, thousands of times, before it reaches new work. +Worse, a kill is not a pipeline failure: the `Purge CDN cache` step is guarded by `when: status: [success, failure]` (`.crow/weekly-rebuild-missing.yaml:206-207`), and on 10910 it produced no output at all. +So the ~4 600 binaries that run did publish stayed hidden behind stale edge copies. + +## Goal + +Turn each slot's rebuild into bounded, parallel, restartable units, without introducing state that can disagree with the bucket. + +## Design + +### 1. Shard the matrix three ways + +Each of the 18 `OS`/`ARCH` rows in `.crow/weekly-rebuild-missing.yaml` gains `SPLIT_INTO: 3` and `SPLIT_INDEX: 1|2|3`, giving 54 rows. +This mirrors `.crow/build-all-versions.yaml:57-98`, which already shards its matrix four ways per arch. + +Routing needs no change. +The cron filter `cron: weekly-rebuild-missing-${OS}-${ARCH}` and the manual `evaluate: weekly_rebuild_missing == "${OS}-${ARCH}"` both match all three shards of a slot. +Placement stays on the `rpkgs-${ARCH}` group label, so shards queue against available capacity rather than oversubscribing it. + +### 2. Extract the loop into `local/rebuild-missing.R` + +The build is currently a single ~1 500-character `R -q -e` argument. +Shard arithmetic and resume logic do not belong in a YAML string, and none of it is testable there. +The loop moves to `local/rebuild-missing.R`, invoked as `Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX`, mirroring `local/build-all.R`. +Its body is unchanged in substance: read `/tmp/rebuild_pkgs.txt`, subtract `local/excluded-packages.json`, loop with `tryCatch` around `bincraft::build_binary_package()`. + +The slice is **interleaved**, not contiguous: + +```r +# the list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, +# rstan*), so contiguous thirds would be badly unbalanced +mine <- pkgs[seq(split_index, length(pkgs), by = split_into)] +``` + +`local/build-all.R:64` uses contiguous chunks via `cut()`. +That is fine there because its list is every CRAN package and version, so the chunks average out. +Here the list is a filtered backlog in which expensive families sit adjacent, so interleaving is the better default. +Interleaving also makes each shard's `[i/n]` progress representative of the slot as a whole. + +### 3. Resume by re-deriving state from the bucket + +Before the loop, the shard performs one `s3fs::s3_dir_info()` on `devxy-rpkgs-binaries///latest/src/contrib` and reads the `etag` column. +It fetches CRAN's `PACKAGES` once for the latest version and published `MD5sum` of every package. +A package is still outstanding if and only if the object at `_.tar.gz` has an ETag equal to CRAN's `MD5sum` for that version, which is the definition `check_s3_root_package()` already applies one package at a time. + +```r +# one paginated listing instead of ~2900 sequential HEAD requests per shard +info <- s3fs::s3_dir_info(slot_dir) +etag <- setNames(gsub('^"|"$', "", info$etag), basename(info$uri)) + +key <- sprintf("%s_%s.tar.gz", mine, cran_version[mine]) +# keep a package when no object exists yet, or when the object is still +# byte-identical to CRAN's source; drop it once a real binary is published +mine <- mine[is.na(etag[key]) | etag[key] == cran_md5[key]] +``` + +This is the whole resume mechanism. +There is no progress file, no volume, and no database cursor. +A restarted shard recomputes ground truth and continues where it stopped, and it is correct even when a sibling shard, a `process-updates` cron, or a manual `just rebuild` completed something in the meantime. + +Three properties make this the right source of truth: + +- **It is what the build itself checks.** Any other store can disagree with the bucket; this one cannot. +- **It is agent-independent.** `.crow/weekly-rebuild-missing.yaml` mounts no `volumes:`, unlike `.crow/build-all-versions.yaml:132-133`, so `/mnt/cache` is per-job and cannot carry progress anyway. +- **It costs one listing.** `cranlike`'s `s3` fork already does exactly this call against this bucket at ~24 000 objects, so the approach is proven at the required scale. + +It must read ETags rather than the slot index's `Built` field, which is how `local/packages-to-build.R:104-130` answers the same question. +Under this design the index is not rewritten until the dependent re-index pipeline runs (section 5), so mid-run it cannot reflect the current run's progress. + +Packages that genuinely fail to build re-publish their CRAN source, so they stay outstanding and would be retried on every restart. +That is already handled upstream: `bincraft::filter_packages_with_errors()` (`R/build_binaries.R:1018`, `:1143`) drops anything with `error_occurred = TRUE`, and `store_build_metadata = TRUE` is passed on every call. +No additional poison-pill filter is needed here. + +Only the flat `src/contrib` path is considered. +The rebuild call passes no `is_r_minor_sensitive`, so it defaults to `FALSE` and only ever targets the flat path; the resume filter matches that scope deliberately. + +### 4. Give each shard a wall-clock budget + +`local/rebuild-missing.R` takes a budget, defaulting to 20 hours, and breaks out of the loop once it is exceeded: + +```r +# exit cleanly rather than being killed, so the dependent re-index still runs +if (difftime(Sys.time(), started, units = "hours") > budget_hours) { + cat(sprintf("Budget of %sh reached after %d/%d packages; stopping cleanly\n", budget_hours, i, n)) + break +} +``` + +It exits 0 and reports how much of the slice it covered. +Every run then has a terminating condition, the re-index and purge always fire, and the remainder is picked up by the next run with no bookkeeping, because section 3 recomputes the outstanding set from scratch. + +### 5. Move the re-index and purge into `.crow/weekly-rebuild-reindex.yaml` + +Three shards per slot means three concurrent `upload_package_index()` calls on the same S3 prefix. +`cranlike::update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work. +The re-index steps (`.crow/weekly-rebuild-missing.yaml:171-176`) and the purge step (`:187-207`) therefore leave that file entirely. + +The new file carries: + +```yaml +depends_on: + - weekly-rebuild-missing +runs_on: [success, failure] +``` + +`runs_on: [success, failure]` validates as a workflow-level key under `crow lint`, so a failing shard no longer withholds the re-index. +The file uses the same 18-row matrix and the same `when:` gating as `weekly-rebuild-missing`, so it only re-indexes slots that actually ran. +Each row re-indexes the flat slot and every per-minor slot. +`scripts/purge_cdn_zone.sh` runs once on a single row, because all hostnames share pull zone `3857050` and 18 identical zone purges would be waste. + +## Failure behaviour + +| case | today | after | +| -------------------------- | ----------------------------------- | ----------------------------------------------------- | +| one package errors | `tryCatch` logs, loop continues | unchanged | +| a shard fails outright | purge runs, re-index does not | re-index and purge run via `runs_on` | +| a shard exceeds its budget | cannot happen, runs until killed | exits 0, re-index and purge run | +| a shard is killed | nothing runs | still nothing; trigger the re-index pipeline alone | +| a shard restarts | re-walks the list, HEAD per package | one listing, resumes at the first outstanding package | + +The known cost of `depends_on` being file-level rather than row-level: on the weekly cron no slot is re-indexed until the slowest of all 54 jobs finishes. +The 20-hour budget bounds that at roughly one day. + +## Out of scope + +- `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row for the platform and arch, which is precisely the source-fallback set. That is a separate change. +- Bunny Perma-Cache eviction. `scripts/purge_cdn_zone.sh` purges the regular edge cache only; see the note in `CLAUDE.md` and issue history. +- The audit that produces the rebuild list is unchanged. + +## Verification + +- `crow lint .crow/` passes for both pipeline files. +- `local/rebuild-missing.R` gets unit coverage in `local/tests/` for the two pure pieces: the interleaved slice (disjoint, covering, deterministic) and the outstanding-set filter (source-served ETag kept, binary ETag dropped, absent object kept). +- A single-slot manual run of `alpine-324-amd64` shard 1 confirms the listing shortcut against the live bucket, and that the reported outstanding count is close to the 8 917 measured above divided by three. +- Restarting that shard mid-run confirms it resumes rather than replaying, by comparing the outstanding count it reports on the second start. From aa4c95457f0ce7f79adea6705d54b098a10c50dd Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 13 Aug 2026 13:38:28 +0000 Subject: [PATCH 050/118] fix(rebuild): harden split workflow setup (#164) ## Motivation Weekly rebuild shards can all hit a transient CRAN DNS/index outage at once, and the dependent CDN purge always fails because it tries to clone over the checkout preserved from the re-index step. ## Changes - Retry `uvr add` resolution up to four times with bounded backoff. - Reuse the existing Crow workspace checkout in the CDN purge step. - Remove the purge step's unused Git package and repository token. ## Validation - `crow lint .crow/` - `shellcheck local/uvr-install.sh scripts/purge_cdn_zone.sh` - `git diff --check` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/164 --- .crow/weekly-rebuild-reindex.yaml | 6 ++---- local/uvr-install.sh | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 224596b..a67875d 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -173,14 +173,12 @@ steps: OTEL_R_METRICS_EXPORTER: none BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN # All hostnames on the zone share this id, so one purge covers # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. BUNNY_PULLZONE: '3857050' commands: - - apk add --no-cache -q bash curl git - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - apk add --no-cache -q bash curl + # Crow carries the checkout from the re-index step into this step. - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" # Runs on every row rather than on one designated slot: a cron fires only # its own slot's row, so gating on a named slot would leave every other diff --git a/local/uvr-install.sh b/local/uvr-install.sh index 1e25e47..3966e85 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -84,9 +84,20 @@ trap 'rm -rf "$project_dir"' EXIT cd "$project_dir" "$uvr_bin" init --here --r-version "$r_full" -# --no-install: resolve and lock only. The install happens in the sync below, -# which is the only command that honours --library. -"$uvr_bin" add --no-install "$@" +# --no-install resolves and locks only; retry because concurrent shards can +# expose short-lived DNS or CRAN-index failures and uvr rolls the manifest back +# cleanly after an unsuccessful resolution. +add_attempt=1 +while ! "$uvr_bin" add --no-install "$@"; do + if [ "$add_attempt" -ge 4 ]; then + echo "error: uvr add failed after ${add_attempt} attempts" >&2 + exit 1 + fi + add_delay=$((add_attempt * 10)) + echo "warning: uvr add attempt ${add_attempt} failed; retrying in ${add_delay}s" >&2 + sleep "$add_delay" + add_attempt=$((add_attempt + 1)) +done # TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs # `apt-get install` for every resolved system dependency without refreshing the From a1c1f5e78f2e22db0497fd749067661f063f35ff Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 13 Aug 2026 14:08:10 +0000 Subject: [PATCH 051/118] fix(cdn): align repository routing across pull zones (#165) ## Motivation `cran.rpkgs.com` and `cran.allianceswisspass.devxy.io` serve the same B2 repository through separate Bunny pull zones, but only the first zone was managed and purged after weekly reindexing. This allowed the Alliance endpoint to retain stale repository metadata and left locked `renv` restores unable to retrieve versions whose binary archive object was absent. ## Changes - Adopt the Alliance SwissPass pull zone `3265648` into OpenTofu and configure it with the shared B2 origin and middleware script. - Purge both Bunny pull zones after the weekly rebuild reindex. - Preserve the requested public hostname in middleware redirects. - Redirect missing archived binaries to the corresponding CRAN source package, checking whether the version is archived or still current. - Cover the existing archived-binary passthrough behavior in the edge routing matrix. ## Verification - `prek run -a` - `just edge-test` - `crow lint .crow/` - `tofu validate` - `bash -n scripts/purge_cdn_zone.sh` ## Deployment Run `tofu apply` to adopt pull zone `3265648`, publish the middleware release, and align both pull zones. After the apply, rerun the Alliance SwissPass CI restore that requested `cli 3.6.5` and `AzureStor 3.7.1`. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/165 --- .crow/weekly-rebuild-reindex.yaml | 8 ++--- cdn.tf | 56 +++++++++++++++++++++++++++++-- edge/rpkgs-router.test.ts | 7 ++++ edge/rpkgs-router.ts | 35 +++++++++++++++---- scripts/purge_cdn_zone.sh | 39 +++++++++++---------- 5 files changed, 115 insertions(+), 30 deletions(-) diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index a67875d..bb00332 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -173,13 +173,13 @@ steps: OTEL_R_METRICS_EXPORTER: none BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY - # All hostnames on the zone share this id, so one purge covers - # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. - BUNNY_PULLZONE: '3857050' + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + BUNNY_PULLZONES: '3857050 3265648' commands: - apk add --no-cache -q bash curl # Crow carries the checkout from the re-index step into this step. - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES # Runs on every row rather than on one designated slot: a cron fires only # its own slot's row, so gating on a named slot would leave every other # slot unpurged. A manual "all" run therefore purges the zone 18 times, diff --git a/cdn.tf b/cdn.tf index 5e8899d..01191a5 100644 --- a/cdn.tf +++ b/cdn.tf @@ -32,7 +32,7 @@ # cache_stale = ["offline", "updating"] # use_background_update = true - # block_ips = var.cdn_block_ips +# block_ips = var.cdn_block_ips # # 50 TB # limit_bandwidth = 50000000000000 @@ -82,7 +82,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { cache_expiration_time = 31919000 websockets_enabled = false - errorpage_whitelabel = true + errorpage_whitelabel = true origin { type = "OriginUrl" @@ -147,6 +147,58 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } +# Alliance SwissPass historically used a separate, manually configured pull +# zone. Adopt it so both public repositories use the same B2 origin, middleware +# release and cache behavior. +import { + to = bunnynet_pullzone.cran_allianceswisspass + id = "3265648" +} + +resource "bunnynet_pullzone" "cran_allianceswisspass" { + name = "cran-allianceswisspass" + + cache_errors = false + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + + origin { + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router.id + } + + routing { + filters = [ + "scripting", + ] + } + + s3_auth_enabled = true + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" + + cache_enabled = true + request_coalescing_enabled = true + block_post_requests = true + cache_vary_headers = ["User-Agent"] + + limit_requests = 5000 + limit_connections = 1000 + + safehop_enabled = true + add_canonical_header = true + cache_stale = ["offline", "updating"] + block_ips = var.cdn_block_ips + + # 50 TB + limit_bandwidth = 50000000000000 + + block_root_path = true +} + # resource "bunnynet_storage_zone" "devxy-r-binaries" { # name = "devxy-r-binaries-storage" # region = "DE" diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 553185d..9493f8a 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -117,6 +117,13 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.status, 200); }); + await t.step('serves an archived binary when it exists', async () => { + const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`; + const res = await probe(path, UA_R45_MUSL); + assertEquals(res.status, 200); + assertEquals(res.location, null); + }); + await t.step('does not redirect a path already under a minor', async () => { const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); assertEquals(res.location, null); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index 9cd410b..9278d4a 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -27,6 +27,7 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; const CRAN_ORIGIN = 'https://cran.r-project.org'; +const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']); /** Slots ("/", comma separated) whose per-minor index is a union. */ const UNION_SLOTS = new Set( @@ -47,6 +48,10 @@ const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; +/** A binary archive URL whose upstream source counterpart CRAN can serve. */ +const ARCHIVE_TARBALL_REGEX = + /^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/; + const MACOS_BIN_REGEX = /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; @@ -85,6 +90,10 @@ function redirectTo(location: string, status = 302): Response { }); } +function publicCdnOrigin(url: URL): string { + return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; +} + function extractRMinor(userAgent: string): string | null { for (const regex of R_MINOR_REGEXES) { const match = userAgent.match(regex); @@ -181,15 +190,14 @@ BunnySDK.net.http const url = new URL(ctx.request.url); const path = normalizePathname(url.pathname); const userAgent = ctx.request.headers.get('User-Agent') || ''; + const publicOrigin = publicCdnOrigin(url); // macOS clients are served from CRAN's own binary tree. const srcContrib = path.match(SRC_CONTRIB_REGEX); if (srcContrib && /darwin/.test(userAgent)) { const mac = parseMacUserAgent(userAgent); if (mac) { - return Promise.resolve( - redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`), - ); + return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`)); } } @@ -213,7 +221,7 @@ BunnySDK.net.http if (target === path) { return Promise.resolve(ctx.request); } - return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`)); + return Promise.resolve(redirectTo(`${publicOrigin}${target}`)); } // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. @@ -224,12 +232,27 @@ BunnySDK.net.http } const rest = srcContrib ? srcContrib[1] : ''; - return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`)); + return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); } return Promise.resolve(ctx.request); }) - .onOriginResponse((ctx) => { + .onOriginResponse(async (ctx) => { + const path = normalizePathname(new URL(ctx.request.url).pathname); + const archive = path.match(ARCHIVE_TARBALL_REGEX); + + // Binary archives can be incomplete when an older build never succeeded. + // Preserve renv/remotes version restores by falling back to CRAN's source + // package only for an absent archived tarball. A requested version can be + // either archived upstream or still current, so probe the archive first. + // Other 404s remain visible. + if (ctx.response.status === 404 && archive) { + const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`; + const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' }); + const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`; + return redirectTo(sourceUrl); + } + ctx.response.headers.append('X-Via', 'MyMiddleware'); return Promise.resolve(ctx.response); }); diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 391a814..4853245 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -18,35 +18,38 @@ # objects were replaced. The cost is a cold cache for everything else, which is # why this is not used by the daily update path. # -# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io, -# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them. +# The public hostnames currently use separate pull zones, so callers must pass +# every zone that serves the repository. # # Usage: -# purge_cdn_zone.sh +# purge_cdn_zone.sh [...] # set -euo pipefail if (($# < 2)); then - echo "usage: $0 " >&2 + echo "usage: $0 [...]" >&2 exit 2 fi api_key="$1" -zone_id="$2" +shift -echo "Purging BunnyCDN pull zone ${zone_id}" +for zone_id in "$@"; do + echo "Purging BunnyCDN pull zone ${zone_id}" -status=$( - curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \ - -H "AccessKey: ${api_key}" \ - -H "Content-Length: 0" \ - "https://api.bunny.net/pullzone/${zone_id}/purgeCache" -) + response_file="/tmp/purge_zone_response_${zone_id}.txt" + status=$( + curl -sS -o "${response_file}" -w '%{http_code}' -X POST \ + -H "AccessKey: ${api_key}" \ + -H "Content-Length: 0" \ + "https://api.bunny.net/pullzone/${zone_id}/purgeCache" + ) -if [[ "${status}" != "200" && "${status}" != "204" ]]; then - echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 - cat /tmp/purge_zone_response.txt >&2 - exit 1 -fi + if [[ "${status}" != "200" && "${status}" != "204" ]]; then + echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 + cat "${response_file}" >&2 + exit 1 + fi -echo "Purged pull zone ${zone_id} (HTTP ${status})" + echo "Purged pull zone ${zone_id} (HTTP ${status})" +done From 9bded261eecba451d4d544fbde68a68cd1180873 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 13 Aug 2026 14:13:04 +0000 Subject: [PATCH 052/118] fix(cdn): restore Alliance pull-zone hostname (#166) ## Motivation Applying #165 recreated the Alliance SwissPass pull zone without its custom hostname because the hostname association was not represented in OpenTofu. The recreated zone also received a new numeric ID, making the weekly purge configuration stale. ## Changes - Manage `cran.allianceswisspass.devxy.io` as a pull-zone hostname with TLS and forced HTTPS. - Resolve the Alliance pull-zone ID from its hostname before purging instead of persisting a replaceable numeric ID. - Install `jq` in the purge step for the Bunny API lookup. ## Verification - Targeted `prek` hooks pass. - `tofu validate` passes. - `crow lint .crow/` passes. - `just edge-test` passes all 14 routing steps. - `bash -n scripts/purge_cdn_zone.sh` passes. ## Deployment Run `tofu apply` to restore the Alliance hostname on the recreated pull zone. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/166 --- .crow/weekly-rebuild-reindex.yaml | 4 ++-- cdn.tf | 7 ++++++ scripts/purge_cdn_zone.sh | 40 +++++++++++++++++++++++++++---- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index bb00332..250b225 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -175,9 +175,9 @@ steps: from_secret: BUNNYNET_API_KEY # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 3265648' + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' commands: - - apk add --no-cache -q bash curl + - apk add --no-cache -q bash curl jq # Crow carries the checkout from the re-index step into this step. - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES # Runs on every row rather than on one designated slot: a cron fires only diff --git a/cdn.tf b/cdn.tf index 01191a5..96877b2 100644 --- a/cdn.tf +++ b/cdn.tf @@ -199,6 +199,13 @@ resource "bunnynet_pullzone" "cran_allianceswisspass" { block_root_path = true } +resource "bunnynet_pullzone_hostname" "cran_allianceswisspass" { + pullzone = bunnynet_pullzone.cran_allianceswisspass.id + name = "cran.allianceswisspass.devxy.io" + force_ssl = true + tls_enabled = true +} + # resource "bunnynet_storage_zone" "devxy-r-binaries" { # name = "devxy-r-binaries-storage" # region = "DE" diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 4853245..648dfc3 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -19,22 +19,54 @@ # why this is not used by the daily update path. # # The public hostnames currently use separate pull zones, so callers must pass -# every zone that serves the repository. +# every zone that serves the repository. A zone can be identified by its +# numeric ID or by one of its hostnames; hostname lookup avoids persisting IDs +# that change when a zone is recreated. # # Usage: -# purge_cdn_zone.sh [...] +# purge_cdn_zone.sh [...] # set -euo pipefail if (($# < 2)); then - echo "usage: $0 [...]" >&2 + echo "usage: $0 [...]" >&2 exit 2 fi api_key="$1" shift -for zone_id in "$@"; do +resolve_zone_id() { + local zone="$1" + local response_file + local zone_id + + if [[ "${zone}" =~ ^[0-9]+$ ]]; then + echo "${zone}" + return + fi + + response_file=$(mktemp) + curl -sS -o "${response_file}" \ + -H "AccessKey: ${api_key}" \ + "https://api.bunny.net/pullzone" + zone_id=$( + jq -r --arg hostname "${zone}" \ + '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \ + "${response_file}" + ) + rm -f "${response_file}" + + if [[ -z "${zone_id}" ]]; then + echo "Could not find BunnyCDN pull zone for hostname ${zone}" >&2 + exit 1 + fi + + echo "${zone_id}" +} + +for zone in "$@"; do + zone_id=$(resolve_zone_id "${zone}") echo "Purging BunnyCDN pull zone ${zone_id}" response_file="/tmp/purge_zone_response_${zone_id}.txt" From 706fd10d79206ed2784cd7f6bbac1a5e897e390b Mon Sep 17 00:00:00 2001 From: automation-bot Date: Fri, 14 Aug 2026 00:32:00 +0000 Subject: [PATCH 053/118] chore(deps): update terraform bunnynet to ~> 0.18 --- .terraform.lock.hcl | 72 ++++++++++++++++++++++----------------------- provider.tf | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index aca8ce1..a21cf43 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.17.0" - constraints = "~> 0.17" + version = "0.18.0" + constraints = "~> 0.18" hashes = [ - "h1:+qDt35lVSK7acw6a1xHuPYrqmZEcHSmtd+6n1TxNuYw=", - "h1:1dCu2l4DhPBjizVAH/WwAjT1Xbo52K4PMvHoD5zUhuU=", - "h1:Dvn46Auwuel4jqrqZXs2D7kdujNhs17LEmqhuY0k4/4=", - "h1:M5eDL3m2uSEr1XATJW0foHzKl8pFhCtgKuOM24bJRwU=", - "h1:PddaC7nM/gY4x9i3xy6TxOs9MAu2/6g58Xs/gv4DRV8=", - "h1:QVIKiZluI+NQAKu8NpFBl3Nvyx+d81vW9btEUdIQREc=", - "h1:S6TnzXHsRoGYvC1vJBkDiVEc0spceksY4n6x5WN5iYw=", - "h1:VcxZDWqCWMSjcUsC1K4sB6uYEoeoou+BC0ePoJXmf3A=", - "h1:W0y/agBVqls1cJlFGFYMu2VnqoPXFzxVHPIYe3OqfYQ=", - "h1:XmNd5fP9a0O77ve5BMQP2vARExgIa7rYl6KvyUYXPSs=", - "h1:e0EFKrWSQwaa/kGhnha4DXk4T68Av8QxP84mRSdWC9M=", - "h1:eM+/lUiU0pNSgQKoqKPgE3xJrJ0MHIpKG+yhaGB/P0M=", - "h1:fPWWA4T0/y7GX+tCGN23l1jODhZ3uCdR/MKgZDXYpAE=", - "h1:g+r2GVi4gVC4DuQg3PL70gW9BDskgWUzCBIMXTUq63A=", - "h1:gaZ8eALDtVHqykVDHav8004gHiMGaYR/3KwET0FUgao=", - "h1:kbqW25eaiv4N/N/z+sxLdJZ15yh5cgnRD/q6RclPMLc=", - "h1:rGjxue3mXRyQQqpywTXC4zK//JAtf0Cz7RP+uPMMJjw=", - "zh:05943fef14c2028f4722bf078aa1889229e94302f7678cc6f63adb669d8ea612", - "zh:26a163930a92a7408f7bbd0130064b84df8a232b500d8c6c3989952986308539", - "zh:41305feaaade55391447521ec309f3c038b631ca542907ad95132fab71a7e116", - "zh:606919a930f0299948504adbdcd0f239a8af5c418f85741c48f8add370a3d038", - "zh:66963d5b445639511939fc508513fd31da3ee1d4ee1a565ee396c9532897a349", - "zh:6c981ec0c8545556395c43e2511861ab65ee9ecf2a960480e7889c3af0d23af3", - "zh:7334a1bdb726ce1f1bf0a3155f30f84f65206980c229c832ff5f0b0718c44e0b", - "zh:75f6c86bf74511e605423332d113711c76c8028361a32282fb3359d6c7ecae9e", - "zh:7aebb1a01cfe8be54903853202ae06eba14ad99c37d230ed93ce7d6633e05e9b", + "h1:+6VXZmSSeIchab8WY+UFZxaXu6poxZbiJenYwog+Z2M=", + "h1:Bz94gADR83NmrsZC50LheryVcS9erG6tF052hNPqz0U=", + "h1:FGkcn6ieyNPr56Riv75IsCa6okZ9ZHqqFd08Y4i5cPI=", + "h1:GqPjTKaOlhhNIxpPHWJpWdGkEtvUDtuOkjYbVIInfgU=", + "h1:NMDWYRisSFwepjfYsf4MrcyP7Ihjlty937l1BW1Cztg=", + "h1:OusNMAZdIIbxWwHXwH8tYzdbXZPRS8dW7cncSYcHfNI=", + "h1:OzgasS4oZCZjYAOrqGy7RpPNbTPX5wGaaw7TDnasSJM=", + "h1:RyK3DC7cM4T2I83OvnO+UXU+eCXLG5vDlYZ/bWKHq7g=", + "h1:U/JKg4BNWii2mK3bNsOWYMBTpkXqGsWb1ypSJvSKl9Y=", + "h1:UjSUm1AU2wZ8eyC3LDRwWatW9cKD/I0rk7MOeUDDkWc=", + "h1:a888ExTeqWKxaB2GBsDIRSm8jOOMLkPFcABrJxxA8qc=", + "h1:aShYLfSapfo1Ozy33zL5WfmYt5UF2m5EziespPOGNAk=", + "h1:bHdpp2ecmvDoezqg8TirE32SnuDb108/NZ4xBGvJy98=", + "h1:lLYZN5cetXgLPJjo+eR+kwodOYGsQetzV6bg5Ki58tk=", + "h1:szi71DaqI4yB6fI55hBbtVZg1GDWJVkO9+WF/1sZ+IE=", + "h1:vK4jZYZZhD6M4cRmX+171mxoJgb2FSNiBNAUhtm/TNs=", + "h1:yNnSUskn43648XU/YS0e9NUdoHImo7pFMIlBzujBoxY=", + "zh:0c3adf039df2fead1e36b8e7887d965223f03b6b5cdd921ec0c98beaa04fdec0", + "zh:12a3db29733e6619e216fac9aa774cbf67da2105f48c5c4dee8df046d69656fa", + "zh:1330b83c949165434e4d0db5e097a1741376cdb66c06c37b2a97ecea8e06b0f0", + "zh:186113259ab0f80ab9079f0a8a810e93ee097f6740100a0a444f805e8772c12e", + "zh:242fe10185da8a700b0e9a6a52e3d1f6592e8e189aaf5e9a79cf515ad5c7ff25", + "zh:28b005b0aa9485c492326fbb4287d3cb465f25a8d572b8c740e041e040d80aa2", + "zh:3b2cd0daa767dfff67dd39c98d1c82b25e70eb81df77c0d6dd762a0dedec16e1", + "zh:3f14b244740f6c6420d298846708e9f3853c8eda685d3a4d71fb0ffec021ad9c", + "zh:469bfd08deefd90e89c930e5e45bfd06a58a1eadb85dfbee9fc48ae296e8a8c9", + "zh:634d96b84b2e77d25baa074c357e831252c8b8114926ef72e1afb68a4bc9eb65", + "zh:6e837b9485e56f84b5e2bb396fa549b628b8e075257fa01bf8c796ce91d20e58", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9041d0e20c9ceea532de6eebf5cb3a27dad0bb49d3f5b5154be2a08d68fbbf1f", - "zh:a6bbf65431a02be4df0ebb1cbe01185ad357ff6e33c01bd0558f59bed90c8f36", - "zh:c6d075a31096f080c388dfe46036f451c0cc114c3311a4f46ab8dbe1938a202f", - "zh:dd8703f7b55b8bc8e10f8718bea889781100b18e932b04898995b63178c3d36e", - "zh:dd92a5cd4e133a4000e7e5bc8cce876ae0ed803543cedd2f3d590661ba244d04", - "zh:e024fdf121bebc48c1e6debea344c6d4f174117f3ae605fca6e13b9705d92d22", - "zh:ee0e80c31b438e35fa1608f6a2f5824d2806db1e5e8b9f7a90986585c7bcb895", - "zh:fc2d4b705411b48f8c045981f9368a3ea2f74969dd6302008c31ff0bedd51f0a", + "zh:89e02979311e6727a50586b04cb343e9bbec74a6f4d1dba9352971e4334d64c9", + "zh:a6796143fe61ae52d236b7ea96bddbc6317851a0e2117eb2f55cd73c803b4b6f", + "zh:ad2542160e7b57ee8a016cddbf1d32663baf74fe30fcf24df64da75cb13128b0", + "zh:bfee81c153e8b50121fd58eedeeb84bfa64a3d17f23900e35143c4a86474cbd2", + "zh:eeba652697908712cd1ce2c7d2925f6e6bb2d18ea5d3e6ca185e88eeb80f3c80", + "zh:f0b5b8fd6942647c358440591d56a5cb82fda92f7866898654a66ec901174f1f", ] } diff --git a/provider.tf b/provider.tf index badbbc1..d4f2564 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.17" + version = "~> 0.18" } } } From 132d1d2d3cd37611db98733a57d8345a218e152b Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 14 Aug 2026 06:44:10 +0000 Subject: [PATCH 054/118] docs(ci): clarify parallel manual matrix runs --- .crow/process-updates.yaml | 6 +++--- .crow/weekly-audit-missing.yaml | 6 +++--- .crow/weekly-rebuild-missing.yaml | 8 ++++---- .crow/weekly-rebuild-reindex.yaml | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 7600876..021d6cb 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -3,15 +3,15 @@ # Routing is preserved 1:1: # - cron: each existing `process-cran-updates--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `process_cran_updates` dropdown -# ("all" = every os/arch). +# - manual: pick a target from the `process_cran_updates` dropdown; +# "all" fans out every os/arch as parallel matrix workflows. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. process_cran_updates: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 3a9d98f..428aa04 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -3,15 +3,15 @@ # Routing is preserved 1:1: # - cron: each existing `weekly-audit-missing--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `weekly_audit_missing` dropdown -# ("all" = every os/arch). +# - manual: pick a target from the `weekly_audit_missing` dropdown; +# "all" fans out every os/arch as parallel matrix workflows. # Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64). variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. weekly_audit_missing: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 5afb3c3..9639792 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -4,9 +4,9 @@ # - cron: each existing `weekly-rebuild-missing--` cron fires only # its matching matrix rows (via the per-row `cron:` name filter), # which is now all three shards of that slot. -# - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the -# previous bare manual trigger that ran every os/arch); pick a -# single - to run just one. +# - manual: pick a target from the `weekly_rebuild_missing` dropdown; +# "all" fans out every os/arch and shard as parallel matrix +# workflows, while a single - runs its three shards. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). # # The shard picks up its own slice and re-derives what is still outstanding @@ -21,7 +21,7 @@ variables: # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 250b225..2e0f2a0 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -16,7 +16,7 @@ variables: # exactly the slots it rebuilt. A manual pipeline creation instantiates every # file in .crow/, so the default must match no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all From b75fd2f1c43cf0355e3e2eb9e51d73f16a1321eb Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 14 Aug 2026 06:56:41 +0000 Subject: [PATCH 055/118] fix(ci): split oversized weekly rebuild matrix --- .crow/weekly-rebuild-missing-ubuntu-2604.yaml | 140 ++++++++++++++++++ .crow/weekly-rebuild-missing.yaml | 41 +---- .crow/weekly-rebuild-reindex-ubuntu-2604.yaml | 110 ++++++++++++++ .crow/weekly-rebuild-reindex.yaml | 9 -- 4 files changed, 253 insertions(+), 47 deletions(-) create mode 100644 .crow/weekly-rebuild-missing-ubuntu-2604.yaml create mode 100644 .crow/weekly-rebuild-reindex-ubuntu-2604.yaml diff --git a/.crow/weekly-rebuild-missing-ubuntu-2604.yaml b/.crow/weekly-rebuild-missing-ubuntu-2604.yaml new file mode 100644 index 0000000..0082e0e --- /dev/null +++ b/.crow/weekly-rebuild-missing-ubuntu-2604.yaml @@ -0,0 +1,140 @@ +# ubuntu-2604 rows split from weekly-rebuild-missing.yaml. +# +# The complete rebuild matrix has 54 permutations, while Crow accepts at most +# 50 per workflow. Keeping these six rows in a companion workflow lets a manual +# "all" run fan out every slot and shard without exceeding that compiler limit. + +variables: + # This is the same gate as the main rebuild workflow. The safe default must + # match no row because every file in .crow/ is evaluated on a manual run. + weekly_rebuild_missing: + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-rebuild-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + +steps: + - name: 'Rebuild missing binaries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + GIT_USER: pat-s + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ${OS} + ARCH: ${ARCH} + NCPUS: 2 + SPLIT_INTO: ${SPLIT_INTO} + SPLIT_INDEX: ${SPLIT_INDEX} + # Wall clock after which the shard stops cleanly instead of having to be + # killed. A kill matches neither `success` nor `failure`, so it would skip + # the dependent re-index and leave rebuilt binaries behind a stale edge. + REBUILD_BUDGET_HOURS: 20 + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 + - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 3000m + limits: + memory: 18Gi + cpu: 3000m diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 9639792..aeb2f50 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -1,5 +1,7 @@ -# Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). +# Main weekly-rebuild-missing workflow (all slots except ubuntu-2604). # Three matrix rows per OS/arch, one per shard of that slot's rebuild list. +# ubuntu-2604 lives in weekly-rebuild-missing-ubuntu-2604.yaml so a manual +# "all" run stays below Crow's 50-permutation matrix limit. # Routing is preserved 1:1: # - cron: each existing `weekly-rebuild-missing--` cron fires only # its matching matrix rows (via the per-row `cron:` name filter), @@ -346,43 +348,6 @@ matrix: IMG: ubuntu:noble SPLIT_INTO: 3 SPLIT_INDEX: 3 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-${IMG} diff --git a/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml b/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml new file mode 100644 index 0000000..3ab086a --- /dev/null +++ b/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml @@ -0,0 +1,110 @@ +# Re-index ubuntu-2604 after its split rebuild workflow finishes. + +variables: + # Mirrors the shared rebuild gate. The default must match no matrix row so + # unrelated manual runs do not rewrite package indexes. + weekly_rebuild_missing: + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-rebuild-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' + +depends_on: + - weekly-rebuild-missing-ubuntu-2604 + +runs_on: [success, failure] + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + PLATFORM: ${OS} + ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + commands: + - apk add --no-cache -q bash curl jq + # Crow carries the checkout from the re-index step into this step. + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES + # Run even when the re-index above failed: the objects were still replaced, + # and a stale edge is exactly what keeps them hidden. + when: + - status: [success, failure] diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 2e0f2a0..f38f6d5 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -122,15 +122,6 @@ matrix: ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - steps: - name: 'Re-index the slot' image: reg.devxy.io/rpkgs/build-env-${IMG} From 50495f5c3bb6e3e97d9d42d639c5f9828b6207b6 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 14 Aug 2026 07:01:56 +0000 Subject: [PATCH 056/118] Revert "fix(ci): split oversized weekly rebuild matrix" This reverts commit b75fd2f1c43cf0355e3e2eb9e51d73f16a1321eb. --- .crow/weekly-rebuild-missing-ubuntu-2604.yaml | 140 ------------------ .crow/weekly-rebuild-missing.yaml | 41 ++++- .crow/weekly-rebuild-reindex-ubuntu-2604.yaml | 110 -------------- .crow/weekly-rebuild-reindex.yaml | 9 ++ 4 files changed, 47 insertions(+), 253 deletions(-) delete mode 100644 .crow/weekly-rebuild-missing-ubuntu-2604.yaml delete mode 100644 .crow/weekly-rebuild-reindex-ubuntu-2604.yaml diff --git a/.crow/weekly-rebuild-missing-ubuntu-2604.yaml b/.crow/weekly-rebuild-missing-ubuntu-2604.yaml deleted file mode 100644 index 0082e0e..0000000 --- a/.crow/weekly-rebuild-missing-ubuntu-2604.yaml +++ /dev/null @@ -1,140 +0,0 @@ -# ubuntu-2604 rows split from weekly-rebuild-missing.yaml. -# -# The complete rebuild matrix has 54 permutations, while Crow accepts at most -# 50 per workflow. Keeping these six rows in a companion workflow lets a manual -# "all" run fan out every slot and shard without exceeding that compiler limit. - -variables: - # This is the same gate as the main rebuild workflow. The safe default must - # match no row because every file in .crow/ is evaluated on a manual run. - weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." - options: - - none - - all - - alpine-322-amd64 - - alpine-322-arm64 - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - -when: - - event: cron - cron: weekly-rebuild-missing-${OS}-${ARCH} - - event: manual - evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - UVR_CACHE_DIR: /mnt/cache/uvr/cache - UVR_PACKAGES_DIR: /mnt/cache/uvr/packages - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: ${R_VERSION} - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ${OS} - ARCH: ${ARCH} - NCPUS: 2 - SPLIT_INTO: ${SPLIT_INTO} - SPLIT_INDEX: ${SPLIT_INDEX} - # Wall clock after which the shard stops cleanly instead of having to be - # killed. A kill matches neither `success` nor `failure`, so it would skip - # the dependent re-index and leave rebuilt binaries behind a stale edge. - REBUILD_BUDGET_HOURS: 20 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 - backend_options: - docker: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index aeb2f50..9639792 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -1,7 +1,5 @@ -# Main weekly-rebuild-missing workflow (all slots except ubuntu-2604). +# Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). # Three matrix rows per OS/arch, one per shard of that slot's rebuild list. -# ubuntu-2604 lives in weekly-rebuild-missing-ubuntu-2604.yaml so a manual -# "all" run stays below Crow's 50-permutation matrix limit. # Routing is preserved 1:1: # - cron: each existing `weekly-rebuild-missing--` cron fires only # its matching matrix rows (via the per-row `cron:` name filter), @@ -348,6 +346,43 @@ matrix: IMG: ubuntu:noble SPLIT_INTO: 3 SPLIT_INDEX: 3 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-${IMG} diff --git a/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml b/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml deleted file mode 100644 index 3ab086a..0000000 --- a/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Re-index ubuntu-2604 after its split rebuild workflow finishes. - -variables: - # Mirrors the shared rebuild gate. The default must match no matrix row so - # unrelated manual runs do not rewrite package indexes. - weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." - options: - - none - - all - - alpine-322-amd64 - - alpine-322-arm64 - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - -when: - - event: cron - cron: weekly-rebuild-missing-${OS}-${ARCH} - - event: manual - evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' - -depends_on: - - weekly-rebuild-missing-ubuntu-2604 - -runs_on: [success, failure] - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - -steps: - - name: 'Re-index the slot' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GIT_USER: pat-s - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: ${R_VERSION} - PLATFORM: ${OS} - ARCH: ${ARCH} - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - # The codename is detected from the image's /etc/os-release. - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate - # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' - commands: - - apk add --no-cache -q bash curl jq - # Crow carries the checkout from the re-index step into this step. - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES - # Run even when the re-index above failed: the objects were still replaced, - # and a stale edge is exactly what keeps them hidden. - when: - - status: [success, failure] diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index f38f6d5..2e0f2a0 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -122,6 +122,15 @@ matrix: ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + steps: - name: 'Re-index the slot' image: reg.devxy.io/rpkgs/build-env-${IMG} From c7b4dca6e2209c1b56932f37843ef586497fe4e9 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Mon, 17 Aug 2026 00:31:56 +0000 Subject: [PATCH 057/118] chore(deps): lock file maintenance --- .terraform.lock.hcl | 62 ++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index a21cf43..ff935fa 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,38 +2,38 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/http" { - version = "3.6.0" + version = "3.6.1" hashes = [ - "h1:0n4RBz9zNw6TTddh5+x7E8L2+qzPXNwKhK4uoZ/DUwE=", - "h1:22Ob7lpzMBSqdrCvoFN5EgmhGPHPBovV/9qo0c/Cd+A=", - "h1:2IRBvmWOYrq/ooaYYn2i86jZb7iIUvlg0KlmOMfDHoQ=", - "h1:5mucXikk4OcW3un3u94QnMx4AB4Wfih+sXeMd5QxSNk=", - "h1:5oU7Zm+2gAVGmxqtJ9E8uTudUkYy/DEn/y3IWphdv4k=", - "h1:5w0R4b1/VSzpqQF1tXXPr/qmaQLPVRXamOmPKWFcTk4=", - "h1:AEVeJr8xGmwad+JUUQ833C3x5d4W+W2szF5DfwxYppw=", - "h1:CPHJ+0zQbS/cX1m55Y90jIOgf1jV3ocUUnqsXAh+9Eg=", - "h1:JPewnGDOJudNer5+ghqwXoaJkfot3QRq9uiEYvo+JHU=", - "h1:QzbluV2vQLxsJYxjpziQCmPndIoJ/UGS4/UHH/GpwUM=", - "h1:TjUNbUdqweRBq/ycQ4ixpNkx5qaYwpXEOn9QCpqNZP8=", - "h1:XNbcODP60ajj21N/OO7af8bBg1ltIsYkq9egn7BYbiY=", - "h1:tgrbgmX7WYQz9G9ncgu7TkpVB+RlLjJA/Rvp9KPlZH8=", - "h1:vLxthX/ZWsOZ+aHKbAMqmNKqD0K5f4nJ8ppy0Ioyup0=", - "h1:wZOdGBAZkY8OKEPjKz82j1HloAKOmmvtjWyTxM+I110=", - "zh:0f719fa5426bc883e9fa6abf7f6498e48025edafbc29015e2f5c028f1cca3b9d", - "zh:1b4d7dafefd6c61764b2f9ed6943ceb9a200dee3590d18747e3a5f6b20ce85e0", - "zh:1d23a712984866d29f7b07028a4e99c783c71f1a5dddf08bc3d4e7da9d91a1fa", - "zh:257d23d58c3bb024b6bc8eb88736eaf912e934ad47c639d0c3c742bddda849a1", - "zh:479860e1a5468f5e04013b9364c9496d7ed0804bf9a1acd8e07558d57609993d", - "zh:4cb5e681bf599b411b27c4a2c4066a5fb2ed79aaa3a1a3cb5a30002fec062ce9", - "zh:4fb35c3f643dae9f3670d719397a415f815a0b95f8ed7bd8a72f27a94ba78092", - "zh:59ba40825ab38db5b4a0989a2db0df35cc15d8984f898176011ba352f27d77b7", - "zh:61fc1252eb88088638f4c69ea4e2171cde2e5089fa632ac1e943b13787348f73", - "zh:7c5d6dd5f7cbc460e95d368be35c29b4e0402069b8912dbd5d1cd7fa9acef216", - "zh:7f76d756240d4284642f359ad470226e5378670239aadc366ef54d9d914d4d2e", - "zh:8133ad0814098177e0d067c816ccf1bf48bbadacd18f6f2c808c90447505723b", - "zh:c93be06269bb728f1968f8c50506de56c887017ac1d6e4be1f925651d8437eb6", - "zh:ef47b78a10a82e6cf53344a6a85a94041c28286c10a70541c564d762f1cfede0", - "zh:f5796a53a74999135bd9087aff50fddda59129d09b2f9b1902ff8c0c1e047e48", + "h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=", + "h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=", + "h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=", + "h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=", + "h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=", + "h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=", + "h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=", + "h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=", + "h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=", + "h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=", + "h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=", + "h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=", + "h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=", + "h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=", + "h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=", + "zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9", + "zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6", + "zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab", + "zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b", + "zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c", + "zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4", + "zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502", + "zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69", + "zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6", + "zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279", + "zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6", + "zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7", + "zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df", + "zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621", + "zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424", ] } From d1da0c6cb46447d9b295389e4cda211d014d10fb Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sat, 22 Aug 2026 00:32:09 +0000 Subject: [PATCH 058/118] chore(deps): update terraform bunnynet to v0.18.1 --- .terraform.lock.hcl | 70 ++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index ff935fa..ef9e47d 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.18.0" + version = "0.18.1" constraints = "~> 0.18" hashes = [ - "h1:+6VXZmSSeIchab8WY+UFZxaXu6poxZbiJenYwog+Z2M=", - "h1:Bz94gADR83NmrsZC50LheryVcS9erG6tF052hNPqz0U=", - "h1:FGkcn6ieyNPr56Riv75IsCa6okZ9ZHqqFd08Y4i5cPI=", - "h1:GqPjTKaOlhhNIxpPHWJpWdGkEtvUDtuOkjYbVIInfgU=", - "h1:NMDWYRisSFwepjfYsf4MrcyP7Ihjlty937l1BW1Cztg=", - "h1:OusNMAZdIIbxWwHXwH8tYzdbXZPRS8dW7cncSYcHfNI=", - "h1:OzgasS4oZCZjYAOrqGy7RpPNbTPX5wGaaw7TDnasSJM=", - "h1:RyK3DC7cM4T2I83OvnO+UXU+eCXLG5vDlYZ/bWKHq7g=", - "h1:U/JKg4BNWii2mK3bNsOWYMBTpkXqGsWb1ypSJvSKl9Y=", - "h1:UjSUm1AU2wZ8eyC3LDRwWatW9cKD/I0rk7MOeUDDkWc=", - "h1:a888ExTeqWKxaB2GBsDIRSm8jOOMLkPFcABrJxxA8qc=", - "h1:aShYLfSapfo1Ozy33zL5WfmYt5UF2m5EziespPOGNAk=", - "h1:bHdpp2ecmvDoezqg8TirE32SnuDb108/NZ4xBGvJy98=", - "h1:lLYZN5cetXgLPJjo+eR+kwodOYGsQetzV6bg5Ki58tk=", - "h1:szi71DaqI4yB6fI55hBbtVZg1GDWJVkO9+WF/1sZ+IE=", - "h1:vK4jZYZZhD6M4cRmX+171mxoJgb2FSNiBNAUhtm/TNs=", - "h1:yNnSUskn43648XU/YS0e9NUdoHImo7pFMIlBzujBoxY=", - "zh:0c3adf039df2fead1e36b8e7887d965223f03b6b5cdd921ec0c98beaa04fdec0", - "zh:12a3db29733e6619e216fac9aa774cbf67da2105f48c5c4dee8df046d69656fa", - "zh:1330b83c949165434e4d0db5e097a1741376cdb66c06c37b2a97ecea8e06b0f0", - "zh:186113259ab0f80ab9079f0a8a810e93ee097f6740100a0a444f805e8772c12e", - "zh:242fe10185da8a700b0e9a6a52e3d1f6592e8e189aaf5e9a79cf515ad5c7ff25", - "zh:28b005b0aa9485c492326fbb4287d3cb465f25a8d572b8c740e041e040d80aa2", - "zh:3b2cd0daa767dfff67dd39c98d1c82b25e70eb81df77c0d6dd762a0dedec16e1", - "zh:3f14b244740f6c6420d298846708e9f3853c8eda685d3a4d71fb0ffec021ad9c", - "zh:469bfd08deefd90e89c930e5e45bfd06a58a1eadb85dfbee9fc48ae296e8a8c9", - "zh:634d96b84b2e77d25baa074c357e831252c8b8114926ef72e1afb68a4bc9eb65", - "zh:6e837b9485e56f84b5e2bb396fa549b628b8e075257fa01bf8c796ce91d20e58", + "h1:1nbjHMc5QBwiAfkriGGuO+wLkrgwKjbeAK70sNQRDa4=", + "h1:2AkMQZEckeGl8efzFzpGUqIR5o4rhOt4GSmOOMBX8wg=", + "h1:4D42uIgbm5jI0TQWRfVXJZoG6WOyO3Mi0ZgC8KZjPmc=", + "h1:79iaWeho9vn6p5oFHx7cXCzrX9k0R2yS9iF7rNchX7E=", + "h1:7GKjnlsRnA88qGIZ+V4HseEWB1YmuPXkyVen4DOxRBM=", + "h1:9ZpZHafhWNCDM91hE+GK6ikvy9THlU4/opQoC78B6TU=", + "h1:J4eKuqfDRI+e1PRGkt6MJ6UT+ym3pFxYKN+jIm105Es=", + "h1:LvrSDB8WWxh6NcR7X97oxJ9FrB/UVyWau9V+rdxBkxU=", + "h1:NO36kn/RhmKhWVcM0qRUcbZveSv2idT6LUwOREBqBc4=", + "h1:UPrn5yfuJwaEggnIbKqd9eyNCzO9DRJhA0279J1LYLM=", + "h1:b+tjNcFfxqwWGDgumzm4QImqJLyemGanHbZ+F2qw6vU=", + "h1:gppmt6Jbng9QH8ulpSujYeCHSlC5kWEkYoiYzgFISNQ=", + "h1:kZkf+9F8Be+Ztt10sLUWIxnwgfBCHpxi3V66F3+HV5s=", + "h1:mZvRXynxrx0/Q0txogt/5fTVhKmRWX3rWoqLiwLdamw=", + "h1:ogkHPOIbdDgdmfoa0LaPIyX8QTJvWik3andNAJoR6pk=", + "h1:pCkHlvcWgM5FkwEsrxzT9L7S7ZDNfyWagWM0KMXYFBE=", + "h1:rbKmEiaOKwrbrzEf9iihpwFCwvlVVkyxZUfFeVuX180=", + "zh:198f5aab9e8bbbb6fa96e41b7d33a997e72666dfc7369f08849ff12a0f91f7d4", + "zh:1f152ab9c51353422a79d4c4ee965112972b0d4b3246c62a8e6848422f4cac26", + "zh:4776e4fc6f38b1eb4a64c866e617cebdb0344af51c4685aa3e47356a5502707f", + "zh:66828334af0bdecbde3c5913b9c48371ea119569631f43094e28e03ab1b5fa4a", + "zh:800a94528ed366242fb83ba8d22ca1407bb02826ac8805a83a0ab04ab5efb582", + "zh:862f5e7db0f81d5e57292f0cd44b4f530eaa59510756991037d1ed6fbff91e05", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:89e02979311e6727a50586b04cb343e9bbec74a6f4d1dba9352971e4334d64c9", - "zh:a6796143fe61ae52d236b7ea96bddbc6317851a0e2117eb2f55cd73c803b4b6f", - "zh:ad2542160e7b57ee8a016cddbf1d32663baf74fe30fcf24df64da75cb13128b0", - "zh:bfee81c153e8b50121fd58eedeeb84bfa64a3d17f23900e35143c4a86474cbd2", - "zh:eeba652697908712cd1ce2c7d2925f6e6bb2d18ea5d3e6ca185e88eeb80f3c80", - "zh:f0b5b8fd6942647c358440591d56a5cb82fda92f7866898654a66ec901174f1f", + "zh:91f8b0f92b7dd1e131e391c170fcb329c795747553d5cc71296b8803496bc33c", + "zh:95446852691ebbbfcd4998f8941d29577d61b15ac320bc4590ae48f4542aee39", + "zh:9ccb3382c4fc20735c3ee016240afa82d6cf745b8ec458a9e5159d1a03e4cd20", + "zh:a11991312edc9fe2567ecedadb2e87de8b1a41e97a0d117d1a1c58eaa68ae645", + "zh:ab729171af18063c34f5a7d33b598b870e082f2e7ba8ee60ca77dc4dede84432", + "zh:c2b58950da56179b93a7d590d0cf3a488070b973826d788401a94914eaeae072", + "zh:d1b6a91581f1580fad12f11b9593461f566153b6760299ba592746716d595583", + "zh:dad0ce1ac0fc934ab18fe4a744b768d8f9969099488b25d6c18c782b40478bfe", + "zh:e2cb8dfebfa0a358d0b52e15b04cfd82407a4c578c39b6ac62ac919b23ef97a6", + "zh:fc648d9464bcb6b360eb4885791675c2437f0d1c0e1b1b8f34fbb6d93a880ea9", + "zh:fd8250944e3794b9440cdaf4ac06f7a1672744e4bcb2e8a33b94c2ddf32ab988", ] } From f4ab6f9dc55d2e33822abf5d48e6517a33bd46e9 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Wed, 26 Aug 2026 00:32:02 +0000 Subject: [PATCH 059/118] chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker to v3.11.2 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82461cb..2bf38b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.11.1 + rev: v3.11.2 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ From d2333c6cbe599fc5e43c7534d55bb2120e104245 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Thu, 27 Aug 2026 00:33:08 +0000 Subject: [PATCH 060/118] chore(deps): update terraform bunnynet to v0.18.2 --- .terraform.lock.hcl | 70 ++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index ef9e47d..9591b83 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.18.1" + version = "0.18.2" constraints = "~> 0.18" hashes = [ - "h1:1nbjHMc5QBwiAfkriGGuO+wLkrgwKjbeAK70sNQRDa4=", - "h1:2AkMQZEckeGl8efzFzpGUqIR5o4rhOt4GSmOOMBX8wg=", - "h1:4D42uIgbm5jI0TQWRfVXJZoG6WOyO3Mi0ZgC8KZjPmc=", - "h1:79iaWeho9vn6p5oFHx7cXCzrX9k0R2yS9iF7rNchX7E=", - "h1:7GKjnlsRnA88qGIZ+V4HseEWB1YmuPXkyVen4DOxRBM=", - "h1:9ZpZHafhWNCDM91hE+GK6ikvy9THlU4/opQoC78B6TU=", - "h1:J4eKuqfDRI+e1PRGkt6MJ6UT+ym3pFxYKN+jIm105Es=", - "h1:LvrSDB8WWxh6NcR7X97oxJ9FrB/UVyWau9V+rdxBkxU=", - "h1:NO36kn/RhmKhWVcM0qRUcbZveSv2idT6LUwOREBqBc4=", - "h1:UPrn5yfuJwaEggnIbKqd9eyNCzO9DRJhA0279J1LYLM=", - "h1:b+tjNcFfxqwWGDgumzm4QImqJLyemGanHbZ+F2qw6vU=", - "h1:gppmt6Jbng9QH8ulpSujYeCHSlC5kWEkYoiYzgFISNQ=", - "h1:kZkf+9F8Be+Ztt10sLUWIxnwgfBCHpxi3V66F3+HV5s=", - "h1:mZvRXynxrx0/Q0txogt/5fTVhKmRWX3rWoqLiwLdamw=", - "h1:ogkHPOIbdDgdmfoa0LaPIyX8QTJvWik3andNAJoR6pk=", - "h1:pCkHlvcWgM5FkwEsrxzT9L7S7ZDNfyWagWM0KMXYFBE=", - "h1:rbKmEiaOKwrbrzEf9iihpwFCwvlVVkyxZUfFeVuX180=", - "zh:198f5aab9e8bbbb6fa96e41b7d33a997e72666dfc7369f08849ff12a0f91f7d4", - "zh:1f152ab9c51353422a79d4c4ee965112972b0d4b3246c62a8e6848422f4cac26", - "zh:4776e4fc6f38b1eb4a64c866e617cebdb0344af51c4685aa3e47356a5502707f", - "zh:66828334af0bdecbde3c5913b9c48371ea119569631f43094e28e03ab1b5fa4a", - "zh:800a94528ed366242fb83ba8d22ca1407bb02826ac8805a83a0ab04ab5efb582", - "zh:862f5e7db0f81d5e57292f0cd44b4f530eaa59510756991037d1ed6fbff91e05", + "h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=", + "h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=", + "h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=", + "h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=", + "h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=", + "h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=", + "h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=", + "h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=", + "h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=", + "h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=", + "h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=", + "h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=", + "h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=", + "h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=", + "h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=", + "h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=", + "h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=", + "zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c", + "zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178", + "zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd", + "zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef", + "zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738", + "zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc", + "zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:91f8b0f92b7dd1e131e391c170fcb329c795747553d5cc71296b8803496bc33c", - "zh:95446852691ebbbfcd4998f8941d29577d61b15ac320bc4590ae48f4542aee39", - "zh:9ccb3382c4fc20735c3ee016240afa82d6cf745b8ec458a9e5159d1a03e4cd20", - "zh:a11991312edc9fe2567ecedadb2e87de8b1a41e97a0d117d1a1c58eaa68ae645", - "zh:ab729171af18063c34f5a7d33b598b870e082f2e7ba8ee60ca77dc4dede84432", - "zh:c2b58950da56179b93a7d590d0cf3a488070b973826d788401a94914eaeae072", - "zh:d1b6a91581f1580fad12f11b9593461f566153b6760299ba592746716d595583", - "zh:dad0ce1ac0fc934ab18fe4a744b768d8f9969099488b25d6c18c782b40478bfe", - "zh:e2cb8dfebfa0a358d0b52e15b04cfd82407a4c578c39b6ac62ac919b23ef97a6", - "zh:fc648d9464bcb6b360eb4885791675c2437f0d1c0e1b1b8f34fbb6d93a880ea9", - "zh:fd8250944e3794b9440cdaf4ac06f7a1672744e4bcb2e8a33b94c2ddf32ab988", + "zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef", + "zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3", + "zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb", + "zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e", + "zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11", + "zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7", + "zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9", + "zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1", + "zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff", + "zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339", ] } From e9782bb529378d2b86e1bf87b1a32d49a1618ea8 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 28 Aug 2026 08:48:58 +0000 Subject: [PATCH 061/118] fix(ci): install RPostgres for metadata updates --- .crow/process-updates.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 021d6cb..73bfff7 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -203,6 +203,7 @@ steps: - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi @@ -218,6 +219,7 @@ steps: LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true + R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' From 77a2f1f04a645f5001755c477c34e285e73215e0 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 07:31:28 +0000 Subject: [PATCH 062/118] fix(ci): install RPostgres for audit workflows (#172) ## Summary - Install RPostgres before running the missing-binaries audit. - Install RPostgres before running weekly patch proposal and automatic patch workflows. ## Validation - `prek run --files .crow/auto-apply-patches.yaml .crow/weekly-audit-missing.yaml .crow/weekly-patch-proposals.yaml` - `crow lint .crow/` - `git diff --check` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/172 --- .crow/auto-apply-patches.yaml | 2 +- .crow/weekly-audit-missing.yaml | 2 +- .crow/weekly-patch-proposals.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index 7c33c5b..5d6e6fe 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -58,7 +58,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT backend_options: kubernetes: diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 428aa04..25875da 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -147,7 +147,7 @@ steps: - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' backend_options: docker: diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 712e87d..7ea6c97 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -53,7 +53,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: From a8820e6e90a14469d4d566b3ee295aaeb5c51eff Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 12:19:52 +0000 Subject: [PATCH 063/118] fix(cdn): route Ubuntu 26.04 to resolute (#173) ## Motivation The generic `https://cran.rpkgs.com/` repository falls back to upstream CRAN on Ubuntu 26.04 because the edge router does not recognize its release version. This causes Resolute users to download and compile source packages even when matching binaries exist. ## Changes - Map Ubuntu 26.04 user agents to the `resolute` repository slot. - Cover ARM64 Ubuntu 26.04 routing with an edge middleware regression test. ## Verification - `just edge-test`, 1 test with 15 steps passed. - Confirmed the current live generic route redirects the Resolute user agent to upstream CRAN. - Confirmed the explicit `amd64/resolute/latest` repository installs `rlang` as a binary in `reg.devxy.io/r/r-ubuntu:4.6-resolute`. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/173 --- edge/rpkgs-router.test.ts | 9 +++++++++ edge/rpkgs-router.ts | 1 + 2 files changed, 10 insertions(+) diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 9493f8a..e9bbcc9 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -18,6 +18,7 @@ const UNION_SLOTS = 'amd64/alpine324'; const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; +const UA_R45_RESOLUTE = 'R/4.5.3 (Ubuntu 26.04) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)'; const UA_CURL = 'curl/8.0.1'; @@ -141,6 +142,14 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); }); + await t.step('resolves Ubuntu 26.04 to the resolute slot', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_RESOLUTE); + assertEquals( + res.location, + 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz', + ); + }); + await t.step('sends an unidentifiable distro to CRAN', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL); assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index 9278d4a..c9195a4 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -59,6 +59,7 @@ const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i; const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i; const UBUNTU_CODENAMES: Record = { + '26.04': 'resolute', '24.04': 'noble', '22.04': 'jammy', }; From b0d58f3f7c7f3577727be25da9368af3f45b7eff Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 12:25:22 +0000 Subject: [PATCH 064/118] feat(cdn): derive Ubuntu routes from codenames (#174) ## Motivation Ubuntu release codenames cannot be derived from version numbers, so the edge router currently needs a code change for every Ubuntu release. The generic repository setup can already read `VERSION_CODENAME` from `/etc/os-release` and send it in the user agent. ## Changes - Prefer a strictly validated `codename=` token for Ubuntu slot routing. - Retain the existing version-to-codename table for clients using the previous user-agent format. - Add a future Ubuntu 28.04 regression case whose codename does not exist in the middleware table. ## User-agent format `R/4.6.1 (Ubuntu 26.04; codename=resolute) (aarch64-unknown-linux-gnu)` ## Verification - `just edge-test`, 1 test with 16 steps passed. - `git diff --check` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/174 --- edge/rpkgs-router.test.ts | 10 ++++++++++ edge/rpkgs-router.ts | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index e9bbcc9..3020b2d 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -19,6 +19,8 @@ const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; const UA_R45_RESOLUTE = 'R/4.5.3 (Ubuntu 26.04) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; +const UA_R45_FUTURE_UBUNTU = + 'R/4.5.3 (Ubuntu 28.04; codename=dynamic-dugong) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)'; const UA_CURL = 'curl/8.0.1'; @@ -150,6 +152,14 @@ Deno.test('rpkgs-router', async (t) => { ); }); + await t.step('resolves a future Ubuntu release from its codename', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_FUTURE_UBUNTU); + assertEquals( + res.location, + 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz', + ); + }); + await t.step('sends an unidentifiable distro to CRAN', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL); assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index c9195a4..cdeac44 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -58,6 +58,7 @@ const MACOS_BIN_REGEX = const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i; const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i; +const UBUNTU_CODENAME_REGEX = /Ubuntu [\d.]+;\s*codename=([a-z][a-z0-9-]*)/i; const UBUNTU_CODENAMES: Record = { '26.04': 'resolute', '24.04': 'noble', @@ -137,6 +138,11 @@ function parseSlot(userAgent: string): string | null { const ubuntu = userAgent.match(UBUNTU_REGEX); if (ubuntu) { + const codenameMatch = userAgent.match(UBUNTU_CODENAME_REGEX); + if (codenameMatch) { + return `${arch}/${codenameMatch[1].toLowerCase()}`; + } + const codename = UBUNTU_CODENAMES[ubuntu[1]]; if (codename) { return `${arch}/${codename}`; From eeebef8edb8fb166cfb8125c71a690340560dfa3 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 14:26:37 +0000 Subject: [PATCH 065/118] fix(patches): support RcppParallel 6.2.1 This change will:\n\n- Relax the TBB include hunk context to tolerate the upstream TBB_CXXFLAGS block.\n- Pin the source patch to the verified RcppParallel version. --- local/patches/RcppParallel/force-bundled-tbb.patch | 6 +----- local/patches/registry.json | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/local/patches/RcppParallel/force-bundled-tbb.patch b/local/patches/RcppParallel/force-bundled-tbb.patch index c46a05e..5c37fa6 100644 --- a/local/patches/RcppParallel/force-bundled-tbb.patch +++ b/local/patches/RcppParallel/force-bundled-tbb.patch @@ -40,15 +40,11 @@ index 6f6a745..e407986 100644 if (is.null(name)) return(tbbRoot) -@@ -58,7 +58,7 @@ tbbCxxFlags <- function() { - flags <- c("-DRCPP_PARALLEL_USE_TBB=1") - +@@ -58,3 +58,3 @@ tbbCxxFlags <- function() { # if TBB_INC is set, apply those library paths - tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) + tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) if (!file.exists(tbbInc)) { - tbbInc <- system.file("include", package = "RcppParallel") - } @@ -117,7 +117,7 @@ tbbLdFlags <- function() { } diff --git a/local/patches/registry.json b/local/patches/registry.json index a7e29dd..2457304 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -1,7 +1,7 @@ [ { "package": "RcppParallel", - "versions": ">=6.0.0", + "versions": "6.2.1", "platforms": ["*"], "env": {}, "configure_args": [], From aba2063ea0ce8838239a1307a751b1e655916272 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 21:20:16 +0000 Subject: [PATCH 066/118] feat(edge): gate per-minor routing on published minors and add a staging zone (#175) ## Motivation `UNION_SLOTS` is empty, so per-minor routing has never been exercised end to end. Before it can be enabled and advertised, two things were missing: a way to test it without pointing production at it, and evidence that the published indexes actually support it. Verifying the data first turned up a defect that would have broken users the moment the flag was flipped. ## The defect `contribPath()` redirects to `contrib//` whenever the User-Agent carries any R minor, with no existence check and no fallback: ```ts const rMinor = extractRMinor(userAgent); return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; ``` Only `4.4`, `4.5` and `4.6` are published. `4.3` and `4.2` return 404 on all 16 slots. With `UNION_SLOTS` set, an R 4.3 client would be redirected to a non-existent index and see **zero** packages: a silent, total failure rather than a degraded one. R 4.3 is still advertised as supported on the website and in `docs/configuration.mdoc`, though `build-env-images` now pins only 4.6.0/4.5.3/4.4.3. ## Changes - **Gate routing on `KNOWN_MINORS`** (default `4.4,4.5,4.6`), falling back to the flat index for anything else. Unknown minor now behaves exactly as today. - **Honour `EXTRA_PUBLIC_HOSTS`.** `publicCdnOrigin()` falls back to the hardcoded `PUBLIC_CDN_ORIGIN` for any hostname not in `PUBLIC_CDN_HOSTS`, so a staging zone on a `b-cdn.net` hostname would redirect into *production* and silently measure the wrong system. This lets the identical script run on staging and redirect within itself. - **Add the `cran-rpkgs-test` pull zone** with `UNION_SLOTS` pre-enabled for all 16 slots, same B2 origin, served on the bunny default hostname so it needs no DNS record and is never advertised. - **Add `scripts/verify-r-minor-routing.sh`**, covering every `/` slot: index reachability per minor, the union property against flat, `Path:` target resolution, coverage parity across minors, and with `--live` the real User-Agent routing, the non-R User-Agent case, and that tarballs are never rewritten. - **Cover the fallback in the edge test suite** for both an unpublished minor (4.3) and a future one (4.7). ## Findings from the full run 112 passed, 16 failed across the 16 slots. Every failure is the same: no R 4.3 index. All 16 slots carry union indexes that are supersets of flat, every sampled `Path:` target resolves, and all indexes were republished within minutes of each other, so the build side is healthy. Coverage is **not** yet even, which is why "full coverage for ABI-sensitive packages" is not a claim to make yet: | slot | flat | 4.4 | 4.5 | 4.6 | |---|---|---|---|---| | amd64/resolute | 24305 | 24402 | 24748 | 24395 | | amd64/alpine324 | 24397 | 24457 | 24744 | 24448 | | amd64/noble | 24780 | 24805 | 24805 | 24805 | On the R 4.5-built distros (`resolute`, `alpine324`, and their arm64 twins) a 4.4 or 4.6 client sees ~300 fewer packages than a 4.5 client. On `noble`/`jammy`/`rhel9`/`alpine323` the spread is under 5. The new parity check encodes this with a configurable `PARITY_TOLERANCE`. ## Verification - `just edge-test`: 18 steps pass. The two new steps were confirmed to fail with the `KNOWN_MINORS` gate removed and pass with it. - `tofu validate`: passes. **Not applied** - no bunny.net or state credentials were available, so the staging zone still needs a `tofu apply`. - `scripts/verify-r-minor-routing.sh`: full 16-slot run, results above. - `shellcheck`: clean. ## Not done here Applying the staging zone, then running `BASE=https://cran-rpkgs-test.b-cdn.net scripts/verify-r-minor-routing.sh --live` against it. Production `UNION_SLOTS` is deliberately left empty. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/175 --- cdn.tf | 113 +++++++++++ edge/rpkgs-router.test.ts | 39 +++- edge/rpkgs-router.ts | 59 +++++- scripts/verify-r-minor-routing.sh | 309 ++++++++++++++++++++++++++++++ 4 files changed, 508 insertions(+), 12 deletions(-) create mode 100755 scripts/verify-r-minor-routing.sh diff --git a/cdn.tf b/cdn.tf index 96877b2..349486e 100644 --- a/cdn.tf +++ b/cdn.tf @@ -52,6 +52,26 @@ ### cran.rpkgs.com +locals { + rpkgs_slots = [ + for pair in setproduct( + ["amd64", "arm64"], + ["resolute", "noble", "jammy", "rhel8", "rhel9", "rhel10", "alpine323", "alpine324"] + ) : "${pair[0]}/${pair[1]}" + ] + + # The supported R minors: the current one plus the two previous, which is + # exactly what build-env-images installs as R_VERSION_LATEST / PREV1 / PREV2. + # These must stay in step. A minor listed here without a published index + # sends those clients to a 404; a published minor missing from this list + # sends them to CRAN for sources instead of serving the binaries we built. + rpkgs_supported_minors = ["4.4", "4.5", "4.6"] + + # bunny.net serves every pull zone on .b-cdn.net, so staging needs no + # DNS record and is never advertised. + rpkgs_test_hostname = "cran-rpkgs-test.b-cdn.net" +} + # The edge middleware that resolves the bare cran.rpkgs.com form to an # / slot and routes PACKAGES* to the per-R-minor slot. The source of # truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release. @@ -75,6 +95,13 @@ resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { required = false } +resource "bunnynet_compute_script_variable" "rpkgs_router_known_minors" { + script = bunnynet_compute_script.rpkgs_router.id + name = "KNOWN_MINORS" + default_value = join(",", local.rpkgs_supported_minors) + required = false +} + resource "bunnynet_pullzone" "cran_rpkgs_com" { name = "cran-rpkgs" @@ -147,6 +174,92 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } +### Staging zone for edge-router changes + +# Every published / slot. The staging zone enables per-minor routing +# for all of them at once; production adopts the same list only after +# `scripts/verify-r-minor-routing.sh --live` passes against staging. + +# A second copy of the same router, bound to the same B2 origin, so UNION_SLOTS +# can be exercised end to end before production is touched. +resource "bunnynet_compute_script" "rpkgs_router_test" { + type = "middleware" + name = "rpkgs-router-test" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +resource "bunnynet_compute_script_variable" "rpkgs_router_test_union_slots" { + script = bunnynet_compute_script.rpkgs_router_test.id + name = "UNION_SLOTS" + default_value = join(",", local.rpkgs_slots) + required = false +} + +# Without this the staging zone rewrites to PUBLIC_CDN_ORIGIN, so its redirects +# land on production and the test silently measures the wrong system. +resource "bunnynet_compute_script_variable" "rpkgs_router_test_extra_hosts" { + script = bunnynet_compute_script.rpkgs_router_test.id + name = "EXTRA_PUBLIC_HOSTS" + default_value = local.rpkgs_test_hostname + required = false +} + +resource "bunnynet_compute_script_variable" "rpkgs_router_test_known_minors" { + script = bunnynet_compute_script.rpkgs_router_test.id + name = "KNOWN_MINORS" + default_value = join(",", local.rpkgs_supported_minors) + required = false +} + +resource "bunnynet_pullzone" "cran_rpkgs_test" { + name = "cran-rpkgs-test" + + cache_errors = false + + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + + origin { + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router_test.id + } + + routing { + filters = [ + "scripting", + ] + } + + s3_auth_enabled = true + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" + + cache_enabled = true + request_coalescing_enabled = true + block_post_requests = true + + cache_vary_headers = ["User-Agent"] + + # Staging carries only synthetic verification traffic, so the production + # ceilings would be pure headroom. + limit_requests = 500 + limit_connections = 100 + + safehop_enabled = true + add_canonical_header = true + cache_stale = ["offline", "updating"] + block_ips = var.cdn_block_ips + + # 1 TB + limit_bandwidth = 1000000000000 + + block_root_path = true +} + + # Alliance SwissPass historically used a separate, manually configured pull # zone. Adopt it so both public repositories use the same B2 origin, middleware # release and cache behavior. diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 3020b2d..3e2fe8b 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -17,6 +17,8 @@ const UNION_SLOTS = 'amd64/alpine324'; const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R43_MUSL = 'R (4.3.3 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R47_MUSL = 'R (4.7.0 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; const UA_R45_RESOLUTE = 'R/4.5.3 (Ubuntu 26.04) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; const UA_R45_FUTURE_UBUNTU = @@ -96,6 +98,33 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`); }); + // We publish binaries only for the supported window. An excluded minor has + // no slot we can serve safely, so it goes to CRAN for sources rather than + // to a 404 or to binaries built under another minor. + await t.step('sends an excluded R minor to CRAN for the index', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R43_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); + }); + + await t.step('sends a future R minor to CRAN too', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R47_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); + }); + + // The index and the tarballs R resolves against it have to come from the + // same place. Serving one from CRAN and the other from here would hand R a + // binary where it expects a source tarball. + await t.step('sends an excluded minor to CRAN for tarballs as well', async () => { + const res = await probe(`${SLOT}/foo_1.0.tar.gz`, UA_R43_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/foo_1.0.tar.gz'); + }); + + await t.step('leaves an excluded minor alone on a slot outside UNION_SLOTS', async () => { + const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R43_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + await t.step('routes PACKAGES and PACKAGES.rds too', async () => { for (const file of ['PACKAGES', 'PACKAGES.rds']) { const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL); @@ -146,18 +175,12 @@ Deno.test('rpkgs-router', async (t) => { await t.step('resolves Ubuntu 26.04 to the resolute slot', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_RESOLUTE); - assertEquals( - res.location, - 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz', - ); + assertEquals(res.location, 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz'); }); await t.step('resolves a future Ubuntu release from its codename', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_FUTURE_UBUNTU); - assertEquals( - res.location, - 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz', - ); + assertEquals(res.location, 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz'); }); await t.step('sends an unidentifiable distro to CRAN', async () => { diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index cdeac44..05b6e18 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -27,7 +27,17 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; const CRAN_ORIGIN = 'https://cran.r-project.org'; -const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']); +const PUBLIC_CDN_HOSTS = new Set([ + 'cran.rpkgs.com', + 'cran.allianceswisspass.devxy.io', + // Staging hostnames, so the identical script can run on a test pull zone and + // redirect within itself. Without this a test zone rewrites to + // PUBLIC_CDN_ORIGIN, quietly exercising production instead of itself. + ...(Deno.env.get('EXTRA_PUBLIC_HOSTS') ?? '') + .split(',') + .map((host) => host.trim()) + .filter((host) => host.length > 0), +]); /** Slots ("/", comma separated) whose per-minor index is a union. */ const UNION_SLOTS = new Set( @@ -37,6 +47,21 @@ const UNION_SLOTS = new Set( .filter((slot) => slot.length > 0), ); +/** + * R minors for which a per-minor index is actually published. + * + * contribPath() has no way to probe the origin, so a minor that is not + * published here must fall back to the flat index. Routing an unlisted minor + * would send that client to a 404 and it would see no packages at all - a + * silent, total failure rather than a degraded one. + */ +const KNOWN_MINORS = new Set( + (Deno.env.get('KNOWN_MINORS') ?? '4.4,4.5,4.6') + .split(',') + .map((minor) => minor.trim()) + .filter((minor) => minor.length > 0), +); + /** `///latest/src/contrib[/]` */ const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/; @@ -96,6 +121,18 @@ function publicCdnOrigin(url: URL): string { return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; } +/** + * True when the client reports an R minor that we deliberately do not serve. + * + * A client that reports no minor at all is not "unsupported": non-R fetchers + * (mirror scripts, image builds) must keep getting the flat slot. Only a + * known-and-excluded minor falls through to CRAN. + */ +function isExcludedMinor(userAgent: string): boolean { + const rMinor = extractRMinor(userAgent); + return rMinor !== null && !KNOWN_MINORS.has(rMinor); +} + function extractRMinor(userAgent: string): string | null { for (const regex of R_MINOR_REGEXES) { const match = userAgent.match(regex); @@ -177,8 +214,8 @@ function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: * The contrib path a request should be served from, relative to the slot. * * Returns the per-minor path for an index file when the slot is known to carry - * a union index and the client's R minor is known; otherwise the flat path, - * which is what every client sees today. + * a union index and the client's R minor is one we publish; otherwise the flat + * path, which is what every client sees today. */ function contribPath(slot: string, rest: string, userAgent: string): string { const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`; @@ -188,7 +225,7 @@ function contribPath(slot: string, rest: string, userAgent: string): string { } const rMinor = extractRMinor(userAgent); - return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; + return rMinor && KNOWN_MINORS.has(rMinor) ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; } BunnySDK.net.http @@ -224,6 +261,16 @@ BunnySDK.net.http return Promise.resolve(ctx.request); } + // An R minor outside the supported window has no binaries we can safely + // serve, so the whole interaction goes to CRAN: the index and the + // tarballs R will resolve against it. Serving the index from CRAN but + // tarballs from here would hand R a binary where it expects a source + // tarball, which fails in a far more confusing way than not being + // served at all. + if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}/src/contrib/${rest}`)); + } + const target = contribPath(slot, rest, userAgent); if (target === path) { return Promise.resolve(ctx.request); @@ -238,6 +285,10 @@ BunnySDK.net.http return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); } + if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + const rest = srcContrib ? srcContrib[1] : ''; return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); } diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh new file mode 100755 index 0000000..28acd92 --- /dev/null +++ b/scripts/verify-r-minor-routing.sh @@ -0,0 +1,309 @@ +#!/usr/bin/env bash +# +# Verify per-R-minor index routing for cran.rpkgs.com across every published +# / slot. +# +# The edge router (edge/rpkgs-router.ts) rewrites PACKAGES* requests to +# `contrib//` when the slot is listed in UNION_SLOTS and the client's +# User-Agent carries an R minor. Two properties have to hold before a slot may +# be added to UNION_SLOTS: +# +# 1. the per-minor index is a UNION of the per-minor and flat slots, so +# routing to it hides nothing the flat index carries; and +# 2. every R minor a client might report resolves to an index that exists, +# because contribPath() does not check existence and has no fallback. +# +# Modes: +# (default) Resolve routing decisions without depending on UNION_SLOTS being +# set. Safe to run before enabling: it reads the per-minor indexes +# directly and reproduces the router's target path. +# --live Additionally drive the real CDN with R User-Agents and assert the +# bytes served match the expected index. Only meaningful once the +# slot is in UNION_SLOTS. +# +# Usage: +# scripts/verify-r-minor-routing.sh +# scripts/verify-r-minor-routing.sh --live +# MINORS="4.4 4.5" SAMPLE=10 scripts/verify-r-minor-routing.sh +# +# Exits non-zero if any check fails. + +set -uo pipefail + +BASE=${BASE:-https://cran.rpkgs.com} +ARCHES=${ARCHES:-"amd64 arm64"} +DISTROS=${DISTROS:-"resolute noble jammy rhel8 rhel9 rhel10 alpine323 alpine324"} +# The supported window: the current R minor plus the two previous, matching +# build-env-images' R_VERSION_LATEST/PREV1/PREV2 and cdn.tf's +# local.rpkgs_supported_minors. Each of these must have a published index. +MINORS=${MINORS:-"4.4 4.5 4.6"} +# Minors we deliberately do not serve. These must have NO published index and, +# once routing is live, must be sent to CRAN for sources rather than 404ing or +# being handed binaries built under another minor. +EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"} +# How many Path: targets to HEAD-check per slot/minor. 0 disables. +SAMPLE=${SAMPLE:-5} +# Largest package-count shortfall a non-primary minor may have against the best +# minor on the same slot before coverage counts as uneven. A slot built under +# one R minor carries fewer per-minor binaries for the others; until that gap +# closes, "full coverage for ABI-sensitive packages" is not a claim we can make. +PARITY_TOLERANCE=${PARITY_TOLERANCE:-25} +LIVE=0 + +for arg in "$@"; do + case "$arg" in + --live) LIVE=1 ;; + -h | --help) + sed -n '2,32p' "$0" + exit 0 + ;; + *) + echo "unknown argument: $arg" >&2 + exit 2 + ;; + esac +done + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +PASS=0 +FAIL=0 +FAILURES="" + +ok() { + PASS=$((PASS + 1)) + printf ' ok %s\n' "$1" +} + +bad() { + FAIL=$((FAIL + 1)) + FAILURES="${FAILURES}\n - $1" + printf ' FAIL %s\n' "$1" +} + +# Fetch a URL into a file, echoing the HTTP status. Cached per URL. +fetch() { + local url=$1 dest=$2 ua=${3:-} + if [ -s "$dest" ]; then + cat "$dest.status" + return 0 + fi + local status + if [ -n "$ua" ]; then + status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + else + status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + fi + echo "$status" > "$dest.status" + echo "$status" +} + +head_status() { + curl -sS -o /dev/null -w '%{http_code}' -I --max-time 60 "$1" 2>/dev/null +} + +# Package names from a gzipped PACKAGES index, sorted. +pkg_names() { + gunzip -c "$1" 2>/dev/null | awk '/^Package:/ {print $2}' | sort -u +} + +# " " pairs for entries that carry a Path: field. +path_entries() { + gunzip -c "$1" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; ver = ""; path = "" } + /^Version:/ { ver = $2 } + /^Path:/ { path = $2 } + /^$/ { if (pkg != "" && path != "") print pkg, ver, path; pkg = "" } + END { if (pkg != "" && path != "") print pkg, ver, path } + ' +} + +# An R User-Agent of the shape R actually sends. +r_user_agent() { + printf 'R/%s.0 (Ubuntu 24.04; codename=noble) (x86_64-pc-linux-gnu x86_64 linux-gnu)' "$1" +} + +echo "verify-r-minor-routing: $BASE" +echo " slots: $(echo "$ARCHES" | wc -w) arch x $(echo "$DISTROS" | wc -w) os" +echo " minors: $MINORS (excluded: $EXCLUDED_MINORS)" +echo " live: $LIVE" +echo + +for arch in $ARCHES; do + for distro in $DISTROS; do + slot="$arch/$distro" + echo "$slot" + + flat_url="$BASE/$slot/latest/src/contrib/PACKAGES.gz" + flat_file="$WORK/${arch}-${distro}-flat.gz" + flat_status=$(fetch "$flat_url" "$flat_file") + + if [ "$flat_status" != "200" ]; then + bad "$slot flat index unreachable (HTTP $flat_status)" + continue + fi + + pkg_names "$flat_file" > "$flat_file.names" + flat_count=$(wc -l < "$flat_file.names") + if [ "$flat_count" -lt 1000 ]; then + bad "$slot flat index has only $flat_count packages" + continue + fi + ok "$slot flat index: $flat_count packages" + + for minor in $MINORS; do + minor_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" + minor_file="$WORK/${arch}-${distro}-${minor}.gz" + minor_status=$(fetch "$minor_url" "$minor_file") + + # A minor the router would route to must exist, or clients on that R + # version get a 404 and see no packages at all. + if [ "$minor_status" != "200" ]; then + bad "$slot R $minor index missing (HTTP $minor_status) - routing would 404 for R $minor clients" + continue + fi + + pkg_names "$minor_file" > "$minor_file.names" + minor_count=$(wc -l < "$minor_file.names") + + # Union property: nothing the flat index carries may be missing here. + missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5) + missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l) + if [ "$missing_count" -ne 0 ]; then + bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))" + else + ok "$slot R $minor index: $minor_count packages, union holds" + fi + + # Path: entries steer to per-minor binaries; they must resolve. + if [ "$SAMPLE" -gt 0 ]; then + path_entries "$minor_file" > "$minor_file.paths" + total_paths=$(wc -l < "$minor_file.paths") + broken=0 + checked=0 + while read -r pkg ver path; do + [ -z "${pkg:-}" ] && continue + tarball="$BASE/$slot/latest/src/contrib/$path/${pkg}_${ver}.tar.gz" + status=$(head_status "$tarball") + checked=$((checked + 1)) + if [ "$status" != "200" ]; then + broken=$((broken + 1)) + [ "$broken" -le 2 ] && printf ' broken target: %s (HTTP %s)\n' "$tarball" "$status" + fi + done < <(shuf -n "$SAMPLE" "$minor_file.paths" 2>/dev/null || head -n "$SAMPLE" "$minor_file.paths") + + if [ "$broken" -ne 0 ]; then + bad "$slot R $minor: $broken/$checked sampled Path: targets do not resolve (of $total_paths total)" + elif [ "$checked" -gt 0 ]; then + ok "$slot R $minor: $checked/$checked sampled Path: targets resolve (of $total_paths total)" + fi + fi + + # Live routing: what a real R client on this minor actually receives. + if [ "$LIVE" -eq 1 ]; then + ua=$(r_user_agent "$minor") + live_file="$WORK/${arch}-${distro}-${minor}-live.gz" + live_status=$(fetch "$flat_url" "$live_file" "$ua") + if [ "$live_status" != "200" ]; then + bad "$slot R $minor live request failed (HTTP $live_status)" + elif cmp -s "$live_file" "$minor_file"; then + ok "$slot R $minor live request served the per-minor index" + elif cmp -s "$live_file" "$flat_file"; then + bad "$slot R $minor live request served the FLAT index - slot not in UNION_SLOTS?" + else + bad "$slot R $minor live request served neither the per-minor nor the flat index" + fi + fi + done + + # Coverage parity across minors. The union property only guarantees no + # client loses packages relative to the flat index; it says nothing about a + # 4.4 client seeing fewer packages than a 4.5 client on the same slot. + best=0 + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz.names" + [ -s "$f" ] || continue + c=$(wc -l < "$f") + [ "$c" -gt "$best" ] && best=$c + done + if [ "$best" -gt 0 ]; then + uneven="" + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz.names" + [ -s "$f" ] || continue + c=$(wc -l < "$f") + gap=$((best - c)) + [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap" + done + if [ -n "$uneven" ]; then + bad "$slot coverage uneven across minors (vs best $best):$uneven" + else + ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)" + fi + fi + + # Excluded minors: no published index, and under --live a redirect to CRAN. + for minor in $EXCLUDED_MINORS; do + ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" + ex_status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 60 "$ex_url" 2>/dev/null) + if [ "$ex_status" = "200" ]; then + bad "$slot R $minor is excluded but an index is published - the two lists disagree" + else + ok "$slot R $minor correctly has no published index" + fi + + if [ "$LIVE" -eq 1 ]; then + loc=$(curl -sS -o /dev/null -w '%{redirect_url}' -A "$(r_user_agent "$minor")" \ + --max-time 60 "$flat_url" 2>/dev/null) + case "$loc" in + https://cran.r-project.org/*) + ok "$slot R $minor is sent to CRAN ($loc)" + ;; + "") + bad "$slot R $minor was served directly instead of being sent to CRAN" + ;; + *) + bad "$slot R $minor redirected somewhere unexpected: $loc" + ;; + esac + fi + done + + # A client whose User-Agent carries no R version must keep getting the flat + # index, never a per-minor one. + if [ "$LIVE" -eq 1 ]; then + plain_file="$WORK/${arch}-${distro}-plain.gz" + plain_status=$(fetch "$flat_url" "$plain_file" "curl/8.0.0") + if [ "$plain_status" != "200" ]; then + bad "$slot non-R User-Agent request failed (HTTP $plain_status)" + elif cmp -s "$plain_file" "$flat_file"; then + ok "$slot non-R User-Agent still served the flat index" + else + bad "$slot non-R User-Agent was routed away from the flat index" + fi + + # Tarball requests must never be rewritten into a per-minor directory: + # flat-slot packages do not live there. + sample_pkg=$(gunzip -c "$flat_file" | awk '/^Package:/ {p=$2} /^Version:/ {print p, $2; exit}') + if [ -n "$sample_pkg" ]; then + # shellcheck disable=SC2086 # deliberate split into $1 (package) and $2 (version) + set -- $sample_pkg + tb="$BASE/$slot/latest/src/contrib/${1}_${2}.tar.gz" + tb_status=$(curl -sS -o /dev/null -w '%{http_code}' -A "$(r_user_agent 4.5)" --max-time 60 "$tb" 2>/dev/null) + if [ "$tb_status" = "200" ]; then + ok "$slot tarball request under an R User-Agent still resolves" + else + bad "$slot tarball ${1}_${2}.tar.gz broke under an R User-Agent (HTTP $tb_status)" + fi + fi + fi + done +done + +echo +echo "passed: $PASS failed: $FAIL" +if [ "$FAIL" -ne 0 ]; then + printf 'failures:%b\n' "$FAILURES" + exit 1 +fi From f3077677d7852374f18098eef9df961a005c86a2 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 09:55:30 +0000 Subject: [PATCH 067/118] ci: add a reindex-only manual workflow (#177) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation `weekly-rebuild-reindex` exists to run after `weekly-rebuild-missing`: it declares `depends_on: weekly-rebuild-missing` and is gated on that workflow's `weekly_rebuild_missing` variable. Triggering it manually therefore also starts hours of package rebuilds. That is the wrong tool when only the index needs regenerating. After rpkgs/bincraft#113 (v5.1.5), which changes how `union_index_records()` decides what a per-minor index steers to, every object in the bucket is already correct and only `PACKAGES*` is stale. Rebuilding to fix an index is pure waste, and the natural cron would take a full cycle to reach every slot. ## Change Adds `.crow/reindex.yaml`: the index half on its own, manual only, no dependency on a rebuild. It reuses the same matrix and the same steps as `weekly-rebuild-reindex` — install the latest bincraft release, republish the generic index, loop the installed R versions republishing each per-minor index, purge the edge. No package is built. Gated on a new `reindex` variable so it cannot be started by the rebuild gate, defaulting to `none` so a manual pipeline creation (which instantiates every file in `.crow/`) matches no matrix row. ```sh crow pipeline create devxy/build-cran-binaries --var reindex=all crow pipeline create devxy/build-cran-binaries --var reindex=ubuntu-2404-amd64 ``` ## Verification - `crow lint .crow/` passes. - Gate is manual-only and evaluates `reindex`, with no `depends_on` and no `runs_on` carried over from the rebuild coupling. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/177 --- .crow/reindex.yaml | 188 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 .crow/reindex.yaml diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml new file mode 100644 index 0000000..b8f0d86 --- /dev/null +++ b/.crow/reindex.yaml @@ -0,0 +1,188 @@ +# Re-index every slot without rebuilding anything. +# +# `weekly-rebuild-reindex` exists to run after `weekly-rebuild-missing`, so it +# depends on that workflow and shares its gate: triggering it manually also +# starts hours of package rebuilds. That is the wrong tool when only the index +# needs regenerating - after a bincraft release that changes how the index is +# written, for instance, where the objects in the bucket are already correct +# and only `PACKAGES*` is stale. +# +# This workflow does the index half on its own. It installs the latest bincraft +# release, republishes the generic and per-R-minor indexes for each slot, and +# purges the edge. No package is built. +# +# Trigger with the `reindex` variable set to `all` or to a single +# `-`, e.g. +# +# crow pipeline create devxy/build-cran-binaries --var reindex=all + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the + # default must match no matrix row. + reindex: + description: "Re-index target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: manual + evaluate: 'reindex == "all" || reindex == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + PLATFORM: ${OS} + ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + commands: + - apk add --no-cache -q bash curl jq + # Crow carries the checkout from the re-index step into this step. + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES + # Runs on every row rather than on one designated slot: a cron fires only + # its own slot's row, so gating on a named slot would leave every other + # slot unpurged. A manual "all" run therefore purges the zone 18 times, + # which is a cheap API call and rare. + # + # Run it even when the re-index above failed: the objects were still + # replaced, and a stale edge is exactly what keeps them hidden. + when: + - status: [success, failure] From 85295a949536c863ddaf0107b15dacf1401dcc07 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 09:55:37 +0000 Subject: [PATCH 068/118] fix(cdn): resolve a pull zone when the API answers with a bare array (#178) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Every reindex reports `failure` at the purge step: ``` Purging BunnyCDN pull zone 3857050 Purged pull zone 3857050 (HTTP 204) jq: error (at /tmp/tmp.eFPFmO:0): Cannot index array with string "Items" Could not find BunnyCDN pull zone for hostname cran.allianceswisspass.devxy.io ``` `cran.rpkgs.com` purges fine. The Alliance zone never has, so it is still serving objects that rebuilds replaced, behind a ~370-day `cache_expiration_time`. ## The defect ```sh jq -r '(.Items // .)[] | ...' ``` This was meant to accept both response shapes. It accepts neither: indexing an array with a string is an **error** in jq, not a null, so `//` never gets the chance to substitute and the whole expression aborts. The listing endpoint answers with a bare array for this account, so the lookup has always failed. ## Change - Select the array explicitly by type instead of relying on `//` to absorb an error. - Check the HTTP status of the listing call. It was previously used unconditionally, so an auth or rate-limit failure surfaced as "could not find hostname" — pointing at the wrong thing entirely. - Fail when a hostname matches multiple zones rather than silently purging whichever jq emitted first. - Request `perPage=1000`, so a paginated response cannot silently truncate the zone list. ## Verification Ran the current `main` script and the fixed one against a stubbed `curl` returning an array-shaped listing: ``` === BEFORE (main) === Purged pull zone 3857050 (HTTP 204) jq: error (at ...): Cannot index array with string ("Items") Could not find BunnyCDN pull zone for hostname cran.allianceswisspass.devxy.io === AFTER === Purged pull zone 3857050 (HTTP 204) Purging BunnyCDN pull zone 222 Purged pull zone 222 (HTTP 204) ``` The jq expression was also checked against both an array-shaped and an object-shaped (`.Items`) response; the old one fails the array case, the new one handles both. `shellcheck` clean. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/178 --- scripts/purge_cdn_zone.sh | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 648dfc3..6c07eef 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -47,12 +47,31 @@ resolve_zone_id() { fi response_file=$(mktemp) - curl -sS -o "${response_file}" \ - -H "AccessKey: ${api_key}" \ - "https://api.bunny.net/pullzone" + local status + status=$( + curl -sS -o "${response_file}" -w '%{http_code}' \ + -H "AccessKey: ${api_key}" \ + "https://api.bunny.net/pullzone?perPage=1000" + ) + + if [[ "${status}" != "200" ]]; then + echo "Listing BunnyCDN pull zones failed with HTTP ${status}:" >&2 + head -c 500 "${response_file}" >&2 + echo >&2 + rm -f "${response_file}" + exit 1 + fi + + # The endpoint answers with a bare array on some accounts and a paginated + # object on others. `.Items // .` looks like it covers both but does not: + # indexing an array with a string is an *error*, and `//` only substitutes + # for null, so the array case aborted with + # "Cannot index array with string" and the zone was never purged. zone_id=$( jq -r --arg hostname "${zone}" \ - '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \ + '(if type == "object" then (.Items // []) else . end)[] + | select(any(.Hostnames[]?; .Value == $hostname)) + | .Id' \ "${response_file}" ) rm -f "${response_file}" @@ -62,6 +81,12 @@ resolve_zone_id() { exit 1 fi + # Two zones sharing a hostname would purge only whichever jq emitted first. + if [[ $(wc -l <<<"${zone_id}") -gt 1 ]]; then + echo "Hostname ${zone} matched multiple pull zones: ${zone_id//$'\n'/ }" >&2 + exit 1 + fi + echo "${zone_id}" } From a3004695a791a9081e07cf10d4aa1a7ad280d52d Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 09:55:44 +0000 Subject: [PATCH 069/118] test(verify): gate on regressions against the generic slot, not fallback rate (#179) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation The readiness check added in #175 failed a slot when more than 10% of its per-minor entries were source fallbacks. That stopped being a meaningful signal once rpkgs/bincraft#113 and #114 landed. Since bincraft keeps a matching-minor generic binary out of a fallback's shadow, a surviving fallback means the generic slot's binary was built under a **different** minor — unsafe for that client anyway. Serving source there is correct, just slow. Failing on that share blocks slots that are genuinely ready: `amd64/noble` sits at 53% for 4.5 and 4.6 while regressing nobody. ## Change Gate on the thing that actually decides enablement: packages a client of minor M would receive as **source** through per-minor routing while the generic slot holds a binary built under **M itself**. That is strictly worse than not routing at all, and must be zero. Fallback share is still printed, as context rather than a verdict. ## Verification Measured across every reindexed slot and minor after the `reindex=all` run: zero regressions everywhere. | slot | 4.4 | 4.5 | 4.6 | |---|---|---|---| | amd64/noble | 0 | 0 | 0 | | amd64/jammy | 0 | 0 | 0 | | amd64/rhel9 | 0 | 0 | 0 | | amd64/rhel10 | 0 | 0 | 0 | | amd64/resolute | 0 | 0 | 0 | | arm64/noble | 0 | 0 | 0 | `shellcheck` clean; script exercised against the live indexes. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/179 --- scripts/verify-r-minor-routing.sh | 62 +++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh index 28acd92..3e1a7e4 100755 --- a/scripts/verify-r-minor-routing.sh +++ b/scripts/verify-r-minor-routing.sh @@ -48,6 +48,17 @@ SAMPLE=${SAMPLE:-5} # one R minor carries fewer per-minor binaries for the others; until that gap # closes, "full coverage for ABI-sensitive packages" is not a claim we can make. PARITY_TOLERANCE=${PARITY_TOLERANCE:-25} +# Source fallbacks are reported, not failed on. Since bincraft learned to keep +# a matching-minor generic binary out of a fallback's shadow, a remaining +# fallback means the generic slot's binary was built under a *different* minor, +# which is unsafe for this client anyway: serving source there is correct, just +# slow. The gate below is what actually matters. +# +# A REGRESSION is a package this client would receive as source through +# per-minor routing but as a binary built under its own minor from the generic +# slot. That is strictly worse than not routing at all, and must be zero before +# a slot is added to UNION_SLOTS. +MAX_REGRESSIONS=${MAX_REGRESSIONS:-0} LIVE=0 for arg in "$@"; do @@ -108,6 +119,38 @@ pkg_names() { gunzip -c "$1" 2>/dev/null | awk '/^Package:/ {print $2}' | sort -u } +# " " for a per-minor index: how many entries carry a +# Path: field, and how many of those lack a Built: field (i.e. are sources). +fallback_counts() { + gunzip -c "$1" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; path = ""; built = "" } + /^Path:/ { path = $2 } + /^Built:/ { built = $2 } + /^$/ { if (pkg != "" && path != "") { n++; if (built == "") s++ } pkg = "" } + END { if (pkg != "" && path != "") { n++; if (built == "") s++ } + printf "%d %d\n", n, s } + ' +} + +# How many packages a client of would receive as source through +# per-minor routing while the generic slot holds a binary built under that very +# minor. Zero is the bar for enabling a slot. +regression_count() { + local minor_file=$1 flat_file=$2 minor=$3 + gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" ' + /^Package:/ { pkg = $2; built = "" } + /^Built:/ { built = $2 " " $3 } + /^$/ { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" } + ' | sort -u > "$minor_file.flatbin" + gunzip -c "$minor_file" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; path = ""; built = "" } + /^Path:/ { path = $2 } + /^Built:/ { built = $2 } + /^$/ { if (pkg != "" && path != "" && built == "") print pkg; pkg = "" } + ' | sort -u > "$minor_file.src" + comm -12 "$minor_file.src" "$minor_file.flatbin" | wc -l +} + # " " pairs for entries that carry a Path: field. path_entries() { gunzip -c "$1" 2>/dev/null | awk ' @@ -176,6 +219,25 @@ for arch in $ARCHES; do ok "$slot R $minor index: $minor_count packages, union holds" fi + # A per-minor entry that is a source fallback resolves fine but makes the + # client compile. Routing to a slot that is mostly fallbacks does not + # deliver the binaries we advertise. + read -r steered fallbacks <<< "$(fallback_counts "$minor_file")" + if [ "${steered:-0}" -gt 0 ]; then + pct=$((fallbacks * 100 / steered)) + printf ' note: %s/%s per-minor entries are source fallbacks (%s%%)\n' \ + "$fallbacks" "$steered" "$pct" + fi + + # The gate: nothing may arrive as source here that the generic slot would + # have served as a binary built under this same minor. + regressions=$(regression_count "$minor_file" "$flat_file" "$minor") + if [ "${regressions:-0}" -gt "$MAX_REGRESSIONS" ]; then + bad "$slot R $minor: $regressions packages would be served as source but exist as an R $minor binary in the generic slot" + else + ok "$slot R $minor: no regression against the generic slot" + fi + # Path: entries steer to per-minor binaries; they must resolve. if [ "$SAMPLE" -gt 0 ]; then path_entries "$minor_file" > "$minor_file.paths" From d38a4b5746e4f45db67594b9bce50cb149b8560e Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 10:00:49 +0000 Subject: [PATCH 070/118] test(verify): report uneven coverage instead of failing on it (#180) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation The full 16-slot run came back 139 passed, 5 failed. Four of the five were `coverage uneven across minors` on `resolute` and `alpine324` (both arches) — and they are not defects. Those slots are built under R 4.5, so their 4.5 union carries ABI-risky packages that only exist as 4.5 builds. A 4.4 or 4.6 client cannot safely load them, which is the whole reason per-minor slots exist. Their absence from the 4.4 and 4.6 indexes is correct behaviour, and failing the run on it blocks four slots that regress nobody. This is the same mistake as the source-fallback share, which was demoted to a note for the same reason. ## Change Report uneven coverage; do not fail on it. The two checks answer different questions and should not share an exit code: - **`MAX_REGRESSIONS`** gates *enablement*: would routing serve a client source where the generic slot holds a binary of that client's own minor? Must be zero. - **parity** gates the *claim*: can we advertise full coverage for ABI-sensitive packages? Informative, and currently no. ## Verification `amd64/resolute` now passes with the shortfall printed as a note: ``` ok amd64/resolute R 4.6: no regression against the generic slot note: amd64/resolute coverage uneven across minors (vs best 24748): R4.4:-345 R4.6:-352 passed: 8 failed: 0 ``` `shellcheck` clean. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/180 --- scripts/verify-r-minor-routing.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh index 3e1a7e4..3957f67 100755 --- a/scripts/verify-r-minor-routing.sh +++ b/scripts/verify-r-minor-routing.sh @@ -43,10 +43,13 @@ MINORS=${MINORS:-"4.4 4.5 4.6"} EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"} # How many Path: targets to HEAD-check per slot/minor. 0 disables. SAMPLE=${SAMPLE:-5} -# Largest package-count shortfall a non-primary minor may have against the best -# minor on the same slot before coverage counts as uneven. A slot built under -# one R minor carries fewer per-minor binaries for the others; until that gap -# closes, "full coverage for ABI-sensitive packages" is not a claim we can make. +# Package-count shortfall against the best minor on the same slot, above which +# coverage is reported as uneven. Reported, not failed on: the packages a +# non-primary minor lacks are ABI-risky ones built under the primary minor, +# which a client on another minor cannot safely load anyway, so their absence +# is correct. This gates the *claim* ("full coverage for ABI-sensitive +# packages"), not whether routing is safe to enable - MAX_REGRESSIONS does +# that. PARITY_TOLERANCE=${PARITY_TOLERANCE:-25} # Source fallbacks are reported, not failed on. Since bincraft learned to keep # a matching-minor generic binary out of a fallback's shadow, a remaining @@ -299,7 +302,8 @@ for arch in $ARCHES; do [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap" done if [ -n "$uneven" ]; then - bad "$slot coverage uneven across minors (vs best $best):$uneven" + printf ' note: %s coverage uneven across minors (vs best %s):%s\n' \ + "$slot" "$best" "$uneven" else ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)" fi From a2923bf063579d6be84e590f9f0b4b33b4cacb24 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 10:31:35 +0000 Subject: [PATCH 071/118] fix(cdn): purge the staging zone too (#181) ## Motivation `cran-rpkgs-test` was added as a second pull zone on the same B2 origin, but it was never added to `BUNNY_PULLZONES`. It therefore keeps serving pre-reindex indexes behind the same ~370 day `cache_expiration_time` as production, and nothing ever refreshes it. That is not cosmetic. The zone exists to be measured, and a verification run against it measures whatever the edge still holds: ``` production: regressions=0 AGHmatrix Path=NA Built=R 4.5.3; x86_64-pc staging : regressions=161 AGHmatrix Path=4.5 Built=(none) ``` Same objects, same origin, 161 phantom regressions. I chased that number through two wrong diagnoses before noticing production and staging disagreed. ## Change Add `cran-rpkgs-test.b-cdn.net` to the purge list in both reindex pipelines. ## Note A `Cache-Control: no-cache` request header is not a substitute. It was added to the verification script and did **not** clear this: bunny does not honour it for these objects. Purging is the mechanism that works. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/181 --- .crow/reindex.yaml | 7 ++++++- .crow/weekly-rebuild-reindex.yaml | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml index b8f0d86..b13245a 100644 --- a/.crow/reindex.yaml +++ b/.crow/reindex.yaml @@ -172,7 +172,12 @@ steps: from_secret: BUNNYNET_API_KEY # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + # The staging zone is listed too. It shares the B2 origin, so an index + # it still holds is a stale copy of the same object, and its + # cache_expiration_time is the same ~370 days: without a purge here it + # serves pre-reindex indexes indefinitely and any verification run + # against it measures the past. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' commands: - apk add --no-cache -q bash curl jq # Crow carries the checkout from the re-index step into this step. diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 2e0f2a0..c9f9a8a 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -175,7 +175,12 @@ steps: from_secret: BUNNYNET_API_KEY # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + # The staging zone is listed too. It shares the B2 origin, so an index + # it still holds is a stale copy of the same object, and its + # cache_expiration_time is the same ~370 days: without a purge here it + # serves pre-reindex indexes indefinitely and any verification run + # against it measures the past. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' commands: - apk add --no-cache -q bash curl jq # Crow carries the checkout from the re-index step into this step. From 5f901312a61d51b1998c8b37a62cc0ade725d03d Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 12:29:54 +0000 Subject: [PATCH 072/118] feat(build): allow the per-minor pass to run under R 4.6 (#183) ## Motivation The supported window is the latest R minor plus the two previous, which the build images install as `R_VERSION_LATEST=4.6.0`, `PREV1=4.5.3`, `PREV2=4.4.3`. This pipeline's `R_VERSION` offered only the latter two, so **no pipeline could run `local/build-all.R --sensitive-only` under 4.6** and its per-minor slots kept a backlog. That backlog is the live bug. `rlang` is built for 4.4 and 4.5 on `amd64/resolute` but never for 4.6, so an R 4.6.1 client is served the generic 4.5.3 binary and dies with `undefined symbol: SETLENGTH`. 2709 records across the 16 slots are in that state. ## Why not weekly-rebuild-missing I tried that first (#182) and it is the wrong tool, for two independent reasons: - `weekly-missing-binaries-audit.R` reads only `/latest/src/contrib/PACKAGES.gz` and has no `r_minor` awareness, so its candidate list can only contain packages missing from the **generic** slot. - `rebuild-missing.R:73` says it outright: *"rebuild passes no `is_r_minor_sensitive`, so it only ever targets the flat"*. Running it under 4.6 built with the right interpreter and wrote to the wrong slot. It built almost nothing, and I verified it contaminated nothing: `amd64/resolute`'s flat slot is 22503 records at 4.5 and zero at 4.6. #182 should be closed. `build-all-versions` already runs `--sensitive-only`, documented as "the extra per-minor passes under non-primary R versions". It only needed the option. ## Change Adds `4.6.0` to `R_VERSION`. Default unchanged. ```sh crow pipeline create devxy/build-cran-binaries \ --var target_arch=amd64 --var OS=ubuntu --var OS_VERSION=resolute --var R_VERSION=4.6.0 ``` ## Follow-up worth doing separately The audit has no per-minor awareness, so this gap is invisible to every existing check and will silently reopen. Nothing measures per-minor completeness today except `scripts/verify-r-minor-routing.sh`, which was written for routing rather than coverage. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/183 --- .crow/build-all-versions.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index fb1229d..39a8075 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -36,7 +36,13 @@ variables: default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' + # The supported window is latest plus the two previous, which the build + # images install as R_VERSION_LATEST/PREV1/PREV2. 4.6.0 was missing here, + # so no pipeline could run the per-minor pass for it and its slots kept a + # backlog: rlang exists for 4.4 and 4.5 on amd64/resolute but not 4.6, + # which is how an R 4.6.1 client ended up loading a 4.5.3 binary. options: + - 4.6.0 - 4.5.3 - 4.4.3 default: 4.5.3 From 0a6c155dca7936cade21d3633d04bedcb77924f5 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 12:52:29 +0000 Subject: [PATCH 073/118] feat(cdn): enable per-R-minor routing in production (#184) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Everything built today is unreachable until this is set. ``` > install.packages("rlang") trying URL 'https://cran.rpkgs.com/amd64/resolute/latest/src/contrib/rlang_1.3.0.tar.gz' > library(rlang) undefined symbol: SETLENGTH ``` No `4.6/` in that path. With `UNION_SLOTS` empty the client resolves against the generic index and never reaches a per-minor binary: | artifact | size | |---|---| | generic, R 4.5-built | **2079570** — what R downloaded | | `4.6/`, R 4.6-built | 2075106 — correct, unused | The working binary has existed since 12:13 today. Nothing routes anyone to it. ## Change Sets production `UNION_SLOTS` to all 16 slots, from the same `local.rpkgs_slots` the staging zone uses. ## Verified before enabling Against the staging zone, which runs the identical script against the identical origin: | check | result | |---|---| | regressions against the generic slot | 0 across all 16 slots | | R minor served the per-minor index | 48/48 | | excluded R minor sent to CRAN | 16/16 | | client with no R minor still gets generic | 16/16 | | tarball never rewritten | 16/16 | ## Trade-off, stated plainly Coverage on a non-primary minor drops where the per-minor build backlog has not been worked off. `amd64/resolute` serves a 4.6 client 22169 packages rather than the generic slot's 24310. Those ~2100 are ABI-risky packages built under another R minor. They are exactly the ones that would install and then fail at load, so the drop trades a confusing runtime crash for an honest "not available". It shrinks as the 4.6 builds land. If that trade is unwelcome for some slots, `local.rpkgs_slots` can be narrowed to a subset — `amd64/rhel10` and `amd64/alpine323` have the smallest backlogs — and widened as builds catch up. ## After applying ```sh BASE=https://cran.rpkgs.com scripts/verify-r-minor-routing.sh --live ``` and the reported case directly: ```sh docker run --rm --platform linux/amd64 reg.devxy.io/r/r-ubuntu:4.6-resolute \ R -q -e 'install.packages("rlang"); library(rlang); cat("loaded OK\n")' ``` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/184 --- cdn.tf | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cdn.tf b/cdn.tf index 349486e..df30273 100644 --- a/cdn.tf +++ b/cdn.tf @@ -89,9 +89,19 @@ resource "bunnynet_compute_script" "rpkgs_router" { # slot that is not listed here would hide every package the per-minor index does # not carry, so this stays empty until a slot has been backfilled. resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { - script = bunnynet_compute_script.rpkgs_router.id - name = "UNION_SLOTS" - default_value = "" + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + # Enabled. Until this was set, every client resolved against the generic + # index and never reached a per-minor binary: an R 4.6.1 client on resolute + # downloaded the 4.5-built rlang (2079570 bytes) while the correct 4.6 build + # (2075106 bytes) sat unused one directory away, and died at load with + # `undefined symbol: SETLENGTH`. + # + # Verified before enabling, against the staging zone with the same script and + # the same origin: all 16 slots report zero regressions against the generic + # slot, an excluded R minor is sent to CRAN, a client without an R minor + # still gets the generic index, and tarball requests are never rewritten. + default_value = join(",", local.rpkgs_slots) required = false } From 4413f499f15b79d01803ebca63376a1073b7300d Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 13:01:31 +0000 Subject: [PATCH 074/118] test(verify): follow redirects, and allow the guard's deliberate drops (#185) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two checks that no longer matched the system. `fetch()` did not pass `-L`, so every `--live` check against a routed index read a 302 body rather than the index a client receives — 48 spurious failures against production. It also did not bypass the edge cache. Both were fixed on the branch behind #180, but that PR merged at `+9/-5`, capturing only the parity commit, so neither reached `main`. The union check asserted flat ⊆ every per-minor index. Since rpkgs/bincraft#116, that is deliberately false: `amd64/resolute` drops 2228 packages from its 4.6 index because their only binary was built under another R minor. Those absences **are** the fix working. It now asserts the thing that must hold — no generic package built under *this* minor may go missing — and reports the deliberate drops as context. Verified against production: `amd64/resolute` goes from 5 failures to 14 passed / 0 failed. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/185 --- scripts/verify-r-minor-routing.sh | 73 ++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh index 3957f67..190245a 100755 --- a/scripts/verify-r-minor-routing.sh +++ b/scripts/verify-r-minor-routing.sh @@ -104,10 +104,15 @@ fetch() { return 0 fi local status + # -L: the router answers an index request with a redirect, so the bytes a + # client ends up with are only visible by following it. + # + # no-cache: a purge is asynchronous, so a run started right after a reindex + # otherwise measures whatever the edge still holds. if [ -n "$ua" ]; then - status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + status=$(curl -sSL -A "$ua" -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) else - status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + status=$(curl -sSL -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) fi echo "$status" > "$dest.status" echo "$status" @@ -135,6 +140,23 @@ fallback_counts() { ' } +# Packages this index serves from the generic slot with a binary built under a +# different R minor, while some other per-minor slot carries a build of them - +# which proves the ABI classifier called them risky. Serving those is the +# load-time crash the per-minor slots exist to prevent. bincraft drops them at +# index time, so a non-zero count means the slot has not been reindexed since +# that guard shipped. +abi_unsafe_count() { + local minor_file=$1 minor=$2 risky_file=$3 + gunzip -c "$minor_file" 2>/dev/null | awk -v m="$minor" ' + /^Package:/ { pkg = $2; path = ""; built = "" } + /^Path:/ { path = $2 } + /^Built:/ { built = $2 " " $3 } + /^$/ { if (pkg != "" && path == "" && built != "" && built !~ ("^R " m "\\.")) print pkg; pkg = "" } + ' | sort -u > "$minor_file.mismatched" + comm -12 "$minor_file.mismatched" "$risky_file" | wc -l +} + # How many packages a client of would receive as source through # per-minor routing while the generic slot holds a binary built under that very # minor. Zero is the bar for enabling a slot. @@ -214,12 +236,25 @@ for arch in $ARCHES; do minor_count=$(wc -l < "$minor_file.names") # Union property: nothing the flat index carries may be missing here. - missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5) - missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l) - if [ "$missing_count" -ne 0 ]; then - bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))" + # bincraft deliberately drops an ABI-risky package whose only binary was + # built under another R minor: serving it is the load-time crash the + # per-minor slots exist to prevent. Those absences are correct. + # + # What must never go missing is a generic package built under *this* + # minor, which is safe to serve and has no reason to disappear. + comm -23 "$flat_file.names" "$minor_file.names" > "$minor_file.absent" + absent_count=$(wc -l < "$minor_file.absent") + gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" ' + /^Package:/ { pkg = $2; built = "" } + /^Built:/ { built = $2 " " $3 } + /^$/ { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" } + ' | sort -u > "$minor_file.flatsame" + lost=$(comm -12 "$minor_file.absent" "$minor_file.flatsame" | wc -l) + + if [ "${lost:-0}" -ne 0 ]; then + bad "$slot R $minor index dropped $lost generic package(s) built under R $minor, which were safe to serve" else - ok "$slot R $minor index: $minor_count packages, union holds" + ok "$slot R $minor index: $minor_count packages, union holds ($absent_count ABI-unsafe dropped)" fi # A per-minor entry that is a source fallback resolves fine but makes the @@ -309,6 +344,30 @@ for arch in $ARCHES; do fi fi + # Packages carrying a Path in any per-minor index are risky by construction. + : > "$WORK/${arch}-${distro}.risky" + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz" + [ -s "$f" ] || continue + gunzip -c "$f" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; path = "" } + /^Path:/ { path = $2 } + /^$/ { if (pkg != "" && path != "") print pkg; pkg = "" } + ' >> "$WORK/${arch}-${distro}.risky" + done + sort -u -o "$WORK/${arch}-${distro}.risky" "$WORK/${arch}-${distro}.risky" + + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz" + [ -s "$f" ] || continue + unsafe=$(abi_unsafe_count "$f" "$minor" "$WORK/${arch}-${distro}.risky") + if [ "${unsafe:-0}" -gt 0 ]; then + bad "$slot R $minor serves $unsafe ABI-risky package(s) built under another R minor - reindex this slot" + else + ok "$slot R $minor serves no ABI-risky package from another minor" + fi + done + # Excluded minors: no published index, and under --live a redirect to CRAN. for minor in $EXCLUDED_MINORS; do ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" From 448349d075bf7365f7ae5eb4f0e149d3092f4f20 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 13:29:03 +0000 Subject: [PATCH 075/118] revert(build): drop 4.6.0 as a primary R version option (#186) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #183 added `4.6.0` so the per-minor pass could run under 4.6. It was unnecessary, and it is a footgun. **Unnecessary:** `build-all-versions` already loops every installed interpreter and runs `local/build-all.R --sensitive-only` for each non-primary minor. The 4.6 pass happens when the pipeline runs with the slot's normal `R_VERSION=4.5.3`. I proposed #183 without reading that loop closely enough. **Footgun:** `R_VERSION` selects the *primary* minor, and the primary build lands in the **generic** slot. Selecting `4.6.0` for a slot whose generic binaries are 4.5-built would publish 4.6 binaries there and break every 4.5 client — the mirror image of the bug that started all this. The gaps are being filled by running the pipeline as it already stands (11928 on amd64/resolute). Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/186 --- .crow/build-all-versions.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 39a8075..cbbc70d 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -35,14 +35,16 @@ variables: - 'resolute' default: '3.24' R_VERSION: + # The slot's *primary* R minor: what `local/build-all.R` builds into the + # generic slot. The loop below already runs `--sensitive-only` for every + # other installed minor, so filling a non-primary minor's gap needs this + # left alone, not changed. + # + # 4.6.0 was briefly offered here (#183) and removed: selecting it for a + # slot whose generic binaries are 4.5-built would publish 4.6 binaries + # into the generic slot and break every 4.5 client. description: 'Primary R version under /opt/R.' - # The supported window is latest plus the two previous, which the build - # images install as R_VERSION_LATEST/PREV1/PREV2. 4.6.0 was missing here, - # so no pipeline could run the per-minor pass for it and its slots kept a - # backlog: rlang exists for 4.4 and 4.5 on amd64/resolute but not 4.6, - # which is how an R 4.6.1 client ended up loading a 4.5.3 binary. options: - - 4.6.0 - 4.5.3 - 4.4.3 default: 4.5.3 From 4bf88ed378f4cb7aa5093e456b294b0795da0745 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 13:34:33 +0000 Subject: [PATCH 076/118] fix(build): scope the already-attempted skip to the running R minor (#187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation The run meant to close the 4.6 gap on `amd64/resolute` barely built anything: ``` [1] "Skipped 2334 already-attempted package versions; 59 remaining for this job" ``` `single_builds` records `r_version` per attempt — `store_build_metadata()` both writes and queries it — but the skip query here ignored that column: ```sql SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2 ``` So a non-primary pass skipped every package the **primary** pass had already attempted under a different minor. `build-all.R --sensitive-only` running under R 4.6 skipped packages that had only ever been built for 4.5. That is the reason the per-minor slots never fill, and why the backlog cannot be worked off by rebuilding: `amd64/resolute` serves a 4.6 client 22322 packages against the 4.5 slot's 26346. It is also, ultimately, why an R 4.6.1 client got a 4.5-built `rlang` and `undefined symbol: SETLENGTH`. Every other fix in this chain addressed a consequence; this is the cause. ## Change Scope the skip to the R minor the pass is running under. Matched on the `major.minor` prefix rather than the full `r_version` string, so a patch bump (4.6.0 → 4.6.1) does not re-attempt the entire catalogue. Verified the prefix extraction against `4.5.3`, `4.6.0`, `4.4.3` and a bare `4.6`, and that the derivation matches what `store_build_metadata()` records. ## Expected effect The non-primary passes stop skipping wholesale. The first run per slot will be long, since it works off a backlog that has been accumulating for as long as the per-minor slots have existed. ## Verification - `local/build-all.R` parses. - Minor derivation checked under R 4.6.1: `4.6`. - Real effect is only observable from a run; the number to watch is the "Skipped N ... M remaining" line, which should show a far larger `M` for a non-primary pass. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/187 --- local/build-all.R | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/local/build-all.R b/local/build-all.R index 37fc593..18b0a52 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -110,10 +110,25 @@ con <- DBI::dbConnect( password = Sys.getenv("PGPASS"), sslmode = "require" ) +# Scope the skip to the R minor this pass is running under. `single_builds` +# records `r_version` per attempt, but querying without it made a non-primary +# pass skip everything the primary pass had already attempted under a different +# minor - so `--sensitive-only` under 4.6 skipped packages that had only ever +# been built for 4.5, and the per-minor slots never filled. That is why +# amd64/resolute served 4000 fewer packages to a 4.6 client than to a 4.5 one. +r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) built <- DBI::dbGetQuery( con, - "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2", - params = list(platform, arch) + paste( + "SELECT name, tag FROM single_builds", + "WHERE platform = $1 AND arch = $2", + "AND substring(r_version from '^[0-9]+[.][0-9]+') = $3" + ), + params = list(platform, arch, r_minor) ) DBI::dbDisconnect(con) before <- nrow(chunk) @@ -121,8 +136,9 @@ chunk <- chunk[ !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] sprintf( - "Skipped %d already-attempted package versions; %d remaining for this job", + "Skipped %d package versions already attempted under R %s; %d remaining for this job", before - nrow(chunk), + r_minor, nrow(chunk) ) From 213d30cea43f88a09366e7937349337d440a37d0 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 17:12:56 +0000 Subject: [PATCH 077/118] fix(build): hold back packages the cran mirror has not picked up yet (#188) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation A resolute build aborted on a single package: ``` [23/361] AsyPeer_0.0.1 (r_minor_sensitive=TRUE) Error: GitHub API error (404): Not Found x URL not found: Retrying in 2 seconds. ... Retrying in 60 seconds. Error in `rate_sleep()`: ! Request failed after 10 attempts. Execution halted ``` This is not rate limiting — it is a **404**. `check_for_binary()` reads the published version from the `cran` GitHub mirror, and that mirror lags CRAN. `AsyPeer 0.0.1` was published today at 13:50 UTC and has no repository there yet. The 404 is permanent, but the call is wrapped in `purrr::insistently` with `max_times = 10` and `pause_cap = 60`, so it retries on a 1/2/4/8/16/32/60/60/60/60 second backoff — about five minutes — and then aborts the whole shard. ## Change Hold back release versions published within `CRAN_MIRROR_LAG_DAYS` (default 3). Deferring them costs nothing: the daily update pipeline builds new and updated packages anyway, and they arrive here on the next run once the mirror has caught up. ## Measured against the live CRAN index | lag | held back | |---|---| | 1 day | 29 of 24831 (0.12%) | | **3 days** | **135 (0.54%)** | | 7 days | 412 (1.66%) | `AsyPeer` is among the 135 at three days. ## Worth doing separately Retrying a 404 at all is wrong — it can never succeed, and any other permanent 404 (a package pulled from the mirror, say) will abort a shard the same way. `check_for_binary()` should distinguish a permanent 404 from a transient failure and, when the mirror simply lacks the package, treat the version as unknown rather than fatal. That is a bincraft change and I have not made it here. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/188 --- local/packages-to-build.R | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 3b508fb..aa30390 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -68,9 +68,36 @@ archive_versions <- archive_versions[ ] # Now get release versions (assuming cran_release has Package and Version columns) +# +# Packages published in the last few days are held back. `check_for_binary()` +# reads the published version from the `cran` GitHub mirror +# (`GET /repos/cran//commits`), and that mirror lags CRAN: a package that +# has just appeared has no repository there yet. The call then 404s, which is +# permanent, but it is wrapped in `purrr::insistently` and retried ten times +# with a backoff capped at 60s - so one unmirrored package burns about five +# minutes and then aborts the whole shard. +# +# Holding them back costs nothing: the daily update pipeline builds new and +# updated packages anyway, and they arrive here on the next run once the mirror +# has caught up. +mirror_lag_days <- as.numeric( + Sys.getenv("CRAN_MIRROR_LAG_DAYS", unset = "3") +) +published <- as.POSIXct(cran_release$Published, tz = "UTC") +too_recent <- !is.na(published) & + published > (Sys.time() - mirror_lag_days * 86400) +if (any(too_recent)) { + message(sprintf( + "Holding back %d package(s) published in the last %g day(s); the cran GitHub mirror will not have them yet: %s", + sum(too_recent), + mirror_lag_days, + paste(utils::head(cran_release$Package[too_recent], 10L), collapse = ", ") + )) +} + release_versions <- data.table( - Package = cran_release$Package, - Version = as.character(cran_release$Version) + Package = cran_release$Package[!too_recent], + Version = as.character(cran_release$Version[!too_recent]) ) pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE)) From 94e6c697cf831965de5464391c54db571901ba0f Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 21:36:11 +0000 Subject: [PATCH 078/118] fix(build): stop per-minor objects masking the per-minor candidate list (#189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation `arm64/alpine324` (pipeline 11953) finished in minutes having uploaded 57 packages, and reported: ``` Skipped 0 package versions already attempted under R 4.4; 0 remaining Skipped 0 package versions already attempted under R 4.6; 3 remaining ``` The same run's index step dropped **2407** packages as missing for 4.4 and **2436** for 4.6. Nothing to build, and thousands missing — the candidate list is wrong. ## Two omissions **1. Per-minor objects mask the per-minor candidates.** ```r s3_pkgs <- s3fs::s3_dir_ls(".../latest/src/contrib", recurse = TRUE) file_names <- basename(s3_pkgs) ``` `recurse = TRUE` walks `4.4/`, `4.5/`, `4.6/`; `basename()` throws the directory away. `4.5/curl_1.0.tar.gz` and `curl_1.0.tar.gz` collapse to one name, so a package present under **any** R minor counts as built for **all** of them — pruning exactly the packages a per-minor pass exists to build. Per-minor objects are now excluded, and presence in a specific minor is decided downstream where the running R version is known: `build-all.R` filters on it, and `build_binary_package()` checks the per-minor path per package and skips what is already there. `Archive/` is kept. Those are versions built and later superseded; dropping them would make every archived version look unbuilt. Validated against real path shapes: | path | | |---|---| | `curl_1.0.tar.gz` | keep | | `4.4/curl_1.0.tar.gz` | exclude | | `4.6/rlang_1.3.0.tar.gz` | exclude | | `Archive/curl/curl_0.9.tar.gz` | keep | | `PACKAGES.gz` | keep | **2. The error query ignores `r_version`.** ```sql SELECT error_occurred FROM single_builds WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4 ``` A failure under the primary minor drops the package from every other minor's candidate list. This is the same omission fixed in `local/build-all.R` (#187) and in bincraft's `check_package_error()` (rpkgs/bincraft#119). This is the third and last consumer of that table — I have grepped the rest; `bincraft::R/cran-internal.R` also reads it, but to list packages present rather than to skip, where the R minor does not apply. ## Expected effect The per-minor passes get real candidate lists. Expect slots that reported "0 remaining" to report thousands, and correspondingly long runs. There is a cost: the list is no longer pruned by per-minor presence, so each pass asks `build_binary_package()` about packages that may already exist, and it answers `already exists in S3 ... Skipping build` per package. Slower per pass, and correct — the pruning it replaces was removing the wrong things. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/189 --- local/packages-to-build.R | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index aa30390..9cc1ed6 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -116,7 +116,28 @@ s3_pkgs <- s3fs::s3_dir_ls( recurse = TRUE ) -file_names <- basename(s3_pkgs) +# `recurse = TRUE` walks the per-minor slots as well, and `basename()` throws +# the directory away - so `4.5/curl_1.0.tar.gz` and `curl_1.0.tar.gz` collapse +# to one name and a package present under *any* R minor counts as built for +# *all* of them. The candidate list then prunes exactly the packages a +# per-minor pass exists to build: arm64/alpine324 reported "0 remaining" for +# both 4.4 and 4.6 while its indexes were dropping 2400+ packages as missing. +# +# Per-minor objects are therefore excluded here. Presence in a specific minor +# is decided downstream, where the running R version is known: build-all.R +# filters on it, and `build_binary_package()` checks the per-minor path per +# package and skips what is already there. +# +# Archive/ is kept. Those are versions that were built and then superseded; +# dropping them would make every archived version look unbuilt. +per_minor_object <- grepl("/[0-9]+\\.[0-9]+/[^/]+$", s3_pkgs) +if (any(per_minor_object)) { + cat(sprintf( + "Excluding %d per-minor object(s) from the presence check; those are decided per pass\n", + sum(per_minor_object) + )) +} +file_names <- basename(s3_pkgs[!per_minor_object]) # An object occupying a key is not proof a binary was built: a package whose # build failed has its CRAN source published under exactly that name. Left in @@ -179,11 +200,22 @@ s3_dt <- data.table( ### Get all packages with build errors +# Scoped to the R minor this snapshot is computed under. A failure is a fact +# about one interpreter: without the scope a package that failed under the +# primary minor is dropped from the candidate list for every other minor too, +# which is the same omission fixed in local/build-all.R and in bincraft's +# check_package_error(). +snapshot_r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) sql_query <- paste0( # nolint "SELECT error_occurred FROM ", "single_builds", - " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4" + " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4", + " AND substring(r_version from '^[0-9]+[.][0-9]+') = $5" ) # Function to query for a single package-version query_error <- function(pkg, ver) { @@ -191,7 +223,7 @@ query_error <- function(pkg, ver) { ~ DBI::dbGetQuery( con, sql_query, - params = list(pkg, ver, platform, arch) + params = list(pkg, ver, platform, arch, snapshot_r_minor) ), rate = purrr::rate_backoff( pause_base = 1L, From 642e07e1d6b327e9ecb435c06100786ecf0ce060 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 31 Aug 2026 22:11:17 +0000 Subject: [PATCH 079/118] fix(build): recompute a stale package snapshot, not just a missing one (#190) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation `arm64/alpine324` reported nothing to build while thousands were missing: ``` line 49: Precomputed 7192 package versions (6871 r-minor-sensitive) <- install-deps agent line 99: Total# of remaining package versions: 43 (sensitive_only=TRUE) <- a build shard line 101: Skipped 0 package versions already attempted under R 4.4; 0 remaining ``` Both numbers come from the **same pipeline**. The same run's index step dropped 2407 packages as missing for 4.4 and 2436 for 4.6. ## Cause ```r if (!all(file.exists(package_cache_files))) { ... recompute ... } ``` Existence is not freshness. The snapshot describes S3 and CRAN state when it was written, and the cache volume is per-agent — the file's own comment says so. An agent that ran an earlier pipeline keeps serving that pipeline's answer forever, and no later fix to how the snapshot is computed (#189) can reach it. ## Change Recompute when the snapshot is stale as well as when it is missing. Keyed on the pipeline when the CI exposes an identifier (`CI_PIPELINE_NUMBER`, `CI_BUILD_NUMBER`, `CI_PIPELINE_ID`), so a new pipeline recomputes once per agent and its shards then share the result. Off CI, or when none is set, an age check with a two hour default (`PACKAGE_SNAPSHOT_TTL_HOURS`). ## Verification | scenario | decision | |---|---| | files missing | RECOMPUTE | | same pipeline id | reuse | | **new pipeline id** | **RECOMPUTE** | | no CI var, recent file | reuse | | no CI var, aged out | RECOMPUTE | I could not confirm which identifier Crow actually sets — none is referenced anywhere in this repo — so all three are tried and the age check backs them up. If none is present the behaviour is the age path, which is still correct, just coarser. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/190 --- local/build-all.R | 48 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/local/build-all.R b/local/build-all.R index 18b0a52..c4d7f97 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -26,9 +26,50 @@ package_cache_files <- c( "/mnt/cache/packages/r_minor_sensitive_pkgs.rds", "/mnt/cache/packages/s3_cache.rds" ) -if (!all(file.exists(package_cache_files))) { +# Existence is not freshness. The snapshot describes S3 and CRAN state at the +# moment it was written, and the volume is per-agent, so an agent that ran an +# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324 +# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43 +# sensitive packages, while the install-deps step in the very same pipeline had +# just computed 6871 on another agent. +# +# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes +# once per agent and its shards then share the result. Off CI, or when no such +# variable is set, fall back to an age check. +snapshot_id_path <- "/mnt/cache/packages/snapshot.id" +snapshot_ttl_hours <- as.numeric( + Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2") +) +current_snapshot_id <- "" +for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) { + val <- Sys.getenv(v, unset = "") + if (nzchar(val)) { + current_snapshot_id <- paste(v, val, sep = "=") + break + } +} + +snapshot_is_stale <- function() { + if (!all(file.exists(package_cache_files))) { + return(TRUE) + } + if (nzchar(current_snapshot_id)) { + cached <- tryCatch( + readLines(snapshot_id_path, warn = FALSE)[1L], + error = function(e) NA_character_, + warning = function(w) NA_character_ + ) + return(!identical(cached, current_snapshot_id)) + } + age_hours <- as.numeric( + difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours") + ) + isTRUE(age_hours > snapshot_ttl_hours) +} + +if (snapshot_is_stale()) { message( - "Package snapshot missing from cache; recomputing via packages-to-build.R" + "Package snapshot missing or stale; recomputing via packages-to-build.R" ) dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { @@ -42,6 +83,9 @@ if (!all(file.exists(package_cache_files))) { pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" ) + if (nzchar(current_snapshot_id)) { + writeLines(current_snapshot_id, snapshot_id_path) + } message("Package snapshot recomputed.") } From 8f97cf99ef490642b62509d0d9f30de339accc86 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 1 Sep 2026 21:54:13 +0000 Subject: [PATCH 080/118] fix(build): give the existence cache a per-minor view of the slot (#191) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation A restarted run recompiled everything it had already built. ``` attempted: 8683 already exists in S3: 3 actually compiled: 8683 ``` `amd64/resolute` had uploaded roughly 3000 binaries before being restarted, and rebuilt all of them. ## Cause This is a regression from #189, which I introduced. `s3_cache.rds` is derived from the same `file_names` that #189 filtered: ```r binary_cache <- setdiff(file_names, source_served) saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") ``` Removing per-minor objects from `file_names` was right for the candidate list and wrong for the existence cache. `build_binary_package()` was handed a cache containing no per-minor object at all, so its "not present in the remote bucket" check answered not-present for every one of them. The two consumers need opposite views of one listing: - the **candidate list** must ignore per-minor objects, or a package present under one minor prunes itself from every other minor's work; - the **existence cache** must not ignore them, or the work is redone. Conflating them was wrong in both directions: before #189 a per-minor pass believed the flat slot's binaries were its own and built nothing; after #189 it believed it had nothing and rebuilt the lot. ## Change The cache keeps paths relative to the slot, and `build-all.R` selects the part matching the pass it is running. `build_binary_package()` compares basenames and cannot distinguish `4.4/curl_1.0.tar.gz` from `curl_1.0.tar.gz`, so the choice has to be made where the running R minor is known. Verified: | pass | selects | |---|---| | primary (writes flat) | `curl_1.0`, `jsonlite_2.0`, `Archive/curl_0.9`, `PACKAGES.gz` | | `--sensitive-only` under 4.4 | `curl_1.0`, `rlang_1.3.0` (from `4.4/` only) | | `--sensitive-only` under 4.6 | `rlang_1.3.0` (from `4.6/` only) | | legacy basename cache | passthrough, unfiltered | `Archive/` counts as present for the primary pass: those are versions built and later superseded. ## Backwards compatibility A cache written before this change holds bare basenames. Filtering those by path would select nothing and trigger the very rebuild this prevents, so they are detected and used unfiltered; #190's staleness check replaces them on the next pipeline. ## Note on the running builds The five pipelines currently running are recompiling packages they already have. Their output is still correct — the binaries carry the right `Built` stamp for their slot — but the work is wasted. They should be restarted once this is merged. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/191 --- local/build-all.R | 38 ++++++++++++++++++++++++++++++++++++-- local/packages-to-build.R | 24 +++++++++++++++++++++++- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/local/build-all.R b/local/build-all.R index c4d7f97..eda92e9 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -189,8 +189,42 @@ sprintf( # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, # saving significant memory for the dependency-installer subprocesses -s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") -sprintf("S3 cache: %s files", length(s3_cache)) +s3_cache_paths <- readRDS("/mnt/cache/packages/s3_cache.rds") + +# The cache is stored as paths relative to the slot, so a pass can select the +# objects that belong to it. `build_binary_package()` compares basenames, which +# cannot distinguish `4.4/curl_1.0.tar.gz` from `curl_1.0.tar.gz`, so the choice +# has to be made here where the running R minor is known. +# +# Getting this wrong is expensive in both directions: hand it everything and a +# per-minor pass believes the flat slot's binaries are its own and builds +# nothing; hand it nothing and it rebuilds what it already has. amd64/resolute +# recompiled 8683 packages that way. +select_cache_for_pass <- function(paths, sensitive_only, r_minor) { + # A cache written before this change holds bare basenames. Filtering those by + # path would select nothing and trigger a full rebuild, so use them as they + # are; #190's staleness check replaces it on the next pipeline anyway. + if (!any(grepl("/", paths, fixed = TRUE))) { + message("S3 cache is in the legacy basename format; using it unfiltered.") + return(paths) + } + in_minor <- grepl(sprintf("^%s/", r_minor), paths) + if (sensitive_only) { + basename(paths[in_minor]) + } else { + # The primary pass writes the flat slot. Archive/ counts as present there: + # those are versions built and later superseded. + basename(paths[!grepl("^[0-9]+\\.[0-9]+/", paths)]) + } +} + +s3_cache <- select_cache_for_pass(s3_cache_paths, sensitive_only, r_minor) +sprintf( + "S3 cache: %s files (%s of %s objects apply to this pass)", + length(s3_cache), + length(s3_cache), + length(s3_cache_paths) +) n <- nrow(chunk) mapply( diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 9cc1ed6..64a2f5f 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -176,6 +176,28 @@ source_served <- tryCatch( ) binary_cache <- setdiff(file_names, source_served) + +# The existence cache and the candidate list need different views of the same +# listing, and conflating them is what made this wrong in both directions. +# +# The candidate list must ignore per-minor objects, or a package present under +# one minor prunes itself from every other minor's work (#189). The existence +# cache must NOT ignore them, or `build_binary_package()` is told nothing is +# present under any minor and recompiles the lot: amd64/resolute recompiled +# 8683 packages it had already built, reporting "already exists in S3" three +# times. +# +# So the cache keeps the path relative to the slot, and `build-all.R` selects +# the part that matches the pass it is running: the flat slot for the primary, +# `/` for a per-minor pass. +contrib_prefix <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib/", + arch, + codename +) +relative_paths <- sub(contrib_prefix, "", s3_pkgs, fixed = TRUE) +source_basenames <- source_served +existence_cache <- relative_paths[!basename(relative_paths) %in% source_basenames] cat(sprintf( "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", length(file_names), @@ -186,7 +208,7 @@ cat(sprintf( # Save the S3 file listing for the build step to use as s3_package_cache. # This avoids loading s3fs/reticulate in the build container, saving memory for # the dependency-installer subprocesses -saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") +saveRDS(existence_cache, "/mnt/cache/packages/s3_cache.rds") # Built from the filtered listing, not the raw one: `s3_dt` is subtracted from # the build list below, so a source fallback left in here would exclude the very # package that needs building. From d4f093923ae3a041f5463e98d18cc7e38ab4753e Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 2 Sep 2026 07:11:10 +0000 Subject: [PATCH 081/118] fix(build): strip the s3 scheme so per-minor objects reach their pass (#192) ## Why #191 gave the existence cache a per-minor view of the slot, but the paths it filters never carried a minor prefix, so the fix could not take effect. `s3fs::s3_dir_ls()` returns keys with the `s3://` scheme attached: ``` s3://devxy-rpkgs-binaries/amd64/resolute/latest/src/contrib/4.4/oeli_0.7.6.tar.gz ``` The contrib prefix was stripped as a *fixed substring*, so it was removed from the middle of the key and the scheme survived: ``` s3://4.4/oeli_0.7.6.tar.gz ``` That leading `s3://` defeats both `^/` and `^[0-9]+\.[0-9]+/`, so all 21212 per-minor objects on amd64/resolute were classified as flat-slot objects. Pipeline 12011 shows it exactly: ``` sensitive pass: 0 files (0 of 119053 objects apply to this pass) primary pass: 119053 files (119053 of 119053 objects apply to this pass) ``` Each sensitive pass therefore ran with an empty cache and recompiled all 10248 sensitive packages it already had. ## What changed - `local/packages-to-build.R`: anchor the contrib prefix and swallow an optional `s3://` with it. - `local/packages-to-build.R`: abort when the strip leaves fewer per-minor paths than the listing held. The failure mode is silent and only surfaces as a multi-hour rebuild, and both counts derive from the same listing so they must agree exactly. - `local/packages-to-build.R` / `local/build-all.R`: mark the cache with a `slot_relative` attribute and read that, instead of sniffing for a `/`. A slot-relative cache for a slot with no per-minor or Archive object holds bare names too, and would have been misread as legacy and used unfiltered, which makes a per-minor pass believe the flat slot's binaries are its own and build nothing. - `scripts/purge_cdn_zone.sh`: indent the jq continuation lines by a multiple of two. This is unrelated, but it fails editorconfig-checker on `main` and blocks `prek run -a` for everyone. jq ignores the whitespace, and both response shapes still resolve. ## Verification Replaying the real key shapes through the old and new code: ``` sensitive(4.6) primary OLD paths: 0 8 <- reproduces production NEW paths: 3 3 Invariant NEW: listing=4 stripped=4 -> PASS Invariant OLD: listing=4 stripped=0 -> ABORT (would have caught this) ``` Per-minor `Archive/` objects are attributed to their minor, and the flat bucket keeps its own `Archive/`. `prek run -a` passes. Pipelines 12011-12015 were stopped rather than left to spend hours recompiling what they already had. Their uploads are not lost, so a fresh run inherits them. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/192 --- local/build-all.R | 13 ++++++++---- local/packages-to-build.R | 42 ++++++++++++++++++++++++++++++++++++--- scripts/purge_cdn_zone.sh | 4 ++-- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/local/build-all.R b/local/build-all.R index eda92e9..349092f 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -201,10 +201,15 @@ s3_cache_paths <- readRDS("/mnt/cache/packages/s3_cache.rds") # nothing; hand it nothing and it rebuilds what it already has. amd64/resolute # recompiled 8683 packages that way. select_cache_for_pass <- function(paths, sensitive_only, r_minor) { - # A cache written before this change holds bare basenames. Filtering those by - # path would select nothing and trigger a full rebuild, so use them as they - # are; #190's staleness check replaces it on the next pipeline anyway. - if (!any(grepl("/", paths, fixed = TRUE))) { + # A cache written before #191 holds bare basenames. Filtering those by path + # would select nothing and trigger a full rebuild, so use them as they are; + # #190's staleness check replaces it on the next pipeline anyway. + # + # The format is read from the marker `packages-to-build.R` sets, not guessed + # from the content: a slot-relative cache with no per-minor or Archive object + # holds bare names too, and treating that as legacy would hand a per-minor + # pass the flat slot's binaries and build nothing. + if (!isTRUE(attr(paths, "slot_relative"))) { message("S3 cache is in the legacy basename format; using it unfiltered.") return(paths) } diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 64a2f5f..9e17f4d 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -190,14 +190,44 @@ binary_cache <- setdiff(file_names, source_served) # So the cache keeps the path relative to the slot, and `build-all.R` selects # the part that matches the pass it is running: the flat slot for the primary, # `/` for a per-minor pass. +# +# `s3_dir_ls()` returns keys with the `s3://` scheme attached, so stripping the +# prefix as a fixed substring takes it out of the middle and leaves +# `s3://4.4/curl_1.0.tar.gz`. That leading scheme defeats the `^/` test +# downstream, so every per-minor object is read as a flat-slot object: the +# sensitive passes see an empty cache and recompile everything they already +# have. Anchor the pattern and swallow the scheme with it. contrib_prefix <- sprintf( - "devxy-rpkgs-binaries/%s/%s/latest/src/contrib/", + "^(s3://)?devxy-rpkgs-binaries/%s/%s/latest/src/contrib/", arch, codename ) -relative_paths <- sub(contrib_prefix, "", s3_pkgs, fixed = TRUE) +relative_paths <- sub(contrib_prefix, "", s3_pkgs) + +# The strip is load-bearing and fails silently, so assert it. Both counts are +# derived from the same listing and use the same shape of pattern, so they must +# agree exactly; a mismatch means the prefix no longer describes the keys. +stripped_per_minor <- grepl("^[0-9]+\\.[0-9]+/[^/]+$", relative_paths) +if (sum(stripped_per_minor) != sum(per_minor_object)) { + stop(sprintf( + paste0( + "S3 prefix strip failed: %d per-minor objects in the listing, %d after ", + "stripping /%s/. Example key: %s" + ), + sum(per_minor_object), + sum(stripped_per_minor), + contrib_prefix, + if (any(per_minor_object)) { + s3_pkgs[which(per_minor_object)[1L]] + } else { + "" + } + )) +} source_basenames <- source_served -existence_cache <- relative_paths[!basename(relative_paths) %in% source_basenames] +existence_cache <- relative_paths[ + !basename(relative_paths) %in% source_basenames +] cat(sprintf( "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", length(file_names), @@ -208,6 +238,12 @@ cat(sprintf( # Save the S3 file listing for the build step to use as s3_package_cache. # This avoids loading s3fs/reticulate in the build container, saving memory for # the dependency-installer subprocesses +# Mark the format explicitly. `build-all.R` has to tell a slot-relative cache +# from a pre-#191 basename one, and sniffing for a "/" cannot: a new-format +# cache for a slot with no per-minor or Archive objects holds bare names too, +# and would be read as legacy and used unfiltered, which makes a per-minor pass +# believe the flat slot's binaries are its own and build nothing. +attr(existence_cache, "slot_relative") <- TRUE saveRDS(existence_cache, "/mnt/cache/packages/s3_cache.rds") # Built from the filtered listing, not the raw one: `s3_dt` is subtracted from # the build list below, so a source fallback left in here would exclude the very diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 6c07eef..8e85cb6 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -70,8 +70,8 @@ resolve_zone_id() { zone_id=$( jq -r --arg hostname "${zone}" \ '(if type == "object" then (.Items // []) else . end)[] - | select(any(.Hostnames[]?; .Value == $hostname)) - | .Id' \ + | select(any(.Hostnames[]?; .Value == $hostname)) + | .Id' \ "${response_file}" ) rm -f "${response_file}" From 7e0f3d0ac614c7d6cfdb9b3c5e18ddf6523f34d8 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 2 Sep 2026 09:13:54 +0000 Subject: [PATCH 082/118] fix(build): judge each index by its own source fallbacks (#193) ## Why #192 made the per-minor objects visible to their pass, and that part works: every slot now reports a populated per-minor cache where it previously reported `0 of 119053`. But the cache-driven skip rate stayed near **zero**, and the builds kept recompiling packages that already exist. Sampling what pipeline 12027 called missing, against the live CDN: ``` orthoDr_0.6.7.tar.gz 4.4/=404 flat=404 <- genuinely missing osmdata_0.2.4.tar.gz 4.4/=200 flat=404 <- already built, rebuilt anyway osmdata_0.2.5.tar.gz 4.4/=200 flat=404 osmdata_0.3.0.tar.gz 4.4/=200 flat=404 osqp_0.6.3.3.tar.gz 4.4/=200 flat=404 outbreaker2_1.1.0.tar.gz 4.4/=200 flat=404 ``` `source_served` is read from the **flat** slot's `PACKAGES.gz` and lists the packages that index serves as CRAN source instead of a binary. It was applied to every path in the listing: ```r existence_cache <- relative_paths[!basename(relative_paths) %in% source_basenames] ``` A package the flat slot serves as source still has a real `4.4/.tar.gz`, so dropping that basename deleted a genuine binary from the cache. The overlap with the work list is total rather than partial, which is why the skip rate was ~0% and not merely low: the sensitive candidate list is exactly "sensitive packages with no flat binary", which is the same set this was removing. amd64/resolute listed 21212 per-minor objects and cached only 13572; the missing ~7600 are these. ## What changed - Read each per-minor `PACKAGES.gz` and exclude only that minor's own source fallbacks. - Keep judging flat-slot objects by the flat index. - Log the fallback count per index, so a future collapse is visible rather than silent. All per-minor indexes were confirmed reachable (`4.4`, `4.5`, `4.6` return 200 on the slots checked); an unreadable index falls back to keeping that scope's cache in full, which is the safe direction. ## Verification ``` 4.4 pass sees -> OLD: osqp_0.6.3.3 NEW: osmdata_0.2.4, osqp_0.6.3.3 all assertions PASS ``` Correct in both directions: the old code dropped `4.4/osmdata` (a real binary) *and* kept `4.6/broken` (a real source fallback in the 4.6 index). The new code restores the first and still excludes the second. `prek run -a` passes. Pipelines 12025-12029 were stopped rather than left recompiling what they already have; their uploads persist, so a fresh run inherits them. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/193 --- local/packages-to-build.R | 106 +++++++++++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 23 deletions(-) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 9e17f4d..0654a43 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -154,26 +154,30 @@ index_url <- sprintf( arch, codename ) -source_served <- tryCatch( - { - con_idx <- gzcon(url(index_url, open = "rb")) - on.exit(close(con_idx), add = TRUE) - idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) - sprintf( - "%s_%s.tar.gz", - idx[is.na(idx[, "Built"]), "Package"], - idx[is.na(idx[, "Built"]), "Version"] - ) - }, - error = function(e) { - cat(sprintf( - "WARNING: could not read %s (%s); keeping the full S3 cache\n", - index_url, - conditionMessage(e) - )) - character(0) - } -) +read_source_served <- function(index) { + tryCatch( + { + con_idx <- gzcon(url(index, open = "rb")) + on.exit(close(con_idx), add = TRUE) + idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) + sprintf( + "%s_%s.tar.gz", + idx[is.na(idx[, "Built"]), "Package"], + idx[is.na(idx[, "Built"]), "Version"] + ) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read %s (%s); keeping that index's cache in full\n", + index, + conditionMessage(e) + )) + character(0) + } + ) +} + +source_served <- read_source_served(index_url) binary_cache <- setdiff(file_names, source_served) @@ -225,9 +229,65 @@ if (sum(stripped_per_minor) != sum(per_minor_object)) { )) } source_basenames <- source_served -existence_cache <- relative_paths[ - !basename(relative_paths) %in% source_basenames -] + +# A source-fallback list describes ONE index, so it may only be applied to that +# index's objects. Applied to every path it deletes real per-minor binaries from +# the cache: a package the flat slot serves as CRAN source still has a genuine +# `4.4/.tar.gz`, and dropping that name makes every per-minor pass rebuild +# it, every run, forever. +# +# The overlap is total rather than partial, which is why this pinned the skip +# rate near 0%: the sensitive candidate list is exactly "sensitive packages with +# no flat binary", which is the same set this was removing. amd64/resolute +# listed 21212 per-minor objects but cached only 13572, then recompiled osmdata, +# osqp and outbreaker2 while their 4.4 binaries sat in the bucket. +# +# So judge each minor by its own index, and the flat slot by the flat index. +is_per_minor_path <- grepl("^[0-9]+\\.[0-9]+/", relative_paths) +path_minor <- ifelse( + is_per_minor_path, + sub("^([0-9]+\\.[0-9]+)/.*$", "\\1", relative_paths), + "" +) +minors_present <- sort(unique(path_minor[is_per_minor_path])) +per_minor_source <- lapply(minors_present, function(m) { + read_source_served(sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/%s/PACKAGES.gz", + arch, + codename, + m + )) +}) +names(per_minor_source) <- minors_present +cat(sprintf( + "Source fallbacks per index: flat=%d%s\n", + length(source_basenames), + if (length(minors_present)) { + paste0( + ", ", + paste( + sprintf("%s=%d", minors_present, lengths(per_minor_source)), + collapse = ", " + ) + ) + } else { + "" + } +)) + +served_as_source <- vapply( + seq_along(relative_paths), + function(i) { + scope <- if (nzchar(path_minor[i])) { + per_minor_source[[path_minor[i]]] + } else { + source_basenames + } + basename(relative_paths[i]) %in% scope + }, + logical(1L) +) +existence_cache <- relative_paths[!served_as_source] cat(sprintf( "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", length(file_names), From a650f1d18a2991b72ab63eab5a75b337a40e5cbd Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 3 Sep 2026 13:37:04 +0000 Subject: [PATCH 083/118] fix(build): scope the errored-package filter to the snapshot's R minor (#194) ## Why After #192 and #193 the builds stopped recompiling what they already had, but every slot still showed a per-minor gap that a successful run did not close: ``` arm64/resolute 3302 amd64/resolute 1783 amd64/alpine324 1163 arm64/alpine324 1104 arm64/rhel8 572 amd64/rhel8 532 amd64/rhel10 453 arm64/rhel10 176 (others <= 13) ``` amd64/alpine324 had just completed successfully, yet 655 ABI-risky packages were still absent from its 4.6 index. Sampling 25 of them against that run's log: ``` attempted: 0 errored: 0 uploaded: 0 never attempted: 25 ``` They were filtered out before the build loop. The candidate list drops every package with an error row for this platform/arch: ```sql SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s' ``` There is no `r_version` in that predicate, and the row is keyed by `(name, tag, platform, arch)` only. So a single failure under one minor cancels the package's builds under every other minor, permanently. The index data matches that shape rather than a build-failure shape: of the 508 ABI-risky packages missing from alpine324's 4.4 index, **505 are also missing from 4.6** - a 99% overlap. Genuine per-minor failures would not correlate that way (only 3 were 4.4-specific). ## What changed Adds the `substring(r_version from '^[0-9]+[.][0-9]+') = ''` scoping already used by `check_package_error()` (#119) and the per-minor error query (#189). Errors specific to another minor are still respected downstream: `filter_packages_with_errors()` runs per pass with that pass's own `r_version`, so a package that genuinely fails under 4.6 is still skipped there. ## Verification Rendered query: ```sql SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='alpine-324' and arch='amd64' and substring(r_version from '^[0-9]+[.][0-9]+') = '4.5' ``` Routing itself is unaffected and remains green: `verify-r-minor-routing.sh` reports 232 passed / 0 failed across all 16 slots, and `--live` reported 328 passed / 0 failed. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/194 --- local/packages-to-build.R | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 0654a43..ead73b8 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -355,12 +355,30 @@ query_error <- function(pkg, ver) { } # Fetch all relevant columns from the database +# Scoped to the snapshot's own R minor, like `check_package_error()` in +# bincraft (#119) and the per-minor error query (#189). Unscoped, a single +# failure under any one minor removed the package from the candidate list for +# every minor, permanently: the row is keyed by (name, tag, platform, arch) +# only, so an error under 4.6 also cancelled its 4.4 and 4.5 builds. +# +# That is what the residual per-minor gaps were. On amd64/alpine324, 505 of the +# 508 ABI-risky packages missing from 4.4 were also missing from 4.6 -- a 99% +# overlap, which per-minor build failures would not produce. None of them were +# even attempted by the run that had just succeeded. +# +# Errors specific to another minor are still respected downstream: +# `filter_packages_with_errors()` runs per pass with that pass's r_version. errored_pkgs <- DBI::dbGetQuery( con, sprintf( - "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", + paste0( + "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE ", + "and platform='%s' and arch='%s' ", + "and substring(r_version from '^[0-9]+[.][0-9]+') = '%s'" + ), platform, - arch + arch, + snapshot_r_minor ) ) errored_pkgs <- as.data.table(errored_pkgs) From 2c9bd598fb6baf231798a62acf0e3739811a5a67 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 3 Sep 2026 20:09:20 +0000 Subject: [PATCH 084/118] fix(ci): install RPostgres in build shards (#195) Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/195 --- .crow/build-all-versions.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index cbbc70d..4306d6d 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -150,6 +150,10 @@ steps: # install). Pin bincraft here, exactly like the R-minor pass below. - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + # The dependency-prep job may run on another agent whose cache volume is + # not visible here. Install the packages needed to recompute the snapshot + # and query build metadata in every build job. + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -162,6 +166,7 @@ steps: LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true + R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages; first arg is the codename (e.g. "alpine324"), From e4ba623ddc34368f964b610f65b249cd1d15ffe7 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Fri, 4 Sep 2026 00:32:36 +0000 Subject: [PATCH 085/118] chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker to v3.11.3 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2bf38b9..ddaa92b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.11.2 + rev: v3.11.3 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ From d3f24c89dfbdc03a202d2d6a426577b057f5b855 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Fri, 4 Sep 2026 09:51:46 +0000 Subject: [PATCH 086/118] chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker to v4 (#197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [editorconfig-checker/editorconfig-checker](https://github.com/editorconfig-checker/editorconfig-checker) | repository | major | `v3.11.3` → `v4.0.1` | Note: The `pre-commit` manager in Renovate is not supported by the `pre-commit` maintainers or community. Please do not report any problems there, instead [create a Discussion in the Renovate repository](https://github.com/renovatebot/renovate/discussions/new) if you have any questions. --- ### Release Notes
editorconfig-checker/editorconfig-checker (editorconfig-checker/editorconfig-checker) ### [`v4.0.1`](https://github.com/editorconfig-checker/editorconfig-checker/releases/tag/v4.0.1) [Compare Source](https://github.com/editorconfig-checker/editorconfig-checker/compare/v4.0.0...v4.0.1) ##### Bug Fixes - bump golang.org/x/mod to v0.40.0 ([3d2215c](https://github.com/editorconfig-checker/editorconfig-checker/commit/3d2215c10a8495b6eee06863d1582685431a46e8)) - **ci:** build docker images for all our platforms ([a34a6ee](https://github.com/editorconfig-checker/editorconfig-checker/commit/a34a6eeac71bea7a428d4f6333a1683633750c3a)) - update pre-commit-config to use v4 of our repo ([fcd68bf](https://github.com/editorconfig-checker/editorconfig-checker/commit/fcd68bffdacddb356f8c9f7cbfb838744302d17d)) ### [`v4.0.0`](https://github.com/editorconfig-checker/editorconfig-checker/releases/tag/v4.0.0) [Compare Source](https://github.com/editorconfig-checker/editorconfig-checker/compare/v3.11.3...v4.0.0) ##### ⚠ BREAKING CHANGES - internalize our packages ([#​619](https://github.com/editorconfig-checker/editorconfig-checker/issues/619))
It will no longer be possible to import our internal packages in your code. If you rely on one of our packages please let us know by opening an issue. - migrate to goreleaser dockers\_v2. ([#​617](https://github.com/editorconfig-checker/editorconfig-checker/issues/617))
This means we stop publishing the arch-suffixed images like `3.11.2-amd64` or `3.11.2-arm64v8`. All these architectures are still provided in the multiarch tag `3.11.2`. - rename ec to editorconfig-checker ([#​371](https://github.com/editorconfig-checker/editorconfig-checker/issues/371)) - `.ecrc` is no longer discovered as a default config file name. Rename `.ecrc` to `.editorconfig-checker.json`, or pass the file explicitly with `--config .ecrc`. - Docker image tags no longer carry a leading `v`. Pull `3.12.0`, `3.12` and `3` instead of `v3.12.0`, `v3.12` and `v3` - the deprecated `SpacesAftertabs` configuration key has been removed, use `SpacesAfterTabs` instead. - stop accepting latin1 files as utf-8. ([#​598](https://github.com/editorconfig-checker/editorconfig-checker/issues/598))
Previously `charset=utf-8` was satisfied with files encoded as `latin1`. While most of the `latin1` range is identical to `utf-8`, not all is. If you request `utf-8` but have `latin1` encoded files expect these files to be reported as violations now. ##### Features - add safe EditorConfig autofixes ([#​612](https://github.com/editorconfig-checker/editorconfig-checker/issues/612)) ([2f4962c](https://github.com/editorconfig-checker/editorconfig-checker/commit/2f4962c596d85445720acee9e67853a93ff8fabd)) - auto-detect `github-actions` output format in CI ([#​606](https://github.com/editorconfig-checker/editorconfig-checker/issues/606)) ([271c500](https://github.com/editorconfig-checker/editorconfig-checker/commit/271c500a31a9984b8f5221863d82d6b79e3f5631)) - remove support for the deprecated `.ecrc` config file name ([#​603](https://github.com/editorconfig-checker/editorconfig-checker/issues/603)) ([2062751](https://github.com/editorconfig-checker/editorconfig-checker/commit/2062751696a47f09fdca5249b8ae7e42b8471b2a)) ##### Bug Fixes - add missing space in wrong indentation type error message ([#​610](https://github.com/editorconfig-checker/editorconfig-checker/issues/610)) ([e0f1b56](https://github.com/editorconfig-checker/editorconfig-checker/commit/e0f1b561a2cee12b77c83259125bc1e4a01a9c52)) - allow builds with Go 1.26 ([#​615](https://github.com/editorconfig-checker/editorconfig-checker/issues/615)) ([343018a](https://github.com/editorconfig-checker/editorconfig-checker/commit/343018a4a3fc5cb2a9b51e36046203a1d0cdfae3)) - consolidate error blocks that are interleaved with other errors ([#​601](https://github.com/editorconfig-checker/editorconfig-checker/issues/601)) ([76b6313](https://github.com/editorconfig-checker/editorconfig-checker/commit/76b6313bc26a546591020c4d4c1c35848393c95d)) - correct end line in codeclimate formatter output ([#​602](https://github.com/editorconfig-checker/editorconfig-checker/issues/602)) ([2476e7c](https://github.com/editorconfig-checker/editorconfig-checker/commit/2476e7c6716c9c496fe1cbee1da3fb8c13e32b78)) - correct the -disable-end-of-line flag description ([#​600](https://github.com/editorconfig-checker/editorconfig-checker/issues/600)) ([1615efc](https://github.com/editorconfig-checker/editorconfig-checker/commit/1615efcaed1a5fb93179a4f8b6ad554332053825)) - drop a path whose content type could not be read ([#​599](https://github.com/editorconfig-checker/editorconfig-checker/issues/599)) ([46995b1](https://github.com/editorconfig-checker/editorconfig-checker/commit/46995b1db09d5371b2e684bca693d9e56f7fcaaa)) - handle `unset` for `end_of_line` and `insert_final_newline` ([#​594](https://github.com/editorconfig-checker/editorconfig-checker/issues/594)) ([123ba31](https://github.com/editorconfig-checker/editorconfig-checker/commit/123ba319dcad0e048aaa182db1c6f4e37e13b573)), closes [#​580](https://github.com/editorconfig-checker/editorconfig-checker/issues/580) - remove ambiguity between potential folder and file matches ([#​611](https://github.com/editorconfig-checker/editorconfig-checker/issues/611)) ([4c4f7d4](https://github.com/editorconfig-checker/editorconfig-checker/commit/4c4f7d4c30900263dbb37d0c84568c4c719e4792)) - remove leading `v` from Docker image tags ([#​607](https://github.com/editorconfig-checker/editorconfig-checker/issues/607)) ([3827a6c](https://github.com/editorconfig-checker/editorconfig-checker/commit/3827a6c595a1001dc3958448b9b4e645975b3068)) - remove retired goreportcard badge ([2a9dd32](https://github.com/editorconfig-checker/editorconfig-checker/commit/2a9dd32f6e2543ac331eadaadec23f8494d6e499)) - stop accepting latin1 files as utf-8. ([#​598](https://github.com/editorconfig-checker/editorconfig-checker/issues/598))\
Previously `charset=utf-8` was satisfied with files encoded as `latin1`. While most of the `latin1` range is identical to `utf-8`, not all is. If you request `utf-8` but have `latin1` encoded files expect these files to be reported as violations now. ([4d84a77](https://github.com/editorconfig-checker/editorconfig-checker/commit/4d84a778c96073290eb6130fbefdaa9c77b8c2aa)) - typo in config, `SpacesAftertabs` => `SpacesAfterTabs` ([#​604](https://github.com/editorconfig-checker/editorconfig-checker/issues/604)) ([bbc3f03](https://github.com/editorconfig-checker/editorconfig-checker/commit/bbc3f035076dd4d1249baecafb29c8e10bf65a91)) ##### Miscellaneous Chores - internalize our packages ([#​619](https://github.com/editorconfig-checker/editorconfig-checker/issues/619))\
It will no longer be possible to import our internal packages in your code. If you rely on one of our packages please let us know by opening an issue. ([d42bcb7](https://github.com/editorconfig-checker/editorconfig-checker/commit/d42bcb7af0ba6e633a662467d8137a2ce6244e83)) - rename ec to editorconfig-checker ([#​371](https://github.com/editorconfig-checker/editorconfig-checker/issues/371)) ([0e3b27f](https://github.com/editorconfig-checker/editorconfig-checker/commit/0e3b27f7e298718620ad0e60b66e6afc4dc52bb9)) ##### Continuous Integration - migrate to goreleaser dockers\_v2. ([#​617](https://github.com/editorconfig-checker/editorconfig-checker/issues/617))\
This means we stop publishing the arch-suffixed images like `3.11.2-amd64` or `3.11.2-arm64v8`. All these architectures are still provided in the multiarch tag `3.11.2`. ([d4c1442](https://github.com/editorconfig-checker/editorconfig-checker/commit/d4c14421f81ad8099dbfd45ad0dd3b2141a2986d))
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM (`* 0-3 * * *`) - Automerge - Between 12:00 AM and 03:59 AM (`* 0-3 * * *`) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Mend Renovate](https://github.com/renovatebot/renovate). Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/197 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ddaa92b..4e28c6d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.11.3 + rev: v4.0.1 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ From 26406a3e86f62f7df71e71696f816fc2c76cbe29 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 4 Sep 2026 10:24:47 +0000 Subject: [PATCH 087/118] feat(ci): add a targeted per-R-minor rebuild workflow (#198) ## Why A user hit this on amd64/resolute under R 4.6.1: ``` base64enc.so: undefined symbol: SETLENGTH ``` R 4.6 no longer exports `SETLENGTH`: ``` R 4.4: missing -> R_ClosureEnv R_ClosureFormals R_MakeMissingBinding R_PromiseEnv R_PromiseValue R_mkClosure R 4.5: missing -> R_MakeMissingBinding R_PromiseEnv R_PromiseValue R 4.6: missing -> R_PromiseEnv R_PromiseValue SETLENGTH ``` `base64enc` calls `SETLENGTH` (`base64.c:206`, `253`), its flat binary was built under R 4.4.3, and it has no per-minor copy at all, so the union index hands that binary to a 4.6 client. The safety net missed it because the risky set is defined circularly: ```r risky_packages_across_minors <- function(...) { # a package is "risky" iff it already has a tarball in some per-minor slot ``` A package counts as ABI-risky only if it has *already* been built per-minor. `base64enc` never was, and having no new CRAN version it never enters the build list to be classified. So the per-minor protection only covers packages that passed through the pipeline after the scheme existed. ## Scope Scanning all 4,585 compiled binaries in amd64/resolute's flat index for symbols R 4.6 does not export found 10 referencing `SETLENGTH`. Most already have per-minor builds and are served correctly. Two do not: ``` base64enc_0.1-6 flat=200 4.4=404 4.5=404 4.6=404 lobstr_1.2.1 flat=200 4.4=404 4.5=200 4.6=404 ``` (`iotools` lacks a 4.5 copy but 4.5 still exports `SETLENGTH`, so it does not break.) ## What this adds Rebuilding one package into one minor was only possible with `just rebuild` from a workstation holding the S3 and Postgres credentials. This adds the CI path: ```sh crow pipeline create devxy/build-cran-binaries \ --var rebuild_abi_gap=ubuntu-2604-amd64 \ --var ABI_PACKAGES="base64enc@0.1-6 lobstr@1.2.1" \ --var ABI_R_MINOR=4.6 ``` - reuses `reindex.yaml`'s slot matrix verbatim, so the slot-to-image mapping cannot drift - runs `build-one.R --sensitive-only`, which reclassifies each version and skips anything not ABI-risky rather than writing it to the wrong slot - mounts no shared cache volume, so it cannot disturb the build pipeline's per-agent caches - attempts every package, then exits non-zero if any failed `crow lint` passes. ## Not in this PR This is the remediation path, not the fix. The circular `risky_packages_across_minors()` definition is still in place, so new instances of this class can still appear. That is the follow-up. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/198 --- .crow/rebuild-abi-gap.yaml | 218 +++++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 .crow/rebuild-abi-gap.yaml diff --git a/.crow/rebuild-abi-gap.yaml b/.crow/rebuild-abi-gap.yaml new file mode 100644 index 0000000..d346adf --- /dev/null +++ b/.crow/rebuild-abi-gap.yaml @@ -0,0 +1,218 @@ +# Rebuild specific packages into a per-R-minor slot. +# +# The per-minor scheme only ever covered packages that passed through the build +# pipeline after it existed. A package built into the flat slot before that, and +# never updated on CRAN since, is never reclassified and never gets a per-minor +# copy - so the union index serves its flat binary to every minor. When that +# binary references a symbol the client's R no longer exports, the install fails +# at dyn.load(): +# +# base64enc.so: undefined symbol: SETLENGTH +# +# base64enc calls SETLENGTH, its flat binary was built under R 4.4.3, and R 4.6 +# does not export SETLENGTH. There was no way to rebuild one package into one +# minor from CI, only `just rebuild` from a workstation with the S3 and Postgres +# credentials to hand. This adds that path. +# +# `build-one.R --sensitive-only` re-runs the ABI classifier per version, so a +# package that turns out not to be risky is skipped rather than written into the +# wrong slot. +# +# Trigger with the slot and an explicit package list: +# +# crow pipeline create devxy/build-cran-binaries \ +# --var rebuild_abi_gap=ubuntu-2604-amd64 \ +# --var ABI_PACKAGES="base64enc@0.1-6 lobstr@1.2.1" \ +# --var ABI_R_MINOR=4.6 + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the + # default must match no matrix row. + rebuild_abi_gap: + description: "Rebuild target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + ABI_PACKAGES: + description: 'Space-separated package@version list, e.g. "base64enc@0.1-6 lobstr@1.2.1".' + default: '' + ABI_R_MINOR: + description: 'The R minor to build into.' + options: + - '4.4' + - '4.5' + - '4.6' + default: '4.6' + +when: + - event: manual + evaluate: 'rebuild_abi_gap == "all" || rebuild_abi_gap == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Rebuild packages into the per-minor slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + GIT_USER: pat-s + # Pipeline variables are substituted into the YAML, not exported, so they + # have to be bound here to be readable as env vars in the script below. + ABI_PACKAGES: ${ABI_PACKAGES} + ABI_R_MINOR: ${ABI_R_MINOR} + UVR_CACHE_DIR: /tmp/uvr/cache + UVR_PACKAGES_DIR: /tmp/uvr/packages + CCACHE_DIR: /tmp/ccache + NCPUS: 2 + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p $UVR_CACHE_DIR $UVR_PACKAGES_DIR $CCACHE_DIR + - | + set -eu + if [ -z "$$ABI_PACKAGES" ]; then + echo "ABI_PACKAGES is empty; nothing to rebuild." >&2 + exit 1 + fi + RBIN=$(ls -d /opt/R/$${ABI_R_MINOR}*/bin 2>/dev/null | head -1) + if [ -z "$$RBIN" ]; then + echo "R $$ABI_R_MINOR is not installed in this image." >&2 + exit 1 + fi + # Container-local library: this workflow mounts no shared cache volume, + # so it cannot disturb the build pipeline's per-agent caches. + LIB="/tmp/R-pkgs-$$ABI_R_MINOR" + mkdir -p "$$LIB" + export R_LIBS_USER="$$LIB" + rm -rf "$$LIB"/00LOCK-* + "$$RBIN/Rscript" local/install-bincraft.R + rc=0 + for spec in $$ABI_PACKAGES; do + pkg=$${spec%@*} + ver=$${spec#*@} + if [ "$$pkg" = "$$ver" ]; then + echo "malformed spec '$$spec'; expected package@version" >&2 + rc=1 + continue + fi + echo "=== rebuilding $$pkg $$ver into $$ABI_R_MINOR ===" + "$$RBIN/Rscript" local/build-one.R --sensitive-only "$$pkg" "$$ver" || rc=1 + done + exit $$rc + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 1000m From 853b9247654f9e0dcdc29739a35b3a644d3b5eb9 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 4 Sep 2026 10:34:49 +0000 Subject: [PATCH 088/118] feat(abi): backfill classifier verdicts for pre-per-minor packages (#199) ## Why bincraft PR #124 makes the `abi_classification` cache authoritative for the risky-package set, breaking the circularity where a package counted as ABI-risky only once it had already been built per-minor. On its own that changes nothing for the packages actually at risk. The cache only holds verdicts for packages that passed through the build pipeline after the per-minor scheme existed; everything older has no verdict, and no verdict means the flat binary is carried into every per-minor index. That is exactly how a `base64enc` built under R 4.4.3 reached an R 4.6.1 client: ``` base64enc.so: undefined symbol: SETLENGTH ``` This is the backfill that gives those packages a verdict. ## What it does `local/backfill-abi-classification.R` enumerates a slot's flat index and runs each compiled package through `bincraft::classify_r_minor_sensitive()`, which looks the verdict up in the cache, and on a miss downloads the CRAN source, classifies it and stores the result. The verdict is a property of the source, independent of OS, arch and R minor, so **one run populates the cache for every slot**. The arch/codename arguments only choose which flat index to enumerate. Only `NeedsCompilation=yes` packages are classified: a package with no compiled code has no symbols to bind and cannot fail this way, so proving it by download would be wasted work. On amd64/resolute that is 4585 packages rather than 23218. Re-running is cheap and resumable: cached verdicts return without touching the network, so a second run costs one query per package and downloads only the remainder. ## Verification The enumeration was run against the live indexes and matches an independent symbol scan of the same slot exactly: ``` flat binaries: 23218 | due classification: 4585 base64enc included: TRUE lobstr included: TRUE ``` 4585 is the same population I scanned with `nm -D --undefined-only` when establishing the blast radius, and both packages confirmed to break under 4.6 are in it. `crow lint` passes. ## Sequencing Merge after bincraft #124 and after a bincraft release the pipelines pick up, otherwise the verdicts are written but nothing reads them yet. Writing them early is harmless. Then re-index the slots so the per-minor indexes are rebuilt with the widened risky set. Expect coverage in the per-minor indexes to *drop* slightly as newly-recognised risky packages are dropped rather than served from a mismatched minor; those then need per-minor builds, either through the normal pipeline or the targeted workflow in devxy/build-cran-binaries#198. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/199 --- .crow/backfill-abi-classification.yaml | 90 +++++++++++++++++ local/backfill-abi-classification.R | 129 +++++++++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 .crow/backfill-abi-classification.yaml create mode 100644 local/backfill-abi-classification.R diff --git a/.crow/backfill-abi-classification.yaml b/.crow/backfill-abi-classification.yaml new file mode 100644 index 0000000..aeab591 --- /dev/null +++ b/.crow/backfill-abi-classification.yaml @@ -0,0 +1,90 @@ +# Populate the ABI classifier cache for packages that predate the per-minor scheme. +# +# bincraft decides whether a flat binary may be carried into a per-minor index +# from its risky-package set. That set is now the union of the per-minor tarball +# scan and the `abi_classification` cache, but the cache only holds verdicts for +# packages that passed through the build pipeline after the scheme existed. +# Everything older has no verdict, and no verdict means "carry it" - which is how +# a base64enc built under R 4.4.3 reached R 4.6 clients and died at dyn.load() +# on SETLENGTH. +# +# The verdict is computed from the package source, so it does not depend on OS, +# arch or R minor: one run populates the cache for every slot. The arch/codename +# below only choose which flat index to enumerate. +# +# Re-running is cheap. Cached verdicts are returned without touching the network, +# so a second run costs one query per package and downloads only what is still +# unclassified. +# +# crow pipeline create devxy/build-cran-binaries --var backfill_abi=run + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the default + # must be a value the `when` clause rejects. + backfill_abi: + description: "Set to 'run' to classify the flat index into the ABI cache." + options: + - none + - run + default: none + BACKFILL_ARCH: + description: 'Arch of the flat index to enumerate.' + options: + - amd64 + - arm64 + default: amd64 + BACKFILL_CODENAME: + description: 'Codename of the flat index to enumerate.' + default: resolute + BACKFILL_LIMIT: + description: 'Classify at most this many packages (0 = no limit).' + default: '0' + +when: + - event: manual + evaluate: 'backfill_abi == "run"' + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Classify the flat index into the ABI cache' + image: reg.devxy.io/rpkgs/build-env-ubuntu:resolute + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + GIT_USER: pat-s + # Pipeline variables are substituted into the YAML, not exported, so they + # have to be bound here to be readable as env vars in the script below. + BACKFILL_ARCH: ${BACKFILL_ARCH} + BACKFILL_CODENAME: ${BACKFILL_CODENAME} + BACKFILL_LIMIT: ${BACKFILL_LIMIT} + R_LIBS_USER: /tmp/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /tmp/R-pkgs + - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R + - | + set -eu + LIMIT="" + if [ "$$BACKFILL_LIMIT" != "0" ]; then + LIMIT="$$BACKFILL_LIMIT" + fi + /opt/R/4.5.3/bin/Rscript local/backfill-abi-classification.R \ + "$$BACKFILL_ARCH" "$$BACKFILL_CODENAME" $$LIMIT + backend_options: + docker: + resources: + requests: + memory: 4Gi + cpu: 1000m diff --git a/local/backfill-abi-classification.R b/local/backfill-abi-classification.R new file mode 100644 index 0000000..0f48392 --- /dev/null +++ b/local/backfill-abi-classification.R @@ -0,0 +1,129 @@ +### Classify every compiled package in a slot's flat index, into the ABI cache. +# +# Usage: Rscript local/backfill-abi-classification.R [limit] +# +# The per-minor scheme only ever classified packages that passed through the +# build pipeline after it existed. Everything built before that, and not updated +# on CRAN since, has no verdict at all -- and bincraft's risky set is what +# decides whether a flat binary may be carried into a per-minor index. With no +# verdict the answer defaults to "carry it", which is how a base64enc built +# under R 4.4.3 reached R 4.6 clients and died at dyn.load() on SETLENGTH. +# +# The verdict is a property of the source, so it is independent of OS, arch and +# R minor: one slot's backfill populates the cache for every slot. The slot +# argument only selects which flat index to enumerate. +# +# Only packages with NeedsCompilation=yes are classified. A package with no +# compiled code has no symbols to bind and cannot fail this way, so downloading +# its source to prove that would be wasted work. +# +# Resumable: `classify_r_minor_sensitive()` returns cached verdicts without +# touching the network, so a re-run costs one query per package and only the +# unclassified remainder is downloaded. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2L) { + stop( + "usage: backfill-abi-classification.R [limit]", + call. = FALSE + ) +} +arch <- args[[1L]] +codename <- args[[2L]] +limit <- if (length(args) >= 3L) as.integer(args[[3L]]) else NA_integer_ + +db <- list( + metadata_db_type = "postgres", + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_port = 15432L, + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require" +) + +read_index <- function(index_url) { + con <- gzcon(url(index_url, open = "rb")) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "Built")) +} + +flat <- read_index(sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", + arch, + codename +)) +# Source fallbacks have no binary to be unsafe, so skip them. +flat <- flat[!is.na(flat[, "Built"]), , drop = FALSE] + +cran <- read.dcf( + gzcon(url( + "https://cloud.r-project.org/src/contrib/PACKAGES.gz", + open = "rb" + )), + fields = c("Package", "NeedsCompilation") +) +compiled <- cran[ + !is.na(cran[, "NeedsCompilation"]) & cran[, "NeedsCompilation"] == "yes", + "Package" +] + +todo <- flat[flat[, "Package"] %in% compiled, , drop = FALSE] +cat(sprintf( + "%s/%s: %d binaries in the flat index, %d compiled and due classification\n", + arch, + codename, + nrow(flat), + nrow(todo) +)) +if (!is.na(limit) && limit > 0L && limit < nrow(todo)) { + todo <- todo[seq_len(limit), , drop = FALSE] + cat(sprintf("Limited to the first %d\n", limit)) +} + +sensitive <- 0L +failed <- 0L +for (i in seq_len(nrow(todo))) { + pkg <- todo[i, "Package"] + ver <- todo[i, "Version"] + verdict <- tryCatch( + do.call( + bincraft::classify_r_minor_sensitive, + c(list(package_name = pkg, tag = ver), db) + ), + error = function(e) { + cat(sprintf(" %s %s: %s\n", pkg, ver, conditionMessage(e))) + NA + } + ) + if (is.na(verdict)) { + failed <- failed + 1L + } else if (isTRUE(verdict)) { + sensitive <- sensitive + 1L + } + if (i %% 250L == 0L) { + cat(sprintf( + " classified %d/%d (%d r-minor-sensitive, %d failed)\n", + i, + nrow(todo), + sensitive, + failed + )) + utils::flush.console() + } +} + +cat(sprintf( + "Done: %d classified, %d r-minor-sensitive, %d failed\n", + nrow(todo), + sensitive, + failed +)) From f9ebb3dba489a73c0d7c73c372ce14fbb0834231 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 4 Sep 2026 10:54:09 +0000 Subject: [PATCH 089/118] fix(abi): prune flat-slot binaries of ABI-risky packages (#200) ## Why `base64enc` still fails to install under R 4.6 even after it was built into the 4.6 slot: ``` base64enc.so: undefined symbol: SETLENGTH ``` The per-minor binary is correct and serves correctly to any client the router can identify. But the **flat slot still holds a binary built under R 4.4.3**, and that is what every client receives when it cannot be routed by R minor. That object should not exist. `upload_single_binary()` writes an ABI-risky package only to the per-minor path: ```r if (is_r_minor_sensitive) { remote_tarball_path <- file.path(remote_bin_path, minor_version, tarball_name) } else { remote_tarball_path <- file.path(remote_bin_path, tarball_name) } ``` `base64enc`'s flat binary is a legacy artifact from before it was classified risky. Removing it makes the outcome correct regardless of routing, user agent, downloader or renv configuration, which is the part I could not otherwise guarantee. ## What it does `local/prune-stale-flat-binaries.R` lists a slot's flat objects, intersects them with the risky set, and deletes them. Deleting is safe here: the source fallback republishes the CRAN source under the same key, so the package stays installable everywhere and compiles against the client's own R, while routed clients keep getting the per-minor binary. Safety posture, given this deletes production objects: - **dry run by default**; nothing is removed without `--apply` - takes an **explicit package list**, so the first runs need not trust the classifier cache at all - **refuses to proceed on an empty risky set**, which is indistinguishable from an unreachable cache and would otherwise report success having pruned nothing - **keeps objects whose package CRAN no longer ships**, since there would be no source to fall back to and the package would vanish - **re-indexes and purges in the same run**. This is not optional: a published index naming deleted files serves 404s, which is worse than the binary it replaced. ## Use ```sh # dry run first crow pipeline create devxy/build-cran-binaries \ --var prune_flat=ubuntu-2604-amd64 --var PRUNE_PACKAGES="base64enc lobstr" # then apply crow pipeline create devxy/build-cran-binaries \ --var prune_flat=ubuntu-2604-amd64 --var PRUNE_PACKAGES="base64enc lobstr" \ --var PRUNE_APPLY=yes ``` `crow lint` passes. ## Sequencing Run with an explicit list first, on one slot, and confirm the flat URL then serves source while the per-minor URLs still serve binaries. Only widen to the cache-derived set once the backfill has finished and its output looks sane: the first 250 classifications came back 232 r-minor-sensitive, and a rate that high suggests the source-grep classifier is broader than the set that actually breaks, which is worth checking before pruning thousands of flat binaries. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/200 --- .crow/prune-stale-flat-binaries.yaml | 245 +++++++++++++++++++++++++++ local/prune-stale-flat-binaries.R | 136 +++++++++++++++ 2 files changed, 381 insertions(+) create mode 100644 .crow/prune-stale-flat-binaries.yaml create mode 100644 local/prune-stale-flat-binaries.R diff --git a/.crow/prune-stale-flat-binaries.yaml b/.crow/prune-stale-flat-binaries.yaml new file mode 100644 index 0000000..a759daf --- /dev/null +++ b/.crow/prune-stale-flat-binaries.yaml @@ -0,0 +1,245 @@ +# Remove flat-slot binaries belonging to ABI-risky packages, then re-index. +# +# `upload_single_binary()` writes an ABI-risky package only to the per-minor +# slot, so a risky package should have no flat binary at all. Packages built +# before they were classified risky still do, and the flat slot is what every +# client that cannot be routed by R minor receives. That binary was built under +# whichever R was primary at the time, so it dies at load under any minor that +# dropped a symbol it uses: +# +# base64enc.so: undefined symbol: SETLENGTH +# +# Deleting the flat object does not create a hole. The source fallback +# republishes the CRAN source under the same key, so the package stays +# installable everywhere and compiles against the client's own R, while routed +# clients keep getting the per-minor binary. +# +# Dry run unless PRUNE_APPLY is 'yes'. Start with an explicit package list: +# +# crow pipeline create devxy/build-cran-binaries \ +# --var prune_flat=ubuntu-2604-amd64 \ +# --var PRUNE_PACKAGES="base64enc lobstr" \ +# --var PRUNE_APPLY=yes + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the default + # must match no matrix row. + prune_flat: + description: "Prune target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + PRUNE_PACKAGES: + description: 'Space-separated package names. Empty means every package the classifier cache calls risky.' + default: '' + PRUNE_APPLY: + description: "Set to 'yes' to actually delete. Anything else is a dry run." + options: + - 'no' + - 'yes' + default: 'no' + +when: + - event: manual + evaluate: 'prune_flat == "all" || prune_flat == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + CODENAME: alpine322 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + CODENAME: alpine322 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + CODENAME: alpine323 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + CODENAME: alpine323 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + CODENAME: alpine324 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + CODENAME: alpine324 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + CODENAME: rhel8 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + CODENAME: rhel8 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + CODENAME: rhel9 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + CODENAME: rhel9 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + CODENAME: rhel10 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + CODENAME: rhel10 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + CODENAME: jammy + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + CODENAME: jammy + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + CODENAME: noble + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + CODENAME: noble + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + CODENAME: resolute + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + CODENAME: resolute + IMG: ubuntu:resolute + +steps: + - name: 'Prune stale flat binaries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /tmp/R-pkgs + # Pipeline variables are substituted into the YAML, not exported, so they + # have to be bound here to be readable as env vars in the script below. + PRUNE_ARCH: ${ARCH} + PRUNE_CODENAME: ${CODENAME} + PRUNE_PACKAGES: ${PRUNE_PACKAGES} + PRUNE_APPLY: ${PRUNE_APPLY} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /tmp/R-pkgs + - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R + - | + set -eu + APPLY="" + if [ "$$PRUNE_APPLY" = "yes" ]; then + APPLY="--apply" + fi + /opt/R/4.5.3/bin/Rscript local/prune-stale-flat-binaries.R \ + "$$PRUNE_ARCH" "$$PRUNE_CODENAME" $$APPLY $$PRUNE_PACKAGES + backend_options: + docker: + resources: + requests: + memory: 3Gi + cpu: 1000m + + # Deleting objects without re-indexing is worse than leaving them: the + # published index keeps naming files that no longer exist, so every client + # gets a 404 instead of a working source fallback. The re-index is what + # republishes them as CRAN source, so it belongs in the same run. + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + R_LIBS_USER: /tmp/R-pkgs + PRUNE_APPLY: ${PRUNE_APPLY} + commands: + - | + set -eu + if [ "$$PRUNE_APPLY" != "yes" ]; then + echo "Dry run: nothing was deleted, so there is nothing to re-index." + exit 0 + fi + /opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + backend_options: + docker: + resources: + requests: + memory: 3Gi + cpu: 1000m + + # The edge caches indexes for ~370 days, so without this the deletion is + # invisible to clients and they keep being handed the removed binary. + - name: 'Purge CDN cache' + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + # Same zones as reindex.yaml: the production pull zone id, the alliance + # hostname and the staging zone, which shares the B2 origin. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' + PRUNE_APPLY: ${PRUNE_APPLY} + commands: + - | + set -eu + if [ "$$PRUNE_APPLY" != "yes" ]; then + echo "Dry run: nothing changed at the origin, so no purge is needed." + exit 0 + fi + apk add --no-cache -q bash curl jq + bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES diff --git a/local/prune-stale-flat-binaries.R b/local/prune-stale-flat-binaries.R new file mode 100644 index 0000000..dbb0ab1 --- /dev/null +++ b/local/prune-stale-flat-binaries.R @@ -0,0 +1,136 @@ +### Remove flat-slot binaries of packages that are ABI-risky. +# +# Usage: +# Rscript local/prune-stale-flat-binaries.R [--apply] [pkg ...] +# +# `upload_single_binary()` writes an ABI-risky package only to the per-minor +# slot, never to the flat one, so a risky package should have no flat binary at +# all. Packages built before they were classified risky still do, and the flat +# slot is what every client that cannot be routed by R minor receives. That +# binary was built under whichever R was primary at the time, so it fails at +# load under any minor that dropped a symbol it uses: +# +# base64enc.so: undefined symbol: SETLENGTH +# +# Deleting the flat object does not create a hole: bincraft's source fallback +# publishes the CRAN source under the same key, so the package stays installable +# everywhere and compiles against the client's own R. Clients that are routed +# keep getting the per-minor binary. +# +# Dry run by default. Nothing is deleted without --apply. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +args <- commandArgs(trailingOnly = TRUE) +apply_changes <- "--apply" %in% args +args <- args[args != "--apply"] +if (length(args) < 2L) { + stop( + "usage: prune-stale-flat-binaries.R [--apply] [pkg ...]", + call. = FALSE + ) +} +arch <- args[[1L]] +codename <- args[[2L]] +explicit <- args[-c(1L, 2L)] + +bucket <- "devxy-rpkgs-binaries" +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) + +contrib <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename) +objects <- s3fs::s3_dir_ls(contrib, recurse = TRUE) +relative <- sub(sprintf("^(s3://)?%s/", contrib), "", objects) + +is_tarball <- grepl("\\.tar\\.gz$", relative) +# Flat objects only: no `/` prefix and not under Archive/. +is_flat <- is_tarball & !grepl("/", relative, fixed = TRUE) +flat_files <- relative[is_flat] +flat_pkg <- sub("_.*$", "", flat_files) + +if (length(explicit) > 0L) { + risky <- explicit + cat(sprintf("Restricted to %d named package(s)\n", length(risky))) +} else { + risky <- bincraft:::abi_cache_risky_packages() + cat(sprintf("Classifier cache reports %d risky package(s)\n", length(risky))) + if (length(risky) == 0L) { + stop( + "The classifier cache returned nothing. Refusing to proceed: an empty risky set here is indistinguishable from an unreachable cache, and would silently prune nothing while reporting success.", # nolint + call. = FALSE + ) + } +} + +doomed <- flat_files[flat_pkg %in% risky] +cat(sprintf( + "%s/%s: %d flat binaries, %d belong to ABI-risky packages\n", + arch, + codename, + length(flat_files), + length(doomed) +)) + +# Never remove the last artifact of a package. The source fallback republishes +# the CRAN source under this key, but only for packages CRAN still ships, so +# confirm the source exists before deleting the binary. +cran <- tryCatch( + { + con <- gzcon(url( + "https://cloud.r-project.org/src/contrib/PACKAGES.gz", + open = "rb" + )) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = "Package")[, "Package"] + }, + error = function(e) character(0) +) +if (length(cran) == 0L) { + stop("Could not read CRAN's index; refusing to prune blind.", call. = FALSE) +} + +orphaned <- doomed[!(sub("_.*$", "", doomed) %in% cran)] +if (length(orphaned) > 0L) { + cat(sprintf( + "Keeping %d object(s) whose package is no longer on CRAN, so there is no source to fall back to: %s\n", + length(orphaned), + paste(utils::head(orphaned, 5L), collapse = ", ") + )) + doomed <- setdiff(doomed, orphaned) +} + +if (length(doomed) == 0L) { + cat("Nothing to prune.\n") + quit(status = 0) +} + +for (f in doomed) { + cat(sprintf(" %s %s\n", if (apply_changes) "DELETE" else "would delete", f)) +} + +if (!apply_changes) { + cat(sprintf( + "\nDry run: %d object(s) left in place. Re-run with --apply to delete.\n", + length(doomed) + )) + quit(status = 0) +} + +for (f in doomed) { + s3fs::s3_file_delete(file.path(contrib, f)) +} +cat(sprintf( + "Deleted %d flat binary object(s). Re-index the slot so the source fallback republishes them.\n", + length(doomed) +)) From e1114bc14579c150241425f18c9df180edaf0236 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 4 Sep 2026 19:26:55 +0000 Subject: [PATCH 090/118] fix(ci): give the index writers the metadata database password (#201) ## Why Pipeline 12207 logs one of these per slot: ``` Error: connection to server at "r-binaries.devxy.io" (116.203.165.191), port 15432 failed: fe_sendauth: no password supplied ``` bincraft's index build calls `abi_cache_risky_packages()` to decide which packages may be carried from the flat slot into a per-minor index. `reindex.yaml` and `weekly-rebuild-reindex.yaml` never passed `PGPASS`, so the query cannot connect. It fails soft by design, which is what makes this worth fixing rather than tolerating: the run still reports **success** while using an empty cache-derived risky set. The index is then written as though no package were known risky from the classifier, which is the exact state the cache was added to fix. `build-all-versions.yaml` already passes `PGPASS`, which is why classification behaved correctly there and silently degraded here. ## What changed Adds the `PGPASS` secret to both index-writing workflows, with a comment explaining why its absence is invisible rather than loud. `crow lint` passes on both files. ## Note The tarball-derived half of the risky set still worked, so indexes written without this are not wrong so much as incomplete: they miss packages that are known risky from classification but have no per-minor tarball yet. A re-index after this merges applies the full set. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/201 --- .crow/reindex.yaml | 7 +++++++ .crow/weekly-rebuild-reindex.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml index b13245a..528def4 100644 --- a/.crow/reindex.yaml +++ b/.crow/reindex.yaml @@ -142,6 +142,13 @@ steps: from_secret: B2_S3_ACCESS_KEY B2_S3_SECRET_KEY: from_secret: B2_S3_SECRET_KEY + # The index build reads the ABI classifier cache to decide which packages + # may be carried from the flat slot into a per-minor one. Without the + # password that query fails, and it fails soft, so the run still succeeds + # while quietly using an empty risky set: exactly the state the cache was + # added to fix. It logs one connection error per slot. + PGPASS: + from_secret: PGPASS REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN GIT_USER: pat-s diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index c9f9a8a..8de71f7 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -145,6 +145,13 @@ steps: from_secret: B2_S3_ACCESS_KEY B2_S3_SECRET_KEY: from_secret: B2_S3_SECRET_KEY + # The index build reads the ABI classifier cache to decide which packages + # may be carried from the flat slot into a per-minor one. Without the + # password that query fails, and it fails soft, so the run still succeeds + # while quietly using an empty risky set: exactly the state the cache was + # added to fix. It logs one connection error per slot. + PGPASS: + from_secret: PGPASS REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN GIT_USER: pat-s From c216e44578832d4a35168e25f0a0d09a53e779db Mon Sep 17 00:00:00 2001 From: pat-s Date: Sat, 5 Sep 2026 09:35:47 +0000 Subject: [PATCH 091/118] feat(abi): backfill per-object flat-slot safety verdicts (#202) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why bincraft #128 lets `union_index_records()` keep a carried flat record when that specific object is recorded safe. On its own it changes nothing for the records already being dropped, because they are legacy objects with no verdict: ``` 4.4: 16398 dropped across 9 indexes (~1822 each) 4.5: 20076 dropped across 9 indexes (~2230 each) 4.6: 15532 dropped across 9 indexes (~1726 each) ``` Every one of those resolves to CRAN source for clients the routing serves correctly. This backfill inspects the objects already in the bucket and writes their verdicts, which is what actually restores the coverage. ## How the verdict is made By symbols, not heuristics. A binary fails at `dyn.load()` under minor M exactly when it references a symbol M does not export, and the volatile set is derived by diffing the `libR.so` exports of the R builds installed in the image — so a new R release updates it with no curated list to maintain. That matters here because the source-level classifier calls **93%** of compiled packages risky (`LinkingTo: Rcpp` alone is enough), while a symbol scan of the same slot found **17** packages that break under 4.6 and **72** under 4.4. Recording the real answer per object is what makes the difference between ~2000 dropped records and the few dozen that genuinely need dropping. ## Safety - **Refuses to run where fewer than two R minors are installed.** With nothing to compare, every verdict would read "safe" — writing a bucket full of false clearances is far worse than writing none. - Skips packages with no compiled code: they cannot fail this way and are never dropped, so downloading them to prove it is wasted. - Resumable; already-recorded objects cost one query. `crow lint` passes. ## Sequencing Merge and release bincraft #128 first, then run this per slot, then re-index. Verifying the drop counts fall is the point of the exercise, so that re-index is where the fix is confirmed rather than assumed. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/202 --- .crow/backfill-flat-safety.yaml | 178 ++++++++++++++++++++++++++++++++ local/backfill-flat-safety.R | 148 ++++++++++++++++++++++++++ 2 files changed, 326 insertions(+) create mode 100644 .crow/backfill-flat-safety.yaml create mode 100644 local/backfill-flat-safety.R diff --git a/.crow/backfill-flat-safety.yaml b/.crow/backfill-flat-safety.yaml new file mode 100644 index 0000000..e2f850d --- /dev/null +++ b/.crow/backfill-flat-safety.yaml @@ -0,0 +1,178 @@ +# Record, per flat-slot object, whether it loads under every supported R minor. +# +# `union_index_records()` keeps a carried flat record when that object is +# recorded safe and drops it otherwise. Objects uploaded before that record +# existed have no verdict, so they are dropped: a full re-index removed roughly +# 2000 records per per-minor index (16398 across the 4.4 indexes, 20076 across +# 4.5, 15532 across 4.6), each one resolving to CRAN source for clients the +# routing serves correctly. +# +# This inspects what is already in the bucket and writes the verdict, which is +# what restores that coverage. Decided by symbols rather than heuristics: a +# binary fails at dyn.load() under minor M exactly when it references a symbol M +# does not export, and the volatile set is derived by diffing the R builds in +# the image. +# +# Re-running is cheap; recorded objects are skipped. +# +# crow pipeline create devxy/build-cran-binaries --var flat_safety=all + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the default + # must match no matrix row. + flat_safety: + description: "Inspect target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + FLAT_SAFETY_LIMIT: + description: 'Inspect at most this many objects per slot (0 = no limit).' + default: '0' + +when: + - event: manual + evaluate: 'flat_safety == "all" || flat_safety == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Inspect flat objects and record safety' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /tmp/R-pkgs + FLAT_SAFETY_ARCH: ${ARCH} + FLAT_SAFETY_LIMIT: ${FLAT_SAFETY_LIMIT} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /tmp/R-pkgs + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - | + set -eu + LIMIT="" + if [ "$$FLAT_SAFETY_LIMIT" != "0" ]; then + LIMIT="$$FLAT_SAFETY_LIMIT" + fi + # The codename comes from the image's /etc/os-release, exactly as the + # re-index step derives it, so it cannot drift from the matrix. + CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))') + echo "slot: $$FLAT_SAFETY_ARCH/$$CODENAME" + /opt/R/$R_VERSION/bin/Rscript local/backfill-flat-safety.R \ + "$$FLAT_SAFETY_ARCH" "$$CODENAME" $$LIMIT + backend_options: + docker: + resources: + requests: + memory: 4Gi + cpu: 1000m diff --git a/local/backfill-flat-safety.R b/local/backfill-flat-safety.R new file mode 100644 index 0000000..fe23735 --- /dev/null +++ b/local/backfill-flat-safety.R @@ -0,0 +1,148 @@ +### Record, per flat-slot object, whether it loads under every supported R minor. +# +# Usage: Rscript local/backfill-flat-safety.R [limit] +# +# `union_index_records()` keeps a carried flat record when that object is +# recorded safe, and drops it otherwise. Objects uploaded before that record +# existed have no verdict, so they are dropped: a full re-index removed ~2000 +# records per per-minor index, all resolving to CRAN source for clients the +# routing serves correctly. +# +# This inspects the objects already in the bucket and writes the verdict, which +# is what restores that coverage. The question is decided by symbols, not +# heuristics: a binary fails at dyn.load() under minor M exactly when it +# references a symbol M does not export, and the volatile set is derived by +# diffing the R builds installed in this image. +# +# Only compiled packages are inspected. A package with no shared object cannot +# fail this way and is never dropped, so proving it would be wasted downloads. +# +# Resumable: objects already recorded are skipped, so a re-run costs one query. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2L) { + stop("usage: backfill-flat-safety.R [limit]", call. = FALSE) +} +arch <- args[[1L]] +codename <- args[[2L]] +limit <- if (length(args) >= 3L) as.integer(args[[3L]]) else NA_integer_ + +sets <- bincraft:::installed_r_symbol_sets() +if (is.null(sets)) { + stop( + "Fewer than two R minors are installed here, so nothing can be compared. Refusing to record verdicts that would all read 'safe'.", # nolint + call. = FALSE + ) +} +cat(sprintf( + "Comparing R minors: %s (%d symbols vary across them)\n", + paste(names(sets), collapse = ", "), + length(bincraft:::cross_minor_volatile_symbols(sets)) +)) + +read_index <- function(index_url) { + con <- gzcon(url(index_url, open = "rb")) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "Built", "NeedsCompilation")) +} + +flat <- read_index(sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", + arch, + codename +)) +# Source fallbacks carry no binary, so there is nothing to inspect. +flat <- flat[!is.na(flat[, "Built"]), , drop = FALSE] +compiled <- flat[ + !is.na(flat[, "NeedsCompilation"]) & flat[, "NeedsCompilation"] == "yes", + , + drop = FALSE +] +cat(sprintf( + "%s/%s: %d flat binaries, %d compiled and worth inspecting\n", + arch, + codename, + nrow(flat), + nrow(compiled) +)) +if (!is.na(limit) && limit > 0L && limit < nrow(compiled)) { + compiled <- compiled[seq_len(limit), , drop = FALSE] + cat(sprintf("Limited to the first %d\n", limit)) +} + +base <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib", + arch, + codename +) +safe <- 0L +unsafe <- 0L +skipped <- 0L +for (i in seq_len(nrow(compiled))) { + pkg <- compiled[i, "Package"] + ver <- compiled[i, "Version"] + tarball <- file.path(tempdir(), sprintf("%s_%s.tar.gz", pkg, ver)) + ok <- tryCatch( + { + utils::download.file( + sprintf("%s/%s_%s.tar.gz", base, pkg, ver), + tarball, + quiet = TRUE, + mode = "wb" + ) + TRUE + }, + error = function(e) FALSE, + warning = function(w) FALSE + ) + if (!ok) { + skipped <- skipped + 1L + next + } + verdict <- bincraft:::tarball_abi_verdict(tarball, sets) + unlink(tarball) + if (!isTRUE(verdict$inspected)) { + skipped <- skipped + 1L + next + } + bincraft:::flat_safety_store( + pkg, + ver, + codename, + arch, + safe = !isTRUE(verdict$sensitive), + unsupported = verdict$unsupported + ) + if (isTRUE(verdict$sensitive)) { + unsafe <- unsafe + 1L + } else { + safe <- safe + 1L + } + if (i %% 250L == 0L) { + cat(sprintf( + " inspected %d/%d (%d safe, %d unsafe, %d skipped)\n", + i, + nrow(compiled), + safe, + unsafe, + skipped + )) + utils::flush.console() + } +} + +cat(sprintf( + "Done: %d inspected, %d safe for every minor, %d unsafe, %d skipped\n", + nrow(compiled), + safe, + unsafe, + skipped +)) From 844ad9749b98ebf1e7f030a8a26603201d6095c7 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sat, 5 Sep 2026 09:41:57 +0000 Subject: [PATCH 092/118] fix(abi): enumerate the flat slot from S3, not through the edge (#203) ## Why The first backfill run reported: ``` amd64/resolute: 27105 flat binaries, 8187 compiled and worth inspecting Done: 300 inspected, 7 safe for every minor, 1 unsafe, 292 skipped ``` 292 of 300 skipped, and `27105` is larger than the flat index (~24k). Both have the same cause: the script read `PACKAGES.gz` over HTTPS from inside R, which sends an R User-Agent, so **the edge routed it to a per-minor index**. It then enumerated union records carrying `Path: `, whose objects do not exist at the flat path, and every download 404'd. An ironic failure given what this whole change is about, and one the trial run existed to catch. ## What changed - List the flat slot with `s3fs::s3_dir_ls()`, keeping only objects with no `/` prefix and nothing under `Archive/`. The bucket cannot be routed and is authoritative about what is actually in the flat slot. - Fetch each object from S3 too. The edge does not rewrite tarball requests, but it does cache them, and the verdict is about the object. - `NeedsCompilation` still comes from CRAN's own index, which is not behind our edge. ## Verification The next trial run is the check: `flat objects` should now match the flat index rather than exceed it, and `skipped` should be near zero. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/203 --- local/backfill-flat-safety.R | 77 +++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/local/backfill-flat-safety.R b/local/backfill-flat-safety.R index fe23735..a8a8bbe 100644 --- a/local/backfill-flat-safety.R +++ b/local/backfill-flat-safety.R @@ -48,29 +48,56 @@ cat(sprintf( length(bincraft:::cross_minor_volatile_symbols(sets)) )) -read_index <- function(index_url) { - con <- gzcon(url(index_url, open = "rb")) - on.exit(close(con), add = TRUE) - read.dcf(con, fields = c("Package", "Version", "Built", "NeedsCompilation")) -} - -flat <- read_index(sprintf( - "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", +# The flat slot is listed from S3, not from PACKAGES.gz over HTTPS. Reading the +# index from inside R sends an R User-Agent, which the edge routes to a +# per-minor index; that enumerates union records carrying `Path: `, whose +# objects do not exist at the flat path. Every download then 404s: a first run +# skipped 292 of 300 that way. The bucket cannot be routed and is authoritative. +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) +contrib <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename -)) -# Source fallbacks carry no binary, so there is nothing to inspect. -flat <- flat[!is.na(flat[, "Built"]), , drop = FALSE] -compiled <- flat[ - !is.na(flat[, "NeedsCompilation"]) & flat[, "NeedsCompilation"] == "yes", - , - drop = FALSE +) +objects <- s3fs::s3_dir_ls(contrib, recurse = TRUE) +relative <- sub(sprintf("^(s3://)?%s/", contrib), "", objects) +# Flat objects only: no `/` prefix and nothing under Archive/. +flat_files <- relative[ + grepl("\\.tar\\.gz$", relative) & !grepl("/", relative, fixed = TRUE) ] +flat_pkg <- sub("_.*$", "", flat_files) +flat_ver <- sub("\\.tar\\.gz$", "", sub("^[^_]*_", "", flat_files)) + +# Only compiled packages can fail at dyn.load(), so the rest are not worth +# downloading. CRAN's own index says which; it is not behind our edge. +cran <- read.dcf( + gzcon(url( + "https://cloud.r-project.org/src/contrib/PACKAGES.gz", + open = "rb" + )), + fields = c("Package", "NeedsCompilation") +) +compiled_names <- cran[ + !is.na(cran[, "NeedsCompilation"]) & cran[, "NeedsCompilation"] == "yes", + "Package" +] +keep <- flat_pkg %in% compiled_names +compiled <- data.frame( + Package = flat_pkg[keep], + Version = flat_ver[keep], + stringsAsFactors = FALSE +) cat(sprintf( - "%s/%s: %d flat binaries, %d compiled and worth inspecting\n", + "%s/%s: %d flat objects, %d compiled and worth inspecting\n", arch, codename, - nrow(flat), + length(flat_files), nrow(compiled) )) if (!is.na(limit) && limit > 0L && limit < nrow(compiled)) { @@ -78,11 +105,8 @@ if (!is.na(limit) && limit > 0L && limit < nrow(compiled)) { cat(sprintf("Limited to the first %d\n", limit)) } -base <- sprintf( - "https://cran.rpkgs.com/%s/%s/latest/src/contrib", - arch, - codename -) +# Fetch from the bucket as well: the edge does not rewrite tarball requests, +# but it does cache them, and the object is what the verdict is about. safe <- 0L unsafe <- 0L skipped <- 0L @@ -92,13 +116,12 @@ for (i in seq_len(nrow(compiled))) { tarball <- file.path(tempdir(), sprintf("%s_%s.tar.gz", pkg, ver)) ok <- tryCatch( { - utils::download.file( - sprintf("%s/%s_%s.tar.gz", base, pkg, ver), + s3fs::s3_file_download( + sprintf("%s/%s_%s.tar.gz", contrib, pkg, ver), tarball, - quiet = TRUE, - mode = "wb" + overwrite = TRUE ) - TRUE + file.exists(tarball) }, error = function(e) FALSE, warning = function(w) FALSE From f3984f337d4e4e10a7e8c49a1d8bc9eb9683ff1c Mon Sep 17 00:00:00 2001 From: pat-s Date: Sat, 5 Sep 2026 15:59:16 +0000 Subject: [PATCH 093/118] fix(ci): authenticate uvr's GitHub resolution in the index workflows (#204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why The post-backfill re-index (12257) reported failure. Five slots could not write their indexes: ``` WARN Failed to fetch git Remote pat-s/cranlike@s3 (GitHub API error: 403 Forbidden); falling back to registry resolution ! Attempt 1/3 to update PACKAGES failed (unused argument (built = built)). Retrying in 5s... Error: Failed to update PACKAGES after 3 attempts. Last error: unused argument (built = built) ``` uvr resolves bincraft's dependencies through the GitHub API, including the `pat-s/cranlike@s3` fork. Unauthenticated, that is rate limited. uvr does not fail on the 403 — it **warns and falls back to the registry copy of cranlike**, which has no `built` parameter, so `update_PACKAGES(built = ...)` dies three retries later. Affected: alpine323, rhel8, rhel9, rhel10, noble. The remaining 13 slots resolved the fork fine and indexed correctly, which is what makes this insidious: the same pipeline passes or fails on identical input depending on the rate-limit window. `build-all-versions.yaml` and `rebuild-abi-gap.yaml` already pass `GITHUB_PAT`, which is why they were never affected. ## What changed Adds the `GITHUB_PAT` secret to `reindex.yaml`, `weekly-rebuild-reindex.yaml` and `backfill-flat-safety.yaml`, with a comment recording why its absence is quiet rather than loud. `crow lint` passes on all three. ## Context The re-index itself did what it was meant to. Drops per per-minor index fell from ~1800-2200 to: ``` 4.4: avg 129 4.5: avg 172 4.6: avg 145 ``` so the per-object safety verdicts recovered roughly 93% of the coverage while still dropping every object that genuinely cannot load. This PR is about the five slots that never got to apply it. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/204 --- .crow/backfill-flat-safety.yaml | 8 ++++++++ .crow/reindex.yaml | 8 ++++++++ .crow/weekly-rebuild-reindex.yaml | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/.crow/backfill-flat-safety.yaml b/.crow/backfill-flat-safety.yaml index e2f850d..e5c9b68 100644 --- a/.crow/backfill-flat-safety.yaml +++ b/.crow/backfill-flat-safety.yaml @@ -150,6 +150,14 @@ steps: from_secret: PGPASS REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN + # uvr resolves bincraft's dependencies through the GitHub API, including + # the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the + # failure is not loud: uvr warns, falls back to the registry copy of + # cranlike, and the index write then dies on `unused argument (built = + # built)` because upstream has no such parameter. Intermittent by nature, + # since it depends on the rate-limit window. + GITHUB_PAT: + from_secret: GITHUB_PAT GIT_USER: pat-s R_LIBS_USER: /tmp/R-pkgs FLAT_SAFETY_ARCH: ${ARCH} diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml index 528def4..71bca2c 100644 --- a/.crow/reindex.yaml +++ b/.crow/reindex.yaml @@ -151,6 +151,14 @@ steps: from_secret: PGPASS REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN + # uvr resolves bincraft's dependencies through the GitHub API, including + # the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the + # failure is not loud: uvr warns, falls back to the registry copy of + # cranlike, and the index write then dies on `unused argument (built = + # built)` because upstream has no such parameter. Intermittent by nature, + # since it depends on the rate-limit window. + GITHUB_PAT: + from_secret: GITHUB_PAT GIT_USER: pat-s R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 8de71f7..66603e3 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -154,6 +154,14 @@ steps: from_secret: PGPASS REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN + # uvr resolves bincraft's dependencies through the GitHub API, including + # the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the + # failure is not loud: uvr warns, falls back to the registry copy of + # cranlike, and the index write then dies on `unused argument (built = + # built)` because upstream has no such parameter. Intermittent by nature, + # since it depends on the rate-limit window. + GITHUB_PAT: + from_secret: GITHUB_PAT GIT_USER: pat-s R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} From cceda03935bc6390c02f3f9b01e25a4cb12e347b Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 7 Sep 2026 06:20:04 +0000 Subject: [PATCH 094/118] fix(ci): do not gate the targeted repair on the source classifier (#205) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why The repair workflow ran `build-one.R --sensitive-only`. That flag skips any version the **source** classifier does not call r-minor-sensitive — and it misses `Rcpp`. So the repair skipped `Rcpp` on every slot, leaving its 4.5-built flat binary as the only copy: ``` flat 200 built under R 4.5.3 4.4/ 404 4.5/ 404 4.6/ 404 ``` Every R 4.4 client on those slots then failed at `dyn.load()`. A real install check across 8 slots x 3 minors x 3 repository forms found **44 such failures**, `Rcpp`, `Matrix` and `RJSONIO` among them. ## What changed Drops `--sensitive-only`. Since bincraft v5.3.0 the destination is decided by the built binary's symbols in `upload_single_binary()`, so building unconditionally is safe and correct: a package that needs a per-minor slot lands there, one that does not lands in the flat slot. Skipping before the build is exactly what removed that decision from the path. `crow lint` passes. ## Companion bincraft #129 is the other half: the index build could not compensate either, because it only considered dropping records whose *package* was flagged risky, and `Rcpp` is not — even though the backfill had recorded that object as unable to load under 4.4. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/205 --- .crow/rebuild-abi-gap.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.crow/rebuild-abi-gap.yaml b/.crow/rebuild-abi-gap.yaml index d346adf..289781d 100644 --- a/.crow/rebuild-abi-gap.yaml +++ b/.crow/rebuild-abi-gap.yaml @@ -14,9 +14,15 @@ # minor from CI, only `just rebuild` from a workstation with the S3 and Postgres # credentials to hand. This adds that path. # -# `build-one.R --sensitive-only` re-runs the ABI classifier per version, so a -# package that turns out not to be risky is skipped rather than written into the -# wrong slot. +# `--sensitive-only` is deliberately NOT passed. That flag gates on the source +# classifier, which misses Rcpp: the repair therefore skipped Rcpp entirely, +# leaving its 4.5-built flat binary as the only copy and breaking every R 4.4 +# client on the slot. +# +# Since bincraft v5.3.0 the destination is decided by the built binary's symbols +# in `upload_single_binary()`, so building unconditionally is safe: a package +# that needs a per-minor slot lands there, and one that does not lands in the +# flat slot. Skipping before the build is what removed that decision. # # Trigger with the slot and an explicit package list: # @@ -207,7 +213,7 @@ steps: continue fi echo "=== rebuilding $$pkg $$ver into $$ABI_R_MINOR ===" - "$$RBIN/Rscript" local/build-one.R --sensitive-only "$$pkg" "$$ver" || rc=1 + "$$RBIN/Rscript" local/build-one.R "$$pkg" "$$ver" || rc=1 done exit $$rc backend_options: From ee4562d44cc5e3619a3f5d83cb0e4cfa238a7f34 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 7 Sep 2026 18:46:50 +0000 Subject: [PATCH 095/118] feat(abi): repair from the recorded verdicts rather than an ad-hoc scan (#206) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why The reprex found 44 routed-path failures: `Rcpp`, `Matrix` and `RJSONIO` served to R 4.4 from 4.5 builds. Two causes, both now fixed (#205 here, #129 in bincraft) — but the repair itself also needs redoing, and the lists it ran from no longer exist. Those lists came from a one-off local symbol scan, seeded from a candidate set derived on a single slot. That made them a lower bound by construction: the scan found 846 package-slot pairs, while inspecting every object found **4077** unsafe. `abi_flat_safety` now holds the authoritative answer for every slot, written by the flat-safety backfill from the binaries themselves: ``` 95418 inspected, 91325 safe, 4077 unsafe (4.3%) ``` against the 93% of compiled packages the source classifier suspects. ## What it does - `local/list-unsafe-flat-objects.R` prints the objects recorded as unable to load under some minor, for one slot. - The workflow rebuilds each under **every** R minor installed in the image, with **no `--sensitive-only` gate**. Since bincraft v5.3.0 the destination is decided by the built binary's symbols, so a package needing a per-minor slot lands there and a portable one lands in the flat slot. Gating on the source classifier is exactly what let `Rcpp` slip through. ## Safety It **fails loudly when a slot has no verdicts at all**, rather than printing an empty list. An empty repair list is indistinguishable from "nothing to repair" and would read as success while doing nothing — the same shape of silent failure as the missing `PGPASS`. `crow lint` passes. ## After this Run it fleet-wide, re-index, then re-run the reprex. The routed paths are the ones that must come back clean; the `flat` form's failures are the separate prune question. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/206 --- .crow/repair-unsafe-flat-objects.yaml | 183 ++++++++++++++++++++++++++ local/list-unsafe-flat-objects.R | 80 +++++++++++ 2 files changed, 263 insertions(+) create mode 100644 .crow/repair-unsafe-flat-objects.yaml create mode 100644 local/list-unsafe-flat-objects.R diff --git a/.crow/repair-unsafe-flat-objects.yaml b/.crow/repair-unsafe-flat-objects.yaml new file mode 100644 index 0000000..9a57107 --- /dev/null +++ b/.crow/repair-unsafe-flat-objects.yaml @@ -0,0 +1,183 @@ +# Rebuild every flat object recorded as unable to load under some R minor. +# +# The list comes from `abi_flat_safety`, written by the flat-safety backfill +# from the binaries themselves, so it names exactly the objects that fail at +# dyn.load() somewhere rather than everything a source heuristic suspects. The +# heuristic calls 93% of compiled packages risky; the binaries say 4.3%. +# +# Each object is rebuilt under every R minor installed in the image, with no +# `--sensitive-only` gate. Since bincraft v5.3.0 the destination is decided by +# the built binary's symbols in `upload_single_binary()`, so a package needing a +# per-minor slot lands there and a portable one lands in the flat slot. Gating +# on the source classifier is what let Rcpp slip through and break every R 4.4 +# client on the slot. +# +# crow pipeline create devxy/build-cran-binaries --var repair_unsafe=all + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the default + # must match no matrix row. + repair_unsafe: + description: "Repair target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: manual + evaluate: 'repair_unsafe == "all" || repair_unsafe == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Rebuild objects recorded unsafe' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + GIT_USER: pat-s + R_LIBS_USER: /tmp/R-pkgs + NCPUS: 2 + REPAIR_ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /tmp/R-pkgs + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - | + set -eu + CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))') + /opt/R/$R_VERSION/bin/Rscript local/list-unsafe-flat-objects.R \ + "$$REPAIR_ARCH" "$$CODENAME" > /tmp/unsafe.txt + echo "objects to rebuild: $(wc -l < /tmp/unsafe.txt)" + rc=0 + for RBIN in /opt/R/[0-9]*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$$RBIN")")") + RMINOR=$(echo "$$RV" | cut -d. -f1-2) + echo "=== rebuilding under R $$RV ===" + LIB="/tmp/R-pkgs-$$RMINOR" + mkdir -p "$$LIB" + R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" local/install-bincraft.R || true + while read -r pkg ver; do + [ -z "$$pkg" ] && continue + R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" \ + local/build-one.R "$$pkg" "$$ver" || rc=1 + done < /tmp/unsafe.txt + done + exit $$rc + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 1000m diff --git a/local/list-unsafe-flat-objects.R b/local/list-unsafe-flat-objects.R new file mode 100644 index 0000000..a495e83 --- /dev/null +++ b/local/list-unsafe-flat-objects.R @@ -0,0 +1,80 @@ +### Print the flat-slot objects recorded as unable to load under some R minor. +# +# Usage: Rscript local/list-unsafe-flat-objects.R +# +# Prints one ` ` per line, for a caller that rebuilds each +# under every installed R minor. +# +# The verdicts come from `abi_flat_safety`, written by the flat-safety backfill +# from the binaries themselves: an object is unsafe when it references a symbol +# some supported minor does not export. That is the authoritative list of what +# needs a per-minor build, and it replaces the ad-hoc symbol scans that produced +# earlier repair lists. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2L) { + stop("usage: list-unsafe-flat-objects.R ", call. = FALSE) +} +arch <- args[[1L]] +codename <- args[[2L]] + +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432L, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE) + +rows <- DBI::dbGetQuery( + con, + paste0( + "SELECT package, version FROM abi_flat_safety ", + "WHERE NOT safe AND codename = $1 AND arch = $2 ", + "ORDER BY package, version" + ), + params = list(codename, arch) +) + +# An empty result is ambiguous: either nothing is unsafe, or the backfill never +# ran for this slot. Say which, rather than let the caller read silence as done. +if (nrow(rows) == 0L) { + total <- DBI::dbGetQuery( + con, + "SELECT count(*) AS n FROM abi_flat_safety WHERE codename = $1 AND arch = $2", + params = list(codename, arch) + )$n[[1L]] + if (total == 0L) { + stop( + sprintf( + "No verdicts recorded for %s/%s at all. Run the flat-safety backfill first; an empty repair list here would look like success.", # nolint + arch, + codename + ), + call. = FALSE + ) + } + message(sprintf( + "%s/%s: %d objects recorded, none unsafe. Nothing to repair.", + arch, + codename, + total + )) + quit(status = 0) +} + +message(sprintf( + "%s/%s: %d objects need a per-minor build", + arch, + codename, + nrow(rows) +)) +cat(sprintf("%s %s", rows$package, rows$version), sep = "\n") From e3b7212eaf32f5d26b8f6b3718b91c2c728566e4 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 9 Sep 2026 10:33:41 +0000 Subject: [PATCH 096/118] feat(abi): skip objects that already have the per-minor builds they need (#207) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why The fleet repair (12410) was interrupted after 14 of 18 slots, having done: ``` 2522 objects enumerated | 2568 uploads | 1451 symbol overrides ``` Resuming would repeat all of it. The verdict in `abi_flat_safety` describes the **flat** object, and building a per-minor copy does not change that object, so the repair list is identical on the next run. Roughly 2500 objects of duplicated compute. ## What changed `list-unsafe-flat-objects.R` now drops rows that are already repaired: an object counts as repaired once a per-minor build exists for **every** minor that cannot load the flat one. - The bucket is the source of truth, listed directly (it cannot be routed). - Where `unsupported` is empty — unsafe without a recorded reason — a build under every minor the image carries is required, rather than assuming one suffices. - It reports how many were skipped, so an unexpectedly short list is visible rather than silent. - It **fails when the bucket cannot be listed**, because an empty listing would make every object look unrepaired and silently restore the full rebuild. ## Verification The predicate, exercised directly: ``` Rcpp needs[4.4,4.6] -> SKIP (repaired) Rcpp needs[4.4,4.5,4.6] -> REBUILD (4.5 missing) yaml needs[4.4] -> SKIP (repaired) lobstr needs[] -> REBUILD (no recorded minors: all required) ``` ## Note ~70 packages fail to build regardless — `glmmTMB`, `tmbstan`, `OpenMx`, `Cyclops`, `RSiena`, `pdftools`, `odbc` among them — all on `uvr sync --install-system-deps`. They keep their unsafe flat binaries and stay broken for the minors they cannot load under. That is a missing-system-dependency problem in the build images, not an ABI one, and needs separate attention. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/207 --- local/list-unsafe-flat-objects.R | 81 +++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/local/list-unsafe-flat-objects.R b/local/list-unsafe-flat-objects.R index a495e83..1937f99 100644 --- a/local/list-unsafe-flat-objects.R +++ b/local/list-unsafe-flat-objects.R @@ -37,13 +37,92 @@ on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE) rows <- DBI::dbGetQuery( con, paste0( - "SELECT package, version FROM abi_flat_safety ", + "SELECT package, version, unsupported FROM abi_flat_safety ", "WHERE NOT safe AND codename = $1 AND arch = $2 ", "ORDER BY package, version" ), params = list(codename, arch) ) +# Drop what is already repaired. The verdict describes the *flat* object, and a +# per-minor build does not change it, so a resumed run would otherwise rebuild +# everything it did last time: ~2500 objects of duplicated compute after the +# first fleet run was interrupted. +# +# An object is repaired once a per-minor build exists for every minor that +# cannot load the flat one. The bucket is the source of truth for that, and it +# cannot be routed. +if (nrow(rows) > 0L) { + s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE + ) + contrib <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", + arch, + codename + ) + objects <- tryCatch( + s3fs::s3_dir_ls(contrib, recurse = TRUE), + error = function(e) character(0) + ) + if (length(objects) == 0L) { + stop( + sprintf( + "Could not list %s; refusing to guess what is already repaired.", + contrib + ), + call. = FALSE + ) + } + present <- sub(sprintf("^(s3://)?%s/", contrib), "", objects) + + # No recorded minors means unsafe without a reason, so require a build under + # every minor the image carries rather than assume one is enough. + installed <- sub( + "^([0-9]+[.][0-9]+).*$", + "\\1", + basename(list.dirs("/opt/R", recursive = FALSE)) + ) + installed <- unique(installed[grepl("^[0-9]+[.][0-9]+$", installed)]) + + repaired <- vapply( + seq_len(nrow(rows)), + function(i) { + u <- rows$unsupported[[i]] + minors <- if (is.na(u) || !nzchar(u)) { + installed + } else { + trimws(strsplit(u, ",", fixed = TRUE)[[1L]]) + } + if (length(minors) == 0L) { + return(FALSE) + } + wanted <- sprintf( + "%s/%s_%s.tar.gz", + minors, + rows$package[[i]], + rows$version[[i]] + ) + all(wanted %in% present) + }, + logical(1L) + ) + if (any(repaired)) { + message(sprintf( + "%s/%s: %d of %d already have per-minor builds for every minor that needs one; skipping those.", + arch, + codename, + sum(repaired), + nrow(rows) + )) + } + rows <- rows[!repaired, , drop = FALSE] +} + # An empty result is ambiguous: either nothing is unsafe, or the backfill never # ran for this slot. Say which, rather than let the caller read silence as done. if (nrow(rows) == 0L) { From 4342fa773a3dce111ff7eff1515db7c37be099c4 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 9 Sep 2026 10:55:15 +0000 Subject: [PATCH 097/118] fix(abi): do not error on the nothing-to-repair path (#208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Pipeline 12480 shows this for slots with nothing to do: ``` Error in sprintf("%s/%s: %d objects recorded, none unsafe. Nothing to repair.", : ``` `RPostgres` returns `count(*)` as `integer64`, and `%d` rejects it. So a slot in the **best possible state** — verdicts recorded, nothing unsafe — failed its pipeline step, while slots with real work to do succeeded. Exactly the wrong signal. ## What changed Formats the count as a string. One line, plus a comment recording why `%d` is wrong here. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/208 --- local/list-unsafe-flat-objects.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/local/list-unsafe-flat-objects.R b/local/list-unsafe-flat-objects.R index 1937f99..60649a2 100644 --- a/local/list-unsafe-flat-objects.R +++ b/local/list-unsafe-flat-objects.R @@ -141,11 +141,13 @@ if (nrow(rows) == 0L) { call. = FALSE ) } + # `count(*)` arrives as a double (or integer64), and `%d` rejects both, so the + # nothing-to-do path used to die on its own success message. message(sprintf( - "%s/%s: %d objects recorded, none unsafe. Nothing to repair.", + "%s/%s: %s objects recorded, none unsafe. Nothing to repair.", arch, codename, - total + format(as.numeric(total), scientific = FALSE) )) quit(status = 0) } From af4cc6f414545922298e3b08df40e264abd80e19 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 9 Sep 2026 12:52:16 +0000 Subject: [PATCH 098/118] feat(ci): verify what clients actually receive from every published slot (#209) ## Why Everything built so far reasons about the *binaries*: `abi_flat_safety` records what an object's symbols say it can load under, and the index writer drops records on that verdict. None of it proves what a **client** actually receives, which is the product of the CDN rewrite, the per-minor indexes and the `Path:` field acting together. The reprex that found the `base64enc` failure ran on a workstation, could only reach amd64, and did not survive the session. This makes it a pipeline. ## What it does For every slot, under every R minor the image carries, it installs a fixed package set through each URL form the website documents and checks that each one loads in a fresh subprocess. | form | repository | user agent | | --- | --- | --- | | `arch` | `https://cran.rpkgs.com///latest` | R's own | | `generic` | `https://cran.rpkgs.com/` | the string the website's snippet builds | | `flat` | the arch URL | `curl/8.5.0`, so the edge cannot read an R minor | `arch` and `generic` are contractual, so a failure there fails the run. `flat` measures what a client hiding its R minor is exposed to: it is reported and never fatal, because pruning the flat slot is a separate decision. Two details that matter: - **CRAN is deliberately absent from `repos`.** A package missing from our index must fail loudly rather than be quietly compiled from source, which would hide exactly the gap this exists to find. - **One process per form.** `available.packages()` caches its index in `tempdir()` keyed by repository URL, and `arch` and `flat` share a URL. In a single process the second form would read the first form's index and prove nothing. ## Verification Run against the live repository in `build-env-ubuntu:noble` under R 4.6.0, mid-repair: ``` amd64/noble R 4.6 via arch: 11 of 11 packages usable amd64/noble R 4.6 via generic: 11 of 11 packages usable amd64/noble R 4.6 via flat: 8 of 11 packages usable ``` The three `flat` failures are `Rcpp`, `data.table` and `yaml`, each `Built: R 4.4.3` and each failing `dyn.load()` under 4.6. That is the unrouted-client exposure, measured rather than estimated. The rows also show the routed forms serving a mix of per-minor and flat objects (`Rcpp` at `R 4.6.0`, `stringi` at `R 4.4.3`), which is the intended behaviour: only objects that cannot load cross-minor get a per-minor copy. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/209 --- .crow/verify-repo-routing.yaml | 156 +++++++++++++++++++++++++++ local/verify-repo-routing.R | 190 +++++++++++++++++++++++++++++++++ 2 files changed, 346 insertions(+) create mode 100644 .crow/verify-repo-routing.yaml create mode 100644 local/verify-repo-routing.R diff --git a/.crow/verify-repo-routing.yaml b/.crow/verify-repo-routing.yaml new file mode 100644 index 0000000..07e67ec --- /dev/null +++ b/.crow/verify-repo-routing.yaml @@ -0,0 +1,156 @@ +# Install a fixed set of packages from a published slot under every R minor the +# image carries, through each URL form the website tells people to configure, +# and check that each one loads. +# +# This is the end-to-end counterpart to the ABI bookkeeping: `abi_flat_safety` +# records what the binaries say about themselves, this proves what a client +# actually receives once the CDN, the per-minor indexes and the `Path:` field +# have had their say. It runs on the build agents rather than a workstation so +# arm64 is covered on real hardware. +# +# crow pipeline create devxy/build-cran-binaries --var verify_routing=all + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the default + # must match no matrix row. + verify_routing: + description: "Verify target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: manual + evaluate: 'verify_routing == "all" || verify_routing == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Verify routing serves loadable binaries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + VERIFY_ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - | + set -eu + CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))') + rc=0 + for RBIN in /opt/R/[0-9]*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$$RBIN")")") + for FORM in arch generic flat; do + echo "=== R $$RV via $$FORM ===" + "$(dirname "$$RBIN")/Rscript" local/verify-repo-routing.R \ + "$$VERIFY_ARCH" "$$CODENAME" "$$FORM" || rc=1 + done + done + exit $$rc + backend_options: + docker: + resources: + requests: + memory: 4Gi + cpu: 1000m diff --git a/local/verify-repo-routing.R b/local/verify-repo-routing.R new file mode 100644 index 0000000..4e984f6 --- /dev/null +++ b/local/verify-repo-routing.R @@ -0,0 +1,190 @@ +# Check that a published slot serves an installable, loadable binary for the R +# running this script, through one of the URL forms a user can configure. +# +# Rscript local/verify-repo-routing.R
+# +# One process per form on purpose: `available.packages()` caches its index in +# `tempdir()` keyed by repository URL, and the `arch` and `flat` forms share a +# URL. In a single process the second form would silently read the first form's +# index and the run would prove nothing. +# +# Forms: +# arch https://cran.rpkgs.com///latest with R's own agent +# generic https://cran.rpkgs.com/ with an agent naming the distribution +# flat the arch URL with a curl agent, so the edge cannot read an R minor +# from the request and the flat slot answers instead of a per-minor +# one +# +# `arch` and `generic` are contractual: they are what the website tells people +# to configure, so a failure there fails the run. `flat` measures what a client +# hiding its R minor is exposed to; it is reported and never fatal. + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) != 3L) { + stop("usage: verify-repo-routing.R ", call. = FALSE) +} +arch <- args[[1L]] +codename <- args[[2L]] +form <- args[[3L]] + +# Compiled packages that exercise the R C API broadly, plus one pure-R package +# as a control: if `brew` fails too, the problem is the repository rather than +# the ABI. +PACKAGES <- c( + "Rcpp", + "data.table", + "stringi", + "xml2", + "curl", + "jsonlite", + "digest", + "base64enc", + "yaml", + "lobstr", + "brew" +) + +arch_url <- sprintf("https://cran.rpkgs.com/%s/%s/latest", arch, codename) + +# The agent the website's generic snippet builds, so this exercises the string +# real users send rather than an approximation of it. +generic_agent <- function() { + os_release <- readLines("/etc/os-release", warn = FALSE) + field <- function(key) { + line <- grep(sprintf("^%s=", key), os_release, value = TRUE) + if (length(line) == 0L) { + return("") + } + gsub('"', "", sub("^[^=]+=", "", line[[1L]])) + } + os <- paste(field("NAME"), field("VERSION_ID")) + codename_id <- field("VERSION_CODENAME") + if (nzchar(codename_id)) { + os <- sprintf("%s; codename=%s", os, codename_id) + } + sprintf( + "R/%s (%s) (%s)", + getRversion(), + os, + paste(R.version["platform"], R.version["arch"], R.version["os"]) + ) +} + +settings <- switch( + form, + arch = list(repo = arch_url, agent = getOption("HTTPUserAgent")), + generic = list(repo = "https://cran.rpkgs.com/", agent = generic_agent()), + flat = list(repo = arch_url, agent = "curl/8.5.0"), + stop(sprintf("unknown form '%s'", form), call. = FALSE) +) + +# No CRAN in `repos`: a package missing from our index must fail loudly rather +# than be quietly compiled from source, which would hide exactly the gap this +# script exists to find. +options(HTTPUserAgent = settings$agent, repos = c(rpkgs = settings$repo)) + +r_minor <- paste(getRversion()[1L, 1:2], collapse = ".") +rscript <- file.path(R.home("bin"), "Rscript") + +emit <- function(package, status, built = "", secs = NA_real_) { + cat( + sprintf( + "ROW\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + form, + arch, + codename, + r_minor, + package, + status, + sprintf("%s|%.1fs", built, secs) + ) + ) +} + +# `library()` in this process would abort the whole run on a bad `dyn.load()`, +# and the point of the exercise is to survive that and report it. +loads <- function(package, lib) { + status <- suppressWarnings(system2( + rscript, + c("-e", shQuote(sprintf('library("%s")', package))), + env = sprintf("R_LIBS=%s", lib), + stdout = FALSE, + stderr = FALSE + )) + identical(as.integer(status), 0L) +} + +built_field <- function(package, lib) { + desc <- file.path(lib, package, "DESCRIPTION") + if (!file.exists(desc)) { + return("") + } + value <- read.dcf(desc, fields = "Built")[[1L]] + if (is.na(value)) "" else sub(";.*$", "", value) +} + +available <- tryCatch( + rownames(available.packages()), + error = function(e) { + message(sprintf( + "index unreadable at %s: %s", + settings$repo, + conditionMessage(e) + )) + character() + } +) + +failures <- 0L +for (package in PACKAGES) { + if (!package %in% available) { + emit(package, "MISSING_FROM_INDEX") + failures <- failures + 1L + next + } + + lib <- file.path(tempdir(), sprintf("lib-%s-%s", form, package)) + dir.create(lib, showWarnings = FALSE, recursive = TRUE) + started <- Sys.time() + ok <- tryCatch( + { + # Hard dependencies come from the same repository and must be present + # for `R CMD INSTALL` to succeed, so they are installed too; Suggests are + # not, since a missing one does not stop a package loading. + install.packages( + package, + lib = lib, + dependencies = c("Depends", "Imports", "LinkingTo"), + quiet = TRUE + ) + package %in% rownames(installed.packages(lib.loc = lib)) + }, + error = function(e) FALSE + ) + secs <- as.numeric(difftime(Sys.time(), started, units = "secs")) + + if (!ok) { + emit(package, "INSTALL_FAILED", secs = secs) + failures <- failures + 1L + } else if (!loads(package, lib)) { + emit(package, "LOAD_FAILED", built_field(package, lib), secs) + failures <- failures + 1L + } else { + emit(package, "OK", built_field(package, lib), secs) + } + unlink(lib, recursive = TRUE, force = TRUE) +} + +message(sprintf( + "%s/%s R %s via %s: %d of %d packages usable", + arch, + codename, + r_minor, + form, + length(PACKAGES) - failures, + length(PACKAGES) +)) + +if (failures > 0L && !identical(form, "flat")) { + quit(status = 1L) +} From a84cca19492a5dcb6cfbe14d077b32c7c3445d3a Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 9 Sep 2026 15:01:21 +0000 Subject: [PATCH 099/118] fix(abi): prune the objects recorded broken, not every risky package (#211) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why A dry run of the prune on `ubuntu-2404-amd64` (pipeline 12494): ``` Dry run: 3013 object(s) left in place. Re-run with --apply to delete. ``` 3013 objects on one slot. The recorded verdicts say roughly **230** of that slot's objects actually fail `dyn.load()` anywhere. The gap is the selector: the prune used `abi_cache_risky_packages()`, the source classifier, which calls **93%** of compiled packages risky where the binaries' own symbols say **4.3%**. Applying it would have deleted ~2800 working binaries and turned each into a source fallback for every client that cannot be routed by R minor. This is the same mistake #129 fixed in the index writer. The prune was written before the per-object verdicts existed (#200, before #128/#129) and was never brought forward. ## What changed - Selection comes from `abi_flat_safety`, every `NOT safe` row for the slot regardless of which minor it fails under, since the flat object is served to clients on all of them. - Matching is by **object**, `_`, not by package name. A package can have a safe flat binary at one version and an unsafe one at another; the old code deleted both. - The explicit `[pkg ...]` override is unchanged in behaviour and still matches by package name, which is what you want when naming a package by hand. - The refusal guard moves with the selector. An empty result now means the slot's flat-safety backfill never ran — the exact state rhel8 was found in today — and pruning nothing while reporting success would have hidden it. ## Not changed Still a dry run unless `PRUNE_APPLY=yes`, and still refuses to delete an object whose package CRAN no longer ships, since there would be no source to fall back to. ## Ordering note A prune converts each deleted binary into a source fallback. bincraft #131 fixes source fallbacks of *archived* versions being advertised as binaries, which breaks `uvr` outright. That should land before any large prune, or this will manufacture new instances of it. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/211 --- .crow/prune-stale-flat-binaries.yaml | 21 ++++--- local/prune-stale-flat-binaries.R | 83 ++++++++++++++++++++-------- 2 files changed, 75 insertions(+), 29 deletions(-) diff --git a/.crow/prune-stale-flat-binaries.yaml b/.crow/prune-stale-flat-binaries.yaml index a759daf..500f008 100644 --- a/.crow/prune-stale-flat-binaries.yaml +++ b/.crow/prune-stale-flat-binaries.yaml @@ -1,11 +1,18 @@ -# Remove flat-slot binaries belonging to ABI-risky packages, then re-index. +# Remove flat-slot binaries recorded unable to load under some R minor, then +# re-index. # -# `upload_single_binary()` writes an ABI-risky package only to the per-minor -# slot, so a risky package should have no flat binary at all. Packages built -# before they were classified risky still do, and the flat slot is what every -# client that cannot be routed by R minor receives. That binary was built under -# whichever R was primary at the time, so it dies at load under any minor that -# dropped a symbol it uses: +# The list comes from `abi_flat_safety`, so it names the objects the binaries +# themselves report as broken, not the ones a source heuristic suspects. That +# distinction is the whole cost of the operation: the heuristic calls 93% of +# compiled packages risky and would delete roughly 3000 objects per slot to fix +# about 230. +# +# `upload_single_binary()` has written a non-portable package only to the +# per-minor slot since v5.3.0, so this is a one-time cleanup of what was +# uploaded before that. The flat slot is what every client that cannot be +# routed by R minor receives, and such a binary was built under whichever R was +# primary at the time, so it dies at load under any minor that dropped a symbol +# it uses: # # base64enc.so: undefined symbol: SETLENGTH # diff --git a/local/prune-stale-flat-binaries.R b/local/prune-stale-flat-binaries.R index dbb0ab1..aea97b4 100644 --- a/local/prune-stale-flat-binaries.R +++ b/local/prune-stale-flat-binaries.R @@ -1,14 +1,23 @@ -### Remove flat-slot binaries of packages that are ABI-risky. +### Remove flat-slot binaries recorded as unable to load under some R minor. # # Usage: # Rscript local/prune-stale-flat-binaries.R [--apply] [pkg ...] # -# `upload_single_binary()` writes an ABI-risky package only to the per-minor -# slot, never to the flat one, so a risky package should have no flat binary at -# all. Packages built before they were classified risky still do, and the flat -# slot is what every client that cannot be routed by R minor receives. That -# binary was built under whichever R was primary at the time, so it fails at -# load under any minor that dropped a symbol it uses: +# The list comes from `abi_flat_safety`, written by the flat-safety backfill +# from the binaries' own symbols, so it names exactly the objects that fail at +# dyn.load() somewhere. It deliberately does *not* use +# `abi_cache_risky_packages()`: that source heuristic calls 93% of compiled +# packages risky where the binaries say 4.3%, and pruning on it would delete +# roughly 3000 working objects per slot to fix about 230 broken ones. Selecting +# per object rather than per package is the same correction #129 made to the +# index writer. +# +# `upload_single_binary()` has written a non-portable package only to the +# per-minor slot since v5.3.0, so this is a one-time cleanup of objects +# uploaded before that, not a recurring sweep. The flat slot is what every +# client that cannot be routed by R minor receives, and such a binary was built +# under whichever R was primary at the time, so it fails at load under any +# minor that dropped a symbol it uses: # # base64enc.so: undefined symbol: SETLENGTH # @@ -60,27 +69,57 @@ flat_files <- relative[is_flat] flat_pkg <- sub("_.*$", "", flat_files) if (length(explicit) > 0L) { - risky <- explicit - cat(sprintf("Restricted to %d named package(s)\n", length(risky))) + doomed <- flat_files[flat_pkg %in% explicit] + cat(sprintf( + "%s/%s: %d flat binaries, %d belong to the %d named package(s)\n", + arch, + codename, + length(flat_files), + length(doomed), + length(explicit) + )) } else { - risky <- bincraft:::abi_cache_risky_packages() - cat(sprintf("Classifier cache reports %d risky package(s)\n", length(risky))) - if (length(risky) == 0L) { + con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432L, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" + ) + on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE) + + # Every recorded-unsafe row, whichever minor it fails under: the flat object + # is served to clients on all of them, so one failing minor condemns it. + rows <- DBI::dbGetQuery( + con, + paste0( + "SELECT package, version FROM abi_flat_safety ", + "WHERE NOT safe AND codename = $1 AND arch = $2" + ), + params = list(codename, arch) + ) + if (nrow(rows) == 0L) { stop( - "The classifier cache returned nothing. Refusing to proceed: an empty risky set here is indistinguishable from an unreachable cache, and would silently prune nothing while reporting success.", # nolint + "No unsafe objects are recorded for this slot. Refusing to proceed: that is indistinguishable from a slot whose flat-safety backfill never ran, and pruning nothing while reporting success would hide it.", # nolint call. = FALSE ) } -} -doomed <- flat_files[flat_pkg %in% risky] -cat(sprintf( - "%s/%s: %d flat binaries, %d belong to ABI-risky packages\n", - arch, - codename, - length(flat_files), - length(doomed) -)) + # Match the object, not the package. A package can have a safe flat binary at + # one version and an unsafe one at another, and deleting both would throw + # away a working binary. + unsafe_objects <- sprintf("%s_%s", rows$package, rows$version) + doomed <- flat_files[sub("\\.tar\\.gz$", "", flat_files) %in% unsafe_objects] + cat(sprintf( + "%s/%s: %d flat binaries, %d recorded unable to load under some R minor\n", + arch, + codename, + length(flat_files), + length(doomed) + )) +} # Never remove the last artifact of a package. The source fallback republishes # the CRAN source under this key, but only for packages CRAN still ships, so From 03e007069a70fdb2f3ba307b0ea74dbfdd5af1d8 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 9 Sep 2026 15:01:29 +0000 Subject: [PATCH 100/118] fix(ci): do not print a missing timing as "NAs" (#210) ## Why A `MISSING_FROM_INDEX` row has no install to time, and `sprintf("%.1fs", NA_real_)` renders it as the literal `NAs`: ``` ROW arch amd64 alpine323 4.5 Rcpp MISSING_FROM_INDEX |NAs ``` Read quickly, `NAs` looks like a unit rather than a missing measurement. ## What changed Rows without a timing print the `built` field alone. Every other row is unchanged. ## Where it showed up Pipeline 12492, the deliberate failure-path check against a slot known to have gaps. That run is otherwise a good result: it caught `Rcpp` missing from the alpine323 4.5 and 4.6 indexes on both contractual URL forms, and it caught the reverse-direction ABI break on the flat slot (objects built under R 4.5.3 failing `dyn.load()` under 4.4), which no previous reprex had surfaced. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/210 --- local/verify-repo-routing.R | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/local/verify-repo-routing.R b/local/verify-repo-routing.R index 4e984f6..c19494d 100644 --- a/local/verify-repo-routing.R +++ b/local/verify-repo-routing.R @@ -96,7 +96,10 @@ emit <- function(package, status, built = "", secs = NA_real_) { r_minor, package, status, - sprintf("%s|%.1fs", built, secs) + # `%.1f` renders NA as the literal "NAs", which reads as a unit rather + # than as a missing measurement. A package that was never in the index + # has no timing at all. + if (is.na(secs)) built else sprintf("%s|%.1fs", built, secs) ) ) } From 523424bd68775557e18439928e2e481ededf70c1 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 9 Sep 2026 18:35:33 +0000 Subject: [PATCH 101/118] fix(abi): inspect flat objects of packages CRAN has removed (#212) ## Why The backfill decided which flat objects were worth inspecting by looking the package up in **CRAN's current index**: ```r compiled_names <- cran[NeedsCompilation == "yes", "Package"] keep <- flat_pkg %in% compiled_names ``` CRAN lists only what it currently ships. A package CRAN has **archived** is absent from that index, so it is never recognised as compiled and never inspected, and no verdict is ever recorded for it. `union_index_records()` meanwhile reads the slot's *own* `NeedsCompilation`, finds `yes`, and drops the record because the source classifier flags the package and no safe verdict exists to clear it. The repair cannot rescue it either: it works from recorded verdicts, and there is no row. The result is unrecoverable, and identical in shape to #132 but from a different cause: dropped from every per-minor index, on every re-index, forever. ## Evidence `mixedMem` and `vcpen` survived a backfill, two re-indexes and a repair while missing from all three rhel8 per-minor indexes on **both** architectures: ``` rhel8 amd64 4.5: missing=3 [mixedMem timbr vcpen] rhel8 arm64 4.5: missing=2 [mixedMem vcpen] ``` Neither appears anywhere in either repair log, because neither was ever in the unsafe set. Their per-minor objects are 404 at every minor. And in CRAN's index: ``` mixedMem CRAN version=ABSENT NeedsCompilation=ABSENT vcpen CRAN version=ABSENT NeedsCompilation=ABSENT ``` Both removed from CRAN. Their flat records in our own index say `NeedsCompilation: yes`. ## What changed `NeedsCompilation` now comes from the slot's own flat index, which describes exactly the objects being inspected rather than a different repository's opinion of them. - Fetched **from the bucket**, not over HTTPS. This runs inside R, so the edge would route the request to a per-minor index and the flat objects would not all be there. Same reason the object listing above it comes from S3 (#203). - Keyed on `_`, not the package name: a slot can carry several versions and only some of them compile. - Refuses to proceed on an empty read, so an unreadable index cannot silently mean "nothing is compiled" and skip the entire slot. ## Effect Re-running the backfill on a slot will now inspect these objects and record verdicts, after which the index writer keeps the safe ones and the repair can act on the unsafe ones. Already-recorded objects are still skipped, so re-running stays cheap. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/212 --- local/backfill-flat-safety.R | 50 ++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/local/backfill-flat-safety.R b/local/backfill-flat-safety.R index a8a8bbe..59faa3c 100644 --- a/local/backfill-flat-safety.R +++ b/local/backfill-flat-safety.R @@ -75,19 +75,47 @@ flat_pkg <- sub("_.*$", "", flat_files) flat_ver <- sub("\\.tar\\.gz$", "", sub("^[^_]*_", "", flat_files)) # Only compiled packages can fail at dyn.load(), so the rest are not worth -# downloading. CRAN's own index says which; it is not behind our edge. -cran <- read.dcf( - gzcon(url( - "https://cloud.r-project.org/src/contrib/PACKAGES.gz", - open = "rb" - )), - fields = c("Package", "NeedsCompilation") +# downloading. Ask the slot's own index rather than CRAN's: CRAN lists only the +# versions it currently ships, so a package it has archived is absent, is never +# recognised as compiled, and is never inspected. No verdict is ever recorded +# for it, while `union_index_records()` reads the slot's own `NeedsCompilation` +# and happily drops it -- with nothing able to clear the flag, because the +# repair works from recorded verdicts too. `mixedMem` and `vcpen`, both removed +# from CRAN, were stuck out of every rhel8 per-minor index that way. +# +# Read it from the bucket, not over HTTPS: this runs inside R, so the edge would +# route the request to a per-minor index and the flat objects would not all be +# there. Same reason the object listing above comes from S3. +index_file <- file.path(tempdir(), "flat-PACKAGES.gz") +s3fs::s3_file_download( + sprintf("%s/PACKAGES.gz", contrib), + index_file, + overwrite = TRUE ) -compiled_names <- cran[ - !is.na(cran[, "NeedsCompilation"]) & cran[, "NeedsCompilation"] == "yes", - "Package" +index <- read.dcf( + gzfile(index_file), + fields = c("Package", "Version", "NeedsCompilation") +) +if (nrow(index) == 0L) { + stop( + sprintf( + "Read no records from %s/PACKAGES.gz; refusing to guess what is compiled.", + contrib + ), + call. = FALSE + ) +} +# Key on the object, not the package: a slot can carry several versions and +# only some of them compile. +compiled_objects <- paste( + index[, "Package"], + index[, "Version"], + sep = "_" +)[ + !is.na(index[, "NeedsCompilation"]) & + tolower(index[, "NeedsCompilation"]) == "yes" ] -keep <- flat_pkg %in% compiled_names +keep <- paste(flat_pkg, flat_ver, sep = "_") %in% compiled_objects compiled <- data.frame( Package = flat_pkg[keep], Version = flat_ver[keep], From 416a2d64d9461395f689ca3ccd5e7f6aac615913 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 11 Sep 2026 07:27:03 +0000 Subject: [PATCH 102/118] chore(alpine): drop the deprecated Alpine 3.22 slot (#213) ## Why Alpine 3.22 is deprecated and should be dropped. It was never a published slot to begin with. `local.rpkgs_slots` in `cdn.tf` enumerates eight distros and alpine322 is not one of them, so `UNION_SLOTS` never gated per-minor rewriting for it. The website's `DistroId` union lists the same eight. Every request to the slot therefore fell through to the flat index, which holds R 4.5 builds. ## Evidence The fleet-wide routing verification (pipeline 12591, `verify_routing=all`) measured this end to end. Across all 16 published slots, both architectures, three R minors and both contractual URL forms: **1056 of 1056 install-and-load checks passed.** Every failure in the run was alpine322, identical on amd64 and arm64: ``` amd64/alpine322 R 4.4 via arch|generic|flat: 8 of 11 amd64/alpine322 R 4.6 via arch|generic|flat: 10 of 11 arm64/alpine322 R 4.4 via arch|generic|flat: 8 of 11 arm64/alpine322 R 4.6 via arch|generic|flat: 10 of 11 ``` The tell is that all three forms agree. On a routed slot `arch` and `generic` diverge from `flat`, because that divergence *is* the routing. On alpine322 there is nothing to diverge from. R 4.5 clients were 11 of 11 on every form, because flat happens to be what they need. The data itself is not broken, only unreachable. `alpine322/contrib/4.4/data.table_1.18.6.1.tar.gz` is genuinely `Built: R 4.4.3` and the 4.6 object is `Built: R 4.6.0`. Correct objects, correct per-minor indexes, never served. ## What changed - alpine-322 matrix rows and dropdown options removed from the 12 pipelines that carried them, so `=all` no longer fans out to a slot nobody can reach. - `3.22` removed from the `OS_VERSION` dropdowns in `build-all-versions.yaml` and `repair-built-stamp.yaml`; the latter's default moves to `3.23`. - Stale `3.22` examples updated in `local/packages-to-build.R`. `crow lint .crow/` reports all 18 configs valid. ## Also done outside this diff The six alpine-322 cron jobs are deleted from the Crow server, since a cron whose matrix row is gone keeps firing against nothing: ``` 5 process-cran-updates-alpine-322-amd64 0 5/8 * * * 6 process-cran-updates-alpine-322-arm64 30 5/8 * * * 68 weekly-audit-missing-alpine-322-amd64 20 0 * * 0 69 weekly-audit-missing-alpine-322-arm64 30 0 * * 0 82 weekly-rebuild-missing-alpine-322-amd64 0 4 * * 0 83 weekly-rebuild-missing-alpine-322-arm64 10 4 * * 0 ``` They are listed here so they can be recreated if this is ever reversed. ## Not done The published objects and indexes under `amd64/alpine322` and `arm64/alpine322` are left in the bucket. Deleting them is a separate decision, and nothing routes to them in the meantime. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/213 --- .crow/archive-missed-packages.yaml | 4 --- .crow/backfill-flat-safety.yaml | 10 ------- .crow/build-all-versions.yaml | 1 - .crow/process-updates.yaml | 14 ---------- .crow/prune-stale-flat-binaries.yaml | 10 ------- .crow/rebuild-abi-gap.yaml | 10 ------- .crow/reindex.yaml | 10 ------- .crow/repair-built-stamp.yaml | 7 +++-- .crow/repair-unsafe-flat-objects.yaml | 10 ------- .crow/trial-build-registry.yaml | 8 ------ .crow/verify-repo-routing.yaml | 10 ------- .crow/weekly-audit-missing.yaml | 10 ------- .crow/weekly-rebuild-missing.yaml | 38 --------------------------- .crow/weekly-rebuild-reindex.yaml | 10 ------- local/packages-to-build.R | 4 +-- 15 files changed, 5 insertions(+), 151 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index bd127d6..71e263e 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -28,10 +28,6 @@ matrix: ARCH: amd64 - CODENAME: redhat-10 ARCH: arm64 - - CODENAME: alpine322 - ARCH: amd64 - - CODENAME: alpine322 - ARCH: arm64 - CODENAME: alpine323 ARCH: amd64 - CODENAME: alpine323 diff --git a/.crow/backfill-flat-safety.yaml b/.crow/backfill-flat-safety.yaml index e5c9b68..0db0735 100644 --- a/.crow/backfill-flat-safety.yaml +++ b/.crow/backfill-flat-safety.yaml @@ -25,8 +25,6 @@ variables: options: - none - all - - alpine-322-amd64 - - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -59,14 +57,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 4306d6d..91b7bcd 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -24,7 +24,6 @@ variables: OS_VERSION: description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' options: - - '3.22' - '3.23' - '3.24' - '8' diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 73bfff7..4cd95a7 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -15,8 +15,6 @@ variables: options: - none - all - - alpine-322-amd64 - - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -48,18 +46,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - OS_ID: alpine322 - PROCESS_NEW: 'FALSE' - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - OS_ID: alpine322 - PROCESS_NEW: 'FALSE' - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/.crow/prune-stale-flat-binaries.yaml b/.crow/prune-stale-flat-binaries.yaml index 500f008..79ca003 100644 --- a/.crow/prune-stale-flat-binaries.yaml +++ b/.crow/prune-stale-flat-binaries.yaml @@ -36,8 +36,6 @@ variables: options: - none - all - - alpine-322-amd64 - - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -76,14 +74,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - CODENAME: alpine322 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - CODENAME: alpine322 - IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 CODENAME: alpine323 diff --git a/.crow/rebuild-abi-gap.yaml b/.crow/rebuild-abi-gap.yaml index 289781d..71ade22 100644 --- a/.crow/rebuild-abi-gap.yaml +++ b/.crow/rebuild-abi-gap.yaml @@ -39,8 +39,6 @@ variables: options: - none - all - - alpine-322-amd64 - - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -80,14 +78,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml index 71bca2c..803d9a5 100644 --- a/.crow/reindex.yaml +++ b/.crow/reindex.yaml @@ -24,8 +24,6 @@ variables: options: - none - all - - alpine-322-amd64 - - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -55,14 +53,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml index e56e1e9..cc09586 100644 --- a/.crow/repair-built-stamp.yaml +++ b/.crow/repair-built-stamp.yaml @@ -16,7 +16,7 @@ # pipeline here gates on a variable named after itself for exactly that reason. # # crow pipeline create --branch main \ -# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \ +# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.23 \ # --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries variables: # Gates this pipeline. A manual pipeline creation instantiates every file in @@ -37,9 +37,8 @@ variables: - ubuntu default: alpine OS_VERSION: - description: 'OS image tag. Must match OS (alpine: 3.22/3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).' + description: 'OS image tag. Must match OS (alpine: 3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).' options: - - '3.22' - '3.23' - '3.24' - '8' @@ -48,7 +47,7 @@ variables: - 'jammy' - 'noble' - 'resolute' - default: '3.22' + default: '3.23' R_VERSION: description: 'R version whose stamp the slot should advertise.' options: diff --git a/.crow/repair-unsafe-flat-objects.yaml b/.crow/repair-unsafe-flat-objects.yaml index 9a57107..e0625c0 100644 --- a/.crow/repair-unsafe-flat-objects.yaml +++ b/.crow/repair-unsafe-flat-objects.yaml @@ -22,8 +22,6 @@ variables: options: - none - all - - alpine-322-amd64 - - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -53,14 +51,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 2195423..7826d3e 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -38,14 +38,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/.crow/verify-repo-routing.yaml b/.crow/verify-repo-routing.yaml index 07e67ec..b20c993 100644 --- a/.crow/verify-repo-routing.yaml +++ b/.crow/verify-repo-routing.yaml @@ -18,8 +18,6 @@ variables: options: - none - all - - alpine-322-amd64 - - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -49,14 +47,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 25875da..2796a47 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -15,8 +15,6 @@ variables: options: - none - all - - alpine-322-amd64 - - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -48,14 +46,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 9639792..955476e 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -25,8 +25,6 @@ variables: options: - none - all - - alpine-322-amd64 - - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -58,42 +56,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 66603e3..698f503 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -20,8 +20,6 @@ variables: options: - none - all - - alpine-322-amd64 - - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -58,14 +56,6 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/local/packages-to-build.R b/local/packages-to-build.R index ead73b8..31b667f 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -12,11 +12,11 @@ library(progressr, quietly = TRUE) suppressPackageStartupMessages(library(data.table)) # Sys.setenv("OS" = "alpine") -# Sys.setenv("OS_VERSION" = "3.22") +# Sys.setenv("OS_VERSION" = "3.23") # Sys.setenv("ARCH" = "arm64") arch <- Sys.getenv("ARCH") -# target: alpine-322, ubuntu-2404, redhat-9, etc. +# target: alpine-323, ubuntu-2404, redhat-9, etc. platform <- paste( Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), From d80d3e814ee5bad1b846fea4c64e663455883fa2 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 11 Sep 2026 13:14:17 +0000 Subject: [PATCH 103/118] feat(slots): add a retirement workflow and a drift check (#214) ## Why Retiring a slot means three separate things in three separate places, and nothing tied them together. Alpine 3.22 is what that costs. It was removed from `local.rpkgs_slots` in `cdn.tf` at some point, so the edge stopped routing it, but it kept: - a matrix row in **twelve** pipelines - **six** cron jobs on the Crow server - roughly **50000 objects** in the bucket Nightly builds kept producing binaries nobody could reach. Worse, an unrouted slot is not inert: every request falls through to the flat index, so R 4.4 and R 4.6 clients got `LOAD_FAILED` at `dyn.load()` rather than simply nothing. It took a fleet-wide `verify_routing=all` run to notice. ## What changed **`local/check-slots.R`** (`just check-slots`) fails when any `.crow/*.yaml` still targets a slot missing from `local.rpkgs_slots`. `cdn.tf` stays the source of truth; the checker reads it rather than restating the list. **`local/retire-slot.R`** (`just retire-slot `) does the cleanup: - strips the slot from every pipeline matrix, dropdown option and `OS_VERSION` tag, and moves a dropdown default that pointed at the retired tag - deletes `amd64//` and `arm64//` from the bucket - prints the cron jobs that are left, with the API call, since they live on the Crow server and cannot be removed from here Dry run by default. It refuses any slot still listed in `cdn.tf`, so it cannot be pointed at a live slot however it is invoked. **`.crow/retire-slot.yaml`** runs the asset half in CI, where the B2 credentials exist. **README** gains a `## Retiring a slot` section with the ordering. Step 4 before step 2 is the one ordering that actively breaks things: the next nightly rebuilds everything just deleted. ## Verification Both scripts were run against the tree as it stood at `523424b`, immediately before #213. The checker names exactly the twelve files that PR edited by hand: ``` .crow/archive-missed-packages.yaml targets unpublished: alpine322 .crow/backfill-flat-safety.yaml targets unpublished: alpine-322 .crow/process-updates.yaml targets unpublished: alpine-322, alpine322 ... 12 file(s) target a slot the edge does not route. ``` `retire-slot.R alpine322 --config-only --apply` reproduces that commit's structural changes byte for byte across all 18 pipeline files, and additionally catches the two `OS_VERSION` dropdowns. The only differences from the hand-made commit are two prose lines, a usage example and a dropdown `description:`, which the script deliberately leaves alone. Guards exercised: a live slot (`alpine323`) is refused, an unknown codename is rejected by name, and the asset half refuses to run without `B2_S3_ACCESS_KEY` rather than treating B2's empty unauthenticated listing as an already-clean slot. `crow lint .crow/` reports all 19 configs valid. `air` and `jarl` are clean on both new scripts. ## Next `alpine321` and `alpine322` still hold objects in the bucket (22922 and 25308 records respectively, both architectures). Once this merges they can be removed with the pipeline this PR adds. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/214 --- .crow/retire-slot.yaml | 72 +++++++++++++ README.md | 22 ++++ justfile | 8 ++ local/check-slots.R | 151 ++++++++++++++++++++++++++ local/retire-slot.R | 240 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 493 insertions(+) create mode 100644 .crow/retire-slot.yaml create mode 100644 local/check-slots.R create mode 100644 local/retire-slot.R diff --git a/.crow/retire-slot.yaml b/.crow/retire-slot.yaml new file mode 100644 index 0000000..1b62ba7 --- /dev/null +++ b/.crow/retire-slot.yaml @@ -0,0 +1,72 @@ +# Delete a retired slot's objects from the bucket. +# +# The config half is a commit, so it runs on a workstation: +# just retire-slot --apply +# +# This half needs the B2 credentials, which only exist in CI. +# `local/retire-slot.R` refuses any slot still listed in `local.rpkgs_slots`, +# so this cannot delete a live slot. Both architectures go together. +# +# Dry run unless RETIRE_APPLY is 'yes': +# +# crow pipeline create devxy/build-cran-binaries --branch main \ +# --var retire_slot=alpine322 --var RETIRE_APPLY=yes + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the + # default must be a value the script never accepts. + retire_slot: + description: "Codename of the slot to retire (e.g. alpine322), or 'none'." + default: none + RETIRE_APPLY: + description: "Set to 'yes' to delete. Anything else reports and changes nothing." + options: + - 'no' + - 'yes' + default: 'no' + +when: + - event: manual + evaluate: 'retire_slot != "none"' + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Delete the retired slot from the bucket' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + R_LIBS_USER: /tmp/R-pkgs + # Pipeline variables are substituted into the YAML, not exported, so they + # have to be bound here to be readable as env vars in the script below. + RETIRE_SLOT: ${retire_slot} + RETIRE_APPLY: ${RETIRE_APPLY} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /tmp/R-pkgs + - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R + - | + set -eu + APPLY="" + if [ "$$RETIRE_APPLY" = "yes" ]; then + APPLY="--apply" + fi + /opt/R/4.5.3/bin/Rscript local/retire-slot.R "$$RETIRE_SLOT" --assets-only $$APPLY + backend_options: + docker: + resources: + requests: + memory: 2Gi + cpu: 1000m diff --git a/README.md b/README.md index 989069e..d137728 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,28 @@ Example: Technically, no date-based snapshots are planned, so this component from the Posit PM structure is not included. +## Retiring a slot + +A slot is only reachable if it is listed in `local.rpkgs_slots` in `cdn.tf`, which feeds `UNION_SLOTS` in `edge/rpkgs-router.ts`. +An unrouted slot serves the flat index to every client, so its binaries fail at `dyn.load()` under every R minor but one. + +In order: + +1. Remove it from `local.rpkgs_slots` in `cdn.tf` and from `DistroId` in the website's `src/src/data/repos.ts`, then `tofu apply`. +2. `just retire-slot --apply`, then commit. +3. Delete its cron jobs on the Crow server; `local/retire-slot.R` prints them with the API call. +4. Delete its objects: + + ```sh + crow pipeline create devxy/build-cran-binaries --branch main \ + --var retire_slot= --var RETIRE_APPLY=yes + ``` + +Steps 2 and 4 refuse any slot still listed in `cdn.tf`. +Running step 4 before step 2 means the next nightly rebuilds everything just deleted. + +`just check-slots` fails when a pipeline still targets an unrouted slot. + ## Helpers Helper scripts are located in `local/`. diff --git a/justfile b/justfile index 6c75b13..36ad0e1 100644 --- a/justfile +++ b/justfile @@ -87,3 +87,11 @@ edge-test: denoland/deno:latest \ deno test -A edge/rpkgs-router.test.ts fi + +# fail if any pipeline still targets a slot the edge does not route +check-slots: + Rscript local/check-slots.R + +# strip a retired slot from .crow/*.yaml (dry run; pass --apply to write) +retire-slot codename *flags: + Rscript local/retire-slot.R {{ codename }} --config-only {{ flags }} diff --git a/local/check-slots.R b/local/check-slots.R new file mode 100644 index 0000000..5345da9 --- /dev/null +++ b/local/check-slots.R @@ -0,0 +1,151 @@ +### Fail when a pipeline targets a slot that is not in `local.rpkgs_slots`. +# +# Rscript local/check-slots.R +# +# That list feeds `UNION_SLOTS`, which `edge/rpkgs-router.ts` gates per-R-minor +# routing on. An unrouted slot serves the flat index to every client, so its +# binaries fail at dyn.load() under every R minor but one. +# +# One-directional: a published slot missing from some pipeline is only +# reported, since several pipelines cover a subset on purpose. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +# Parses the distro list out of the setproduct's second array. +published_codenames <- function(path = "cdn.tf") { + if (!file.exists(path)) { + stop( + sprintf("%s not found; run this from the repository root.", path), + call. = FALSE + ) + } + lines <- readLines(path, warn = FALSE) + start <- grep("rpkgs_slots\\s*=", lines) + if (length(start) != 1L) { + stop( + "Could not find exactly one `rpkgs_slots` local in cdn.tf.", + call. = FALSE + ) + } + block <- paste(lines[start:min(start + 10L, length(lines))], collapse = "\n") + arrays <- regmatches(block, gregexpr("\\[[^]]*\\]", block))[[1L]] + if (length(arrays) < 2L) { + stop( + "`rpkgs_slots` no longer has the two-array setproduct shape this parses.", + call. = FALSE + ) + } + codenames <- gsub( + '"', + "", + regmatches(arrays[[2L]], gregexpr('"[^"]+"', arrays[[2L]]))[[1L]] + ) + if (length(codenames) == 0L) { + stop("Parsed an empty distro list from cdn.tf.", call. = FALSE) + } + codenames +} + +UBUNTU_CODENAMES <- c( + resolute = "ubuntu-2604", + noble = "ubuntu-2404", + jammy = "ubuntu-2204" +) + +as_pipeline_os <- function(codename) { + ifelse( + codename %in% names(UBUNTU_CODENAMES), + UBUNTU_CODENAMES[codename], + ifelse( + grepl("^rhel[0-9]+$", codename), + sub("^rhel", "redhat-", codename), + ifelse( + grepl("^alpine[0-9]+$", codename), + sub("^alpine([0-9])([0-9]+)$", "alpine-\\1\\2", codename), + NA_character_ + ) + ) + ) +} + +# Structural lines only, so a codename in a comment is not a finding. +slots_in_pipeline <- function(path) { + lines <- readLines(path, warn = FALSE) + row <- regmatches( + lines, + regexec("^\\s*-?\\s*(OS|CODENAME|OS_ID):\\s*(\\S+)\\s*$", lines) + ) + opt <- regmatches( + lines, + regexec("^\\s*-\\s*([a-z0-9]+-[a-z0-9]+)-(amd64|arm64)\\s*$", lines) + ) + hits <- c( + vapply( + row, + function(m) if (length(m) == 3L) m[[3L]] else NA_character_, + "" + ), + vapply(opt, function(m) if (length(m) == 4L) m[[2L]] else NA_character_, "") + ) + hits <- hits[!is.na(hits)] + # `OS: alpine` (build-all-versions' bare image name) is not a slot. + unique(hits[grepl("^(alpine|redhat|ubuntu)-?[0-9]", hits)]) +} + +codenames <- published_codenames() +published_os <- unname(as_pipeline_os(codenames)) +if (anyNA(published_os)) { + stop( + sprintf( + "Unrecognised codename(s) in cdn.tf: %s", + paste(codenames[is.na(published_os)], collapse = ", ") + ), + call. = FALSE + ) +} +# Files spell a slot either way, so accept both. +published <- unique(c(published_os, codenames)) + +cat(sprintf( + "Published slots (cdn.tf): %s\n\n", + paste(sort(published_os), collapse = ", ") +)) + +files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE)) +findings <- list() +for (f in files) { + stale <- setdiff(slots_in_pipeline(f), published) + if (length(stale) > 0L) { + findings[[f]] <- stale + cat(sprintf( + "%-45s targets unpublished: %s\n", + f, + paste(sort(stale), collapse = ", ") + )) + } +} + +uncovered <- setdiff( + published_os, + unique(unlist(lapply(files, slots_in_pipeline))) +) +if (length(uncovered) > 0L) { + cat(sprintf( + "\nNote: %s appear(s) in no pipeline at all.\n", + paste(sort(uncovered), collapse = ", ") + )) +} + +if (length(findings) == 0L) { + cat("All pipeline slots are published and routed.\n") + quit(status = 0) +} + +cat(sprintf( + "\n%d file(s) target an unrouted slot. Retire it with:\n just retire-slot \n", + length(findings) +)) +quit(status = 1) diff --git a/local/retire-slot.R b/local/retire-slot.R new file mode 100644 index 0000000..383fd2c --- /dev/null +++ b/local/retire-slot.R @@ -0,0 +1,240 @@ +### Retire a slot: strip it from the pipelines and delete its objects from S3. +# +# Rscript local/retire-slot.R [--apply] [--config-only|--assets-only] +# +# Retiring a distro retires both architectures, because `local.rpkgs_slots` in +# `cdn.tf` is a setproduct over them. +# +# Order: remove the slot from `local.rpkgs_slots` (and the website's +# `DistroId`) and apply that first, then run this. Deleting objects while a +# pipeline still targets the slot just means the next nightly rebuilds them. +# +# Cron jobs live on the Crow server, not here, and are printed at the end. +# +# Only structural YAML is edited. Prose mentioning the slot is left alone, so +# grep for the name afterwards. +# +# Dry run by default. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +args <- commandArgs(trailingOnly = TRUE) +apply_changes <- "--apply" %in% args +config_only <- "--config-only" %in% args +assets_only <- "--assets-only" %in% args +args <- args[!args %in% c("--apply", "--config-only", "--assets-only")] +if (length(args) != 1L || (config_only && assets_only)) { + stop( + "usage: retire-slot.R [--apply] [--config-only|--assets-only]", + call. = FALSE + ) +} +codename <- args[[1L]] + +# The bucket and `CODENAME:` say `alpine322`; `OS:` and the options say +# `alpine-322`. Both have to go. +pipeline_os <- if (grepl("^rhel[0-9]+$", codename)) { + sub("^rhel", "redhat-", codename) +} else if (grepl("^alpine[0-9]+$", codename)) { + sub("^alpine", "alpine-", codename) +} else { + # Single bracket so an unknown codename yields NA, not an error. + c(resolute = "ubuntu-2604", noble = "ubuntu-2404", jammy = "ubuntu-2204")[ + codename + ] +} +if (is.na(pipeline_os) || length(pipeline_os) == 0L) { + stop(sprintf("Unrecognised codename '%s'.", codename), call. = FALSE) +} + +# The `OS_VERSION` dropdowns are keyed on the image tag, not the slot name. +image_tag <- if (grepl("^alpine-", pipeline_os)) { + sub("^alpine-([0-9])([0-9]+)$", "\\1.\\2", pipeline_os) +} else if (grepl("^redhat-", pipeline_os)) { + sub("^redhat-", "", pipeline_os) +} else { + codename +} + +# Refuse a slot that is still routed: deleting its objects would leave the edge +# serving indexes that name files which no longer exist. +tf <- readLines("cdn.tf", warn = FALSE) +start <- grep("rpkgs_slots\\s*=", tf) +if (length(start) != 1L) { + stop( + "Could not find exactly one `rpkgs_slots` local in cdn.tf.", + call. = FALSE + ) +} +block <- paste(tf[start:min(start + 10L, length(tf))], collapse = "\n") +if (grepl(sprintf('"%s"', codename), block, fixed = TRUE)) { + stop( + sprintf( + "'%s' is still listed in `local.rpkgs_slots` in cdn.tf, so the edge still routes it. Remove it there (and from the website's DistroId union) and apply the OpenTofu change first.", # nolint + codename + ), + call. = FALSE + ) +} + +cat(sprintf( + "Retiring %s (pipelines call it '%s', images tag it '%s')\n\n", + codename, + pipeline_os, + image_tag +)) + +## 1. Pipeline definitions ---------------------------------------------------- + +# Line-based on purpose: a YAML round trip would reflow all 18 files. +strip_slot <- function(lines) { + keep <- rep(TRUE, length(lines)) + i <- 1L + while (i <= length(lines)) { + line <- lines[[i]] + row <- regexec( + sprintf( + "^(\\s*)-\\s*(OS|CODENAME):\\s*(%s|%s)\\s*$", + pipeline_os, + codename + ), + line + ) + m <- regmatches(line, row)[[1L]] + if (length(m) == 4L) { + indent <- nchar(m[[2L]]) + keep[[i]] <- FALSE + j <- i + 1L + while (j <= length(lines)) { + nxt <- lines[[j]] + if ( + !nzchar(trimws(nxt)) || nchar(sub("^(\\s*).*$", "\\1", nxt)) <= indent + ) { + break + } + keep[[j]] <- FALSE + j <- j + 1L + } + i <- j + next + } + if (grepl(sprintf("^\\s*-\\s*%s-(amd64|arm64)\\s*$", pipeline_os), line)) { + keep[[i]] <- FALSE + } + if (grepl(sprintf("^\\s*-\\s*'%s'\\s*$", image_tag), line)) { + keep[[i]] <- FALSE + } + i <- i + 1L + } + out <- lines[keep] + # A dropdown cannot default to an option that no longer exists. + bad_default <- grepl(sprintf("^\\s*default:\\s*'%s'\\s*$", image_tag), out) + if (any(bad_default)) { + for (k in which(bad_default)) { + opts <- grep("^\\s*-\\s*'[^']+'\\s*$", out[seq_len(k - 1L)]) + if (length(opts) == 0L) { + next + } + # Walk back to this block's first option, not the nearest one. + first <- max(opts) + while (first > 1L && grepl("^\\s*-\\s*'[^']+'\\s*$", out[[first - 1L]])) { + first <- first - 1L + } + replacement <- sub("^\\s*-\\s*'([^']+)'\\s*$", "\\1", out[[first]]) + out[[k]] <- sub("'[^']+'", sprintf("'%s'", replacement), out[[k]]) + cat(sprintf(" default moved to '%s'\n", replacement)) + } + } + out +} + +if (!assets_only) { + files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE)) + touched <- 0L + for (f in files) { + lines <- readLines(f, warn = FALSE) + out <- strip_slot(lines) + if (length(out) == length(lines)) { + next + } + touched <- touched + 1L + cat(sprintf( + " %s %-45s -%d lines\n", + if (apply_changes) "EDIT " else "would edit", + f, + length(lines) - length(out) + )) + if (apply_changes) { + writeLines(out, f) + } + } + if (touched == 0L) { + cat(" No pipeline references the slot.\n") + } else if (apply_changes) { + cat(sprintf( + " %d file(s) edited. Verify with `crow lint .crow/` and `Rscript local/check-slots.R`.\n", + touched + )) + } + cat("\n") +} + +## 2. Bucket objects ---------------------------------------------------------- + +if (!config_only) { + library(s3fs, quietly = TRUE) + if (!nzchar(Sys.getenv("B2_S3_ACCESS_KEY"))) { + stop( + "B2_S3_ACCESS_KEY is unset. B2 lists nothing without credentials, which is indistinguishable from an already-clean slot.", # nolint + call. = FALSE + ) + } + bucket <- "devxy-rpkgs-binaries" + s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE + ) + + for (arch in c("amd64", "arm64")) { + prefix <- sprintf("%s/%s/%s", bucket, arch, codename) + if (!s3fs::s3_dir_exists(prefix)) { + cat(sprintf(" %s/%s: nothing in the bucket.\n", arch, codename)) + next + } + objects <- s3fs::s3_dir_ls(prefix, recurse = TRUE) + cat(sprintf( + " %s %s/%s: %d object(s)\n", + if (apply_changes) "DELETE" else "would delete", + arch, + codename, + length(objects) + )) + if (apply_changes && length(objects) > 0L) { + s3fs::s3_dir_delete(prefix) + cat(sprintf(" deleted s3://%s\n", prefix)) + } + } + cat("\n") +} + +## 3. Crons ------------------------------------------------------------------ + +if (!apply_changes) { + cat("Dry run: nothing was edited or deleted. Re-run with --apply.\n\n") +} + +cat("Crons still to remove on the Crow server (they are not in this repo):\n") +cat(sprintf( + " crow-at ci.devxy.io repo cron ls devxy/build-cran-binaries | grep %s\n", + pipeline_os +)) +cat(" # `crow repo cron rm` 404s here; the API base is /api/v1\n") +cat(" eval \"$(crow-at --export ci.devxy.io)\"\n") +cat(" curl -s -X DELETE -H \"Authorization: Bearer $CROW_TOKEN\" \\\n") +cat(" https://ci.devxy.io/api/v1/repos/5/cron/\n") From 6b764d2014cb97e69a7ee7bac135afa85f5fc5c1 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 13 Sep 2026 09:35:50 +0000 Subject: [PATCH 104/118] fix(build): build packages requiring newer R on compatible minors (#216) --- .crow/build-all-versions.yaml | 2 + .crow/process-updates.yaml | 2 + README.md | 17 +++++++ justfile | 7 +++ local/build-newer-r.R | 72 ++++++++++++++++++++++++++ local/newer-r-helpers.R | 75 +++++++++++++++++++++++++++ local/tests/test-newer-r-helpers.R | 81 ++++++++++++++++++++++++++++++ 7 files changed, 256 insertions(+) create mode 100644 local/build-newer-r.R create mode 100644 local/newer-r-helpers.R create mode 100644 local/tests/test-newer-r-helpers.R diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 91b7bcd..7451988 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -167,6 +167,8 @@ steps: R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true + # Also build current packages whose required dependency tree needs a newer R. + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION" $SPLIT_INTO $SPLIT_INDEX done # archive missed packages; first arg is the codename (e.g. "alpine324"), # derived via bincraft like the upload step, not paste(OS, OS_VERSION). diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 4cd95a7..a1b9698 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -207,6 +207,8 @@ steps: R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + # Also build current packages whose required dependency tree needs a newer R. + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION" done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/README.md b/README.md index d137728..99d1143 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,23 @@ This elastic server architecture offers a robust and performant backend while mi ## Technical Details +### Packages requiring a newer R + +The daily update and weekly full-build pipelines run `local/build-newer-r.R` under each non-primary installed R, alongside the ABI-sensitive pass. +It selects current CRAN versions whose `Depends: R` constraints cannot be met by the slot's primary R but can be met by the running R, including incompatibility inherited through `Depends`, `Imports`, and `LinkingTo`. +These builds retain bincraft's ABI classification and storage routing: sensitive binaries use the running minor, while safe binaries can be shared. +No shared S3 existence cache is passed, so bincraft checks the actual objects and distinguishes a source fallback from a binary. +The pass reads unfiltered CRAN metadata independently of the weekly snapshot and its primary-R failure exclusions; existing binaries and failures for the running minor retain bincraft's usual skip behavior. +Weekly workers split this extra work across their existing shards. +This does not select older archive versions for unsupported R minors or change index filtering. + +Preview the selection without building or accessing storage or build metadata: + +```sh +# Run with the target R and supply the slot's full primary R version. +/opt/R/4.6.0/bin/Rscript local/build-newer-r.R 4.4.3 --dry-run +``` + ### Creating/Updating the PACKAGES Index Files Currently, the {cranlike} and {desc} packages only work with files on a local file system. diff --git a/justfile b/justfile index 36ad0e1..524ac9a 100644 --- a/justfile +++ b/justfile @@ -95,3 +95,10 @@ check-slots: # strip a retired slot from .crow/*.yaml (dry run; pass --apply to write) retire-slot codename *flags: Rscript local/retire-slot.R {{ codename }} --config-only {{ flags }} + +# Format or lint selected R files, defaulting to local tooling. +format *files="local": + air format {{ files }} + +fix *files="local": + jarl check --fix {{ files }} diff --git a/local/build-newer-r.R b/local/build-newer-r.R new file mode 100644 index 0000000..3d7794f --- /dev/null +++ b/local/build-newer-r.R @@ -0,0 +1,72 @@ +source(file.path("local", "newer-r-helpers.R")) +source(file.path("local", "rebuild-missing-helpers.R")) + +args <- commandArgs(trailingOnly = TRUE) +pos <- args[!startsWith(args, "--")] +if (!length(pos) %in% c(1L, 3L)) { + stop("Usage: build-newer-r.R PRIMARY_R [SPLIT_INTO SPLIT_INDEX] [--dry-run]") +} +primary_r <- pos[1L] +target_r <- as.character(getRversion()) +pkgs <- newer_r_candidates(tools::CRAN_package_db(), primary_r, target_r) +exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] +pkgs <- pkgs[!pkgs$Package %in% exclude, , drop = FALSE] +if (length(pos) == 3L) { + pkgs <- pkgs[ + shard_slice(seq_len(nrow(pkgs)), pos[2L], pos[3L]), + , + drop = FALSE + ] +} +message(sprintf( + "R %s: %d package(s) unsupported by primary R %s", + target_r, + nrow(pkgs), + primary_r +)) +if ("--dry-run" %in% args) { + print(pkgs, row.names = FALSE) +} else { + options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL) + for (i in seq_len(nrow(pkgs))) { + message(sprintf( + "[%d/%d] %s %s", + i, + nrow(pkgs), + pkgs$Package[i], + pkgs$Version[i] + )) + bincraft::build_binary_package( + pkgs$Package[i], + tag = pkgs$Version[i], + # Keep bincraft's normal ABI routing: safe binaries can be shared; + # sensitive ones use this interpreter's minor for presence and output. + is_r_minor_sensitive = bincraft::classify_r_minor_sensitive( + pkgs$Package[i], + pkgs$Version[i], + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432 + ), + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + patches = "local/patches", + upload = TRUE, + store_build_metadata = TRUE + ) + } +} diff --git a/local/newer-r-helpers.R b/local/newer-r-helpers.R new file mode 100644 index 0000000..4b83204 --- /dev/null +++ b/local/newer-r-helpers.R @@ -0,0 +1,75 @@ +# Evaluate every R constraint, including patch-level and upper bounds. +r_requirement_satisfied <- function(depends, r_version) { + if (is.na(depends) || !nzchar(trimws(depends))) { + return(TRUE) + } + entries <- trimws(strsplit(depends, ",", fixed = TRUE)[[1L]]) + entries <- entries[grepl("^R(\\s|\\(|$)", entries)] + all(vapply( + entries, + function(entry) { + if (entry == "R") { + return(TRUE) + } + match <- regmatches( + entry, + regexec( + "^R\\s*\\(\\s*(>=|<=|==|>|<)\\s*([0-9]+(?:[.-][0-9]+)*)\\s*\\)$", + entry, + perl = TRUE + ) + )[[1L]] + if (length(match) != 3L) { + stop("Cannot parse R dependency: ", entry) + } + comparison <- utils::compareVersion(r_version, match[3L]) + switch( + match[2L], + ">=" = comparison >= 0L, + "<=" = comparison <= 0L, + "==" = comparison == 0L, + ">" = comparison > 0L, + "<" = comparison < 0L + ) + }, + logical(1L) + )) +} + +# Use unfiltered current CRAN metadata: available.packages() would hide exactly +# the packages that cannot run under the primary interpreter. +# Propagate R incompatibility through required dependencies, not Suggests. +newer_r_candidates <- function(db, primary_r, target_r) { + db <- as.data.frame(db, stringsAsFactors = FALSE) + # CRAN also lists R-specific Recommended copies (e.g. Matrix for R-devel). + # Prefer the ordinary contrib record, or a future-R copy can incorrectly + # block every package depending on Matrix on all released interpreters. + if ("Path" %in% names(db)) { + db <- db[order(!is.na(db$Path) & nzchar(db$Path)), , drop = FALSE] + } + db <- db[!duplicated(db$Package), , drop = FALSE] + deps <- tools::package_dependencies( + packages = db$Package, + db = as.matrix(db), + which = c("Depends", "Imports", "LinkingTo"), + recursive = FALSE + ) + supported <- function(version) { + ok <- vapply(db$Depends, r_requirement_satisfied, logical(1L), version) + repeat { + blocked <- db$Package[!ok] + next_ok <- ok & + !vapply(deps, function(x) any(x %in% blocked), logical(1L)) + if (identical(ok, next_ok)) { + return(ok) + } + ok <- next_ok + } + } + selected <- !supported(primary_r) & supported(target_r) + if ("OS_type" %in% names(db)) { + selected <- selected & (is.na(db$OS_type) | db$OS_type != "windows") + } + result <- db[selected, c("Package", "Version"), drop = FALSE] + result[order(result$Package), , drop = FALSE] +} diff --git a/local/tests/test-newer-r-helpers.R b/local/tests/test-newer-r-helpers.R new file mode 100644 index 0000000..c202b7f --- /dev/null +++ b/local/tests/test-newer-r-helpers.R @@ -0,0 +1,81 @@ +source(file.path("..", "newer-r-helpers.R")) + +test_that("R constraints respect patch versions, bounds, and missing fields", { + expect_true(r_requirement_satisfied(NA_character_, "4.4.3")) + expect_true(r_requirement_satisfied( + "Matrix (>= 1.4-0), R (>= 4.5.0)", + "4.5.3" + )) + expect_false(r_requirement_satisfied("R (>= 4.5.0)", "4.4.3")) + expect_false(r_requirement_satisfied("R (>= 4.5.4)", "4.5.3")) + expect_true(r_requirement_satisfied("R (>= 4.5), R (< 4.6)", "4.5.3")) + expect_false(r_requirement_satisfied("R (>= 4.5), R (< 4.6)", "4.6.0")) + expect_true(r_requirement_satisfied("R (== 4.5.3)", "4.5.3")) + expect_true(r_requirement_satisfied("R (> 4.5.2), R (<= 4.5.3)", "4.5.3")) + expect_true(r_requirement_satisfied("R, Rcpp (>= 1.0)", "4.4.3")) + expect_error(r_requirement_satisfied("R (~ 4.5)", "4.5.3"), "Cannot parse") +}) + +fixture <- function() { + data.frame( + Package = c( + "arules", + "arulesSequences", + "indirect", + "optional", + "ordinary", + "future", + "blocked" + ), + Version = rep("1.0", 7L), + Depends = c( + "R (>= 4.5.0)", + "arules", + NA, + NA, + "R (>= 4.0)", + "R (>= 4.7)", + NA + ), + Imports = c(NA, NA, "arulesSequences", NA, NA, NA, "arules, future"), + LinkingTo = NA_character_, + Suggests = c(NA, NA, NA, "arules", NA, NA, NA) + ) +} + +test_that("every compatible newer minor includes required dependents only", { + db <- fixture() + for (target in c("4.5.3", "4.6.0")) { + expect_identical( + newer_r_candidates(db, "4.4.3", target)$Package, + c("arules", "arulesSequences", "indirect") + ) + } + expect_equal(nrow(newer_r_candidates(db, "4.4.3", "4.4.3")), 0L) + expect_equal(nrow(newer_r_candidates(db, "4.5.3", "4.4.3")), 0L) + expect_equal(nrow(newer_r_candidates(db, "4.5.3", "4.6.0")), 0L) +}) + +test_that("LinkingTo, dependency cycles, Windows and empty metadata are handled", { + db <- fixture() + db$LinkingTo[5L] <- "arules" + db$Imports[1L] <- "ordinary" + db$OS_type <- c(NA, "windows", rep(NA_character_, 5L)) + expect_identical( + newer_r_candidates(db, "4.4.3", "4.6.0")$Package, + c("arules", "indirect", "ordinary") + ) + expect_equal(nrow(newer_r_candidates(db[FALSE, ], "4.4.3", "4.6.0")), 0L) +}) + +test_that("R-devel Recommended copies cannot block ordinary CRAN dependencies", { + db <- fixture() + db$Path <- NA_character_ + future_copy <- db[1L, ] + future_copy$Depends <- "R (>= 4.7)" + future_copy$Path <- "4.7.0/Recommended" + expect_identical( + newer_r_candidates(rbind(future_copy, db), "4.4.3", "4.6.1")$Package, + c("arules", "arulesSequences", "indirect") + ) +}) From b0bade879b16e2c4d2a09a4f87fe7b867a71060e Mon Sep 17 00:00:00 2001 From: automation-bot Date: Mon, 14 Sep 2026 00:32:09 +0000 Subject: [PATCH 105/118] chore(deps): lock file maintenance --- .terraform.lock.hcl | 62 ++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 9591b83..5e45f89 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,38 +2,38 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/http" { - version = "3.6.1" + version = "3.6.2" hashes = [ - "h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=", - "h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=", - "h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=", - "h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=", - "h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=", - "h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=", - "h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=", - "h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=", - "h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=", - "h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=", - "h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=", - "h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=", - "h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=", - "h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=", - "h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=", - "zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9", - "zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6", - "zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab", - "zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b", - "zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c", - "zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4", - "zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502", - "zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69", - "zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6", - "zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279", - "zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6", - "zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7", - "zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df", - "zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621", - "zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424", + "h1:7+Ewr07oLewLfoe5K7Fhy4T5JUU4y7I7iRGTG2hrzy4=", + "h1:8AvJ6H30J1tgxwEoVKxe0B0Ac9XFn1gwOzidrIc+F5o=", + "h1:AllxtaK2m88aY+QitHui6kik/qJmBakrH5o5W7vOKI0=", + "h1:OimQGm1/9sIkdDCuUb8D5sxDorhektHeCQ337DHDfGk=", + "h1:T5zxkFXQ9To+ZysjCZu6gFqE8G1aTGh58Ol20bCk1Fc=", + "h1:TZlCCHsmKXSI3FyzmSgCU+TarKOHgiOft0wiYh85Y+E=", + "h1:VgyzPdGw1U/KU8gKhefKlIhy5kQYdpoZkpNcGiTgvVs=", + "h1:aF+Ztfy9m0Ll6nFOtC90NxN6WIEKAmfmVFhXAtFF8CI=", + "h1:bgViIoXF7kMXA7xgNikFYfv1D15rregO8I8Sq95sLGQ=", + "h1:d7tCAc1wr1seWpGMBx7tpRyGKOCicvozaHvn5X8ZFfM=", + "h1:dRCrpHH2UrkKs/lCqewyA00L0VfOCETVM14NBhyxHaM=", + "h1:eIqaNIixQTlhmr2GU1fAFH+eVYLQIfeGw7Ec+JT/bEo=", + "h1:hrK+Jzpgivi4Jo356K3LYDoeWBV1bMqKPBEqrEHwgpA=", + "h1:mrpIAC+TK3rQQz2jllAF8o2WLMKM14/WoclLgYKc4WA=", + "h1:zwE/uNBapBDpkKa5I24moawVPTanbG+eKumHRm/qINw=", + "zh:0cf6173a426f65a85e8baf4f9a7808dc94d0e02a82b34bc903555fda1eb45d7a", + "zh:2b5fa2b0487e692014fa86745a2a98d43f34f7aaea044acd3e2cee139627c9f1", + "zh:2c9c18893ce0f34e077559632c88237d172ac6646853bf00c84a5a4ec3b6c241", + "zh:316c848e892764aa10a890cf2455f52b592c6051a92ac8451955ca1d47ca1d2f", + "zh:4300e32ecaa967ca0ed56ca48f8dcd10796e70203bf6b8311e71dc43cd0d2aa1", + "zh:4a0a555d85a31bd257399253b502ad03816ccdba7b8fbbbd1e07a313d04d83fb", + "zh:6278773c31c1605c6d780fc982eacb83944e2807c8d8afe11534253b225c5aa3", + "zh:81b73da8dbcedf54604f131e0acd7ebd36cf673276174b4b4f10af0b0f10de11", + "zh:887beaa6ef6521d3f5e2b1193d458575fb91855f36bc66dd26a10988127a39fb", + "zh:91b2260424c5f8e5968d9e6d5bdbbdb60d61ccc4ff001aa0d637427629349c54", + "zh:a5d9d7032d00a6dd844e8d0d374b5ae89e25f9a6635a90e01c5f65e9dd14a93e", + "zh:bcf8b2aca448774e4701f221b74d762847be55dc312022b4aad66da3b389bf5d", + "zh:d62a3ad37ee1fb943ad0590978f4e7ee0ac064d19554c1c05ec98e9d2bb658f7", + "zh:e517b8b81fe88bf49868923f076515b9157771d1d3a80849cc8a73b9a19039a1", + "zh:fa9e0e39ef3e5ed2e6cf5886a0b569415ca3b89b9289fd6d01824ff11e470618", ] } From 3325a7bda2542835ba95ae3f6a4b64c41355ec01 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Thu, 17 Sep 2026 00:32:19 +0000 Subject: [PATCH 106/118] chore(deps): update pre-commit hook rbubley/mirrors-prettier to v3.9.7 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e28c6d..2c8e4b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.6 + rev: v3.9.7 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit From caefd1131f68634354becf541da7e05a9b790a74 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sat, 19 Sep 2026 00:32:12 +0000 Subject: [PATCH 107/118] chore(deps): update pre-commit hook rbubley/mirrors-prettier to v3.9.8 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c8e4b8..a64e94b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.7 + rev: v3.9.8 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit From 212065f5a439f8391172d74b4c32016537a87316 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sun, 20 Sep 2026 00:31:53 +0000 Subject: [PATCH 108/118] chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker to v4.0.2 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a64e94b..42fa0ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v4.0.1 + rev: v4.0.2 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ From aa6e5e9dd804948a50be36db2dffad0943cb9fa6 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Mon, 21 Sep 2026 00:31:54 +0000 Subject: [PATCH 109/118] chore(deps): update pre-commit hook davidanson/markdownlint-cli2 to v0.23.3 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 42fa0ab..d76cab5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.23.2 + rev: v0.23.3 hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier From 3962ed6d6129806557c5e68334d1a3f0368fd321 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 22 Sep 2026 12:17:52 +0000 Subject: [PATCH 110/118] fix(installer): install gettext headers on Alpine (#222) --- local/uvr-install.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/local/uvr-install.sh b/local/uvr-install.sh index 3966e85..ed6b1ce 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -16,8 +16,8 @@ # precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds # bincraft and its dependencies alongside whatever this script installs). # -# System dependencies come from uvr's vendored r-system-requirements rules, so -# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed. +# System dependencies come from uvr's vendored r-system-requirements rules, +# with explicit prerequisites below for gaps in Alpine support. # # Environment: # UVR_R_BIN R interpreter to install for; set by install-bincraft.R so @@ -117,4 +117,11 @@ if command -v apt-get >/dev/null 2>&1; then apt-get update -qq || echo "warning: apt-get update failed; continuing" >&2 fi +# Alpine's build images lack libintl.h, which data.table needs when R enables +# NLS. uvr does not resolve this prerequisite on Alpine, so install it before +# compiling bincraft's dependencies (including when the library cache is cold). +if command -v apk >/dev/null 2>&1; then + apk add --no-cache gettext-dev +fi + "$uvr_bin" sync --library "$target_lib" --install-system-deps From e8b0df0c910fe6ff55d5242170cb1df4f73ca540 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Wed, 23 Sep 2026 00:32:05 +0000 Subject: [PATCH 111/118] chore(deps): update terraform bunnynet to ~> 0.19 --- .terraform.lock.hcl | 72 ++++++++++++++++++++++----------------------- provider.tf | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 5e45f89..2b8939f 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.18.2" - constraints = "~> 0.18" + version = "0.19.0" + constraints = "~> 0.19" hashes = [ - "h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=", - "h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=", - "h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=", - "h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=", - "h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=", - "h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=", - "h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=", - "h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=", - "h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=", - "h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=", - "h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=", - "h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=", - "h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=", - "h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=", - "h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=", - "h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=", - "h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=", - "zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c", - "zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178", - "zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd", - "zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef", - "zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738", - "zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc", - "zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c", + "h1:+QTK+99uKgipcQerdp7XqyDVQr4+j9Rt+J6IlsRTyig=", + "h1:/6HUvLgktHKpi0iUv5Bom5Svx9nssqfBog/Hy2llYsU=", + "h1:0tCQLvrTOsHlYfgv9tbsy8PsoiEjiSMvswfBVfErBFc=", + "h1:1IY3hW9r6wvjsMknmHgByn5qEBMUrYD7ZyW9vzHGQOk=", + "h1:1nmQH1YxIxDZANJhSAkvyJolHO6MJO82zuL94Gitxwk=", + "h1:2ldfuwhEY+KsRqTx6zGSHIom4R2O2oVb06rDKFq2FKo=", + "h1:6KFzTrYT7t2g9cMfyCk5BLu0LgpyzujChK3vlvZZw4A=", + "h1:7CgJOdr7RUdo/O3J0Qhft/sQvgX0E30IKL4WauzBPCg=", + "h1:9mkBzQFtNWbz2niXa5ciskQQZudk/J+Zj1iG6J/Ks/k=", + "h1:FAjraG7Qgv9ETmIVRurqqVCQUjuHAWDMV3hKSFwnoig=", + "h1:QNq/k9igqqg9vVxGvdGNU6y5ySTUcYgETUiFXr/x10E=", + "h1:UpP4dVhp/M7yE0vHLqDslVUgJlvlKuJBrx2uQTbsOVE=", + "h1:VCnb/kkUIMD4HUss8pjQxl3ynqo6e/eAbj6wJfpUjQI=", + "h1:W57YKtRc1V85to2kn2vUjLDKb8BSr0dbEQASA/MBXzo=", + "h1:ZhWq2+5BPm0/allcpEd/MvVvoZX5+YVTukledDhufDQ=", + "h1:Zyf433smZkkoluBh3FMJFt5rvFsv8cD+CUMHMXNJ6Zk=", + "h1:m8evouaTWTHWl/1fIr4/hcTpYCOubyi+DNQJRkxGP0U=", + "zh:0d80b74cc2d1f1782754246333c470be5ab0e003bf88f942a1ceaa248db70ecd", + "zh:1b3871707de0f68d6b4cdab8cb28fe9023765e30138c5e404688014bcfa56feb", + "zh:23029fa78fb8ca799f499577fcf6d7bb3e7eda450ec3a83e22d2bd646f30a021", + "zh:380ca854544eb5a754a333f585b5d0461d9e2959d933f905569c8e074b47ba46", + "zh:3c867179ba968d77b5440297db16ef1f66559c4db07a9623bd8d4b0ac0b79460", + "zh:56a27b4617e9c3589c529a0d2d6abf9858e675d93a2d883fabf0a6d12b5eebbd", + "zh:578cf3aa09278231f63e20736c917a53e67ecce2d7c083218a96d8e536adb3bd", + "zh:5d6d2b3669263e56649b738d1a71228ca08c512e2a16beea8590d0fd2d94ac5c", + "zh:6996d13035e1d2a459962c2b55438cdde9bd96a575e3eb37096209a2ea6f8ddc", + "zh:75789447b0c1708b39369bb000fc2a03a187cfc07e80263475bf5ecd04d9cebd", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef", - "zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3", - "zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb", - "zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e", - "zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11", - "zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7", - "zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9", - "zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1", - "zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff", - "zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339", + "zh:8b70254b4a929d49a07b75279c4f7a74e403a21dce0d796af5b2ac39ddd59b87", + "zh:93c03840b56c747933785845e59adca266ee838ed48dc07c57ae4d739d1c76da", + "zh:a1e4a968f0ddfad51c32fd022ccb7e788995b2289e94f4ca74cf773ce319e6d8", + "zh:ab3efbf3bb867caa4133ec3a12ba0dcb03346752a0fafe8d8e7c704804f18580", + "zh:ace4370adaccecf7be48fd4f62f308c1e91961e218ec152f50cdf585b4cba546", + "zh:de074682b8953db39ffbc1337251a25633b796d7b5f68c386c4b039088045068", + "zh:f07ecc5afe3254d36c1a98d843feea5efe39313b5df174c27443e7699f641867", ] } diff --git a/provider.tf b/provider.tf index d4f2564..cb5a87d 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.18" + version = "~> 0.19" } } } From b18c21aa8d8dc2585239387cd6ed11792fedf2c7 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Thu, 24 Sep 2026 00:32:00 +0000 Subject: [PATCH 112/118] chore(deps): update pre-commit hook rbubley/mirrors-prettier to v3.9.9 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d76cab5..d57d5eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.8 + rev: v3.9.9 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit From 5231758c00fe612f96e6d40d77c068cdab7c0c45 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Fri, 25 Sep 2026 00:32:19 +0000 Subject: [PATCH 113/118] chore(deps): update terraform bunnynet to v0.19.1 --- .terraform.lock.hcl | 70 ++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 2b8939f..4b118a8 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.19.0" + version = "0.19.1" constraints = "~> 0.19" hashes = [ - "h1:+QTK+99uKgipcQerdp7XqyDVQr4+j9Rt+J6IlsRTyig=", - "h1:/6HUvLgktHKpi0iUv5Bom5Svx9nssqfBog/Hy2llYsU=", - "h1:0tCQLvrTOsHlYfgv9tbsy8PsoiEjiSMvswfBVfErBFc=", - "h1:1IY3hW9r6wvjsMknmHgByn5qEBMUrYD7ZyW9vzHGQOk=", - "h1:1nmQH1YxIxDZANJhSAkvyJolHO6MJO82zuL94Gitxwk=", - "h1:2ldfuwhEY+KsRqTx6zGSHIom4R2O2oVb06rDKFq2FKo=", - "h1:6KFzTrYT7t2g9cMfyCk5BLu0LgpyzujChK3vlvZZw4A=", - "h1:7CgJOdr7RUdo/O3J0Qhft/sQvgX0E30IKL4WauzBPCg=", - "h1:9mkBzQFtNWbz2niXa5ciskQQZudk/J+Zj1iG6J/Ks/k=", - "h1:FAjraG7Qgv9ETmIVRurqqVCQUjuHAWDMV3hKSFwnoig=", - "h1:QNq/k9igqqg9vVxGvdGNU6y5ySTUcYgETUiFXr/x10E=", - "h1:UpP4dVhp/M7yE0vHLqDslVUgJlvlKuJBrx2uQTbsOVE=", - "h1:VCnb/kkUIMD4HUss8pjQxl3ynqo6e/eAbj6wJfpUjQI=", - "h1:W57YKtRc1V85to2kn2vUjLDKb8BSr0dbEQASA/MBXzo=", - "h1:ZhWq2+5BPm0/allcpEd/MvVvoZX5+YVTukledDhufDQ=", - "h1:Zyf433smZkkoluBh3FMJFt5rvFsv8cD+CUMHMXNJ6Zk=", - "h1:m8evouaTWTHWl/1fIr4/hcTpYCOubyi+DNQJRkxGP0U=", - "zh:0d80b74cc2d1f1782754246333c470be5ab0e003bf88f942a1ceaa248db70ecd", - "zh:1b3871707de0f68d6b4cdab8cb28fe9023765e30138c5e404688014bcfa56feb", - "zh:23029fa78fb8ca799f499577fcf6d7bb3e7eda450ec3a83e22d2bd646f30a021", - "zh:380ca854544eb5a754a333f585b5d0461d9e2959d933f905569c8e074b47ba46", - "zh:3c867179ba968d77b5440297db16ef1f66559c4db07a9623bd8d4b0ac0b79460", - "zh:56a27b4617e9c3589c529a0d2d6abf9858e675d93a2d883fabf0a6d12b5eebbd", - "zh:578cf3aa09278231f63e20736c917a53e67ecce2d7c083218a96d8e536adb3bd", - "zh:5d6d2b3669263e56649b738d1a71228ca08c512e2a16beea8590d0fd2d94ac5c", - "zh:6996d13035e1d2a459962c2b55438cdde9bd96a575e3eb37096209a2ea6f8ddc", - "zh:75789447b0c1708b39369bb000fc2a03a187cfc07e80263475bf5ecd04d9cebd", + "h1:02CmekB5A4NtAeptTR7waaECJl66Gf3ww7Wtyg1cy+g=", + "h1:0VWwMHkHQXusR2vdyYWN1pNvfJUPt9OWh6glgZshCck=", + "h1:2NGfMzSWPCcyO2P2T9/lDS5MExYnaOfLuDEMiYvFBg4=", + "h1:2dXsvcYEGXsIf1IHD2Rq9HzgwrNNEA4+qOzBKpbFEpw=", + "h1:8R0H9DumQe0DjXTHRJXNiZUyTBDHT2vjfu/vB+WdRIw=", + "h1:9Gi5M2iKY+Oz8wAYoJh7i0u3mySBk5pretuyzcwgJOw=", + "h1:BZwoP8Ubi1iDueWXhSWb3Tlt/II4bJI0/p6/tqw4i1M=", + "h1:F4yWXkyv55F+97IQi/jYBLhBLyhz4hkokVCyEEQzbM4=", + "h1:MGHXEapStuXemfw3SikP2aLQCT+b7Ee/RrCHevSAAWA=", + "h1:Q10nENYoX8jIsBMz1kP8OBrkMeSNmFI2CfB6/MsWKAc=", + "h1:R9z1oacW8gD9hPTGTO37ngDAhAvlP33/PRhzkgXDStY=", + "h1:UYLBa5/V+giaZ2YYjkbPDKMRO9eQRNcWEnpZqU2I/yU=", + "h1:Wz/QDxt1Kved07fCwqMh+6jtTtwQ/xKi9mXE2StV7vw=", + "h1:Yq0JSIW4Jir3EVO2w7C/iM76yzYKG5eYw51woX6CXJo=", + "h1:e/PEmAJszFbjdfsAMJosDbkxtHwHNnByd/3nKJmE3Sg=", + "h1:lMVwei5IGLQCrPMqAmcpbXs/BjaG7k6HbjhuWtr0kIA=", + "h1:m8dyuq1sXJ9sxjZkU3ZUsjC77BWY87GiraNvxHluSYs=", + "zh:15e84e2fca2bbaec9c9f4d14d1a037d02f4d8cc1d1bdff04709d51085f950774", + "zh:1b9d0cbb7f08e7a554afb8c1e733942a77aa0c8112e8b2b7092c040794f0b113", + "zh:28768988a8ec70e5392ca93375717b83a9aa2eeaa21be2f53c28550273b97065", + "zh:2f357c1d2c013627e4e785c350fc72ccc0bfbebaa6062e536ec7eb9cfecfc913", + "zh:4a41bab7fba5415b8c422f5c03d6fb054ae210e1eb180fab4081d4fa976080b5", + "zh:6c10fad0bd03bbf053b97ad51f0a24e9a0c43a7fc10727033a39eec90ed0a51e", + "zh:795980ac64b0f9bda521f829187eb7b296baa682b7e8d409126f85e6a6f9bcc0", + "zh:841a7e234852ecd516e2dc52c7d31025a704242aeafee15eb36bf392e62f918f", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:8b70254b4a929d49a07b75279c4f7a74e403a21dce0d796af5b2ac39ddd59b87", - "zh:93c03840b56c747933785845e59adca266ee838ed48dc07c57ae4d739d1c76da", - "zh:a1e4a968f0ddfad51c32fd022ccb7e788995b2289e94f4ca74cf773ce319e6d8", - "zh:ab3efbf3bb867caa4133ec3a12ba0dcb03346752a0fafe8d8e7c704804f18580", - "zh:ace4370adaccecf7be48fd4f62f308c1e91961e218ec152f50cdf585b4cba546", - "zh:de074682b8953db39ffbc1337251a25633b796d7b5f68c386c4b039088045068", - "zh:f07ecc5afe3254d36c1a98d843feea5efe39313b5df174c27443e7699f641867", + "zh:8c876a0a26b249151f0918ee836e24660513c2345c23a06e0f54a28c7c6966b7", + "zh:906a791496cf11dcfdfe43a4167414cb473e548f0bd86d4b8a6a95450b523133", + "zh:9431757bf5b7d055ecc1d3c8465216d5e7a648b75e0e35b136ac8fc25382e97d", + "zh:967fd248df86fc246536727e996a7334711f4c6fe9e3d01ec9f03abe621a9783", + "zh:a6710fbd30f8a4844eb025d616057e78bbb255a7d05cc73876dae55a5d6b0d06", + "zh:b5f260899555430c109da35d2c483999d61b127beda6c2a7b39eb67f27d754c8", + "zh:d24882eb63c6452ed27666e896224b1d866ad00645d6ecc53de84d1687d27fc4", + "zh:e573eb0c99ea7195ec1ed3da728caddacc68a126d82679fb490438c781382f4d", + "zh:fcf0a66c0c0b62af505c5f0f8fe59bca77d6085a0b07763be33420101183702e", ] } From 317990a40f2cd01bda2c2d314afc8efbafff1ff5 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Fri, 2 Oct 2026 00:32:22 +0000 Subject: [PATCH 114/118] chore(deps): update pre-commit hook posit-dev/air-pre-commit to v0.12.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d57d5eb..dec6844 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit - rev: 0.11.0 + rev: 0.12.0 hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker From 40a0a6e02d8c49ea98e0c27068591d44eeb279e0 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 4 Oct 2026 07:35:01 +0000 Subject: [PATCH 115/118] fix(patches): cover every RcppParallel 6.x release with the bundled-TBB patch (#228) --- .../force-bundled-tbb-6.0.0.patch | 152 +++++++++++++++++ .../force-bundled-tbb-6.1.x.patch | 157 ++++++++++++++++++ ...bb.patch => force-bundled-tbb-6.2.x.patch} | 68 ++++---- local/patches/registry.json | 44 ++++- 4 files changed, 381 insertions(+), 40 deletions(-) create mode 100644 local/patches/RcppParallel/force-bundled-tbb-6.0.0.patch create mode 100644 local/patches/RcppParallel/force-bundled-tbb-6.1.x.patch rename local/patches/RcppParallel/{force-bundled-tbb.patch => force-bundled-tbb-6.2.x.patch} (67%) diff --git a/local/patches/RcppParallel/force-bundled-tbb-6.0.0.patch b/local/patches/RcppParallel/force-bundled-tbb-6.0.0.patch new file mode 100644 index 0000000..a7f91dc --- /dev/null +++ b/local/patches/RcppParallel/force-bundled-tbb-6.0.0.patch @@ -0,0 +1,152 @@ +diff --git a/R/aaa.R b/R/aaa.R +index 568a2aa..2744059 100644 +--- a/R/aaa.R ++++ b/R/aaa.R +@@ -5,4 +5,21 @@ TBB_LIB <- "" + TBB_INC <- "" + + TBB_NAME <- "tbb" +-TBB_MALLOC_NAME <- "tbbmalloc" +\ No newline at end of file ++TBB_MALLOC_NAME <- "tbbmalloc" ++ ++# bincraft patch: build images (ours did until build-env-images c51ed09) and ++# plenty of user environments export TBB_ROOT / TBB_LIB / TBB_INC. A binary we ++# publish always carries its own oneTBB in RcppParallel/lib -- see the ++# companion changes in tools/config/configure.R and src/install.libs.R -- so ++# honouring those variables at run time is actively harmful: .onLoad() would ++# dyn.load() a second, unrelated TBB into the process next to the bundled one ++# (two copies of the same symbols in the global scope, which segfaults R on ++# load), and RcppParallelLibs() / CxxFlags() would hand that system TBB to ++# dependents such as rstan, putting NEEDED libtbb.so.12 back into their ++# binaries. Read the variables only when explicitly opted back in. ++bincraftGetenv <- function(name, unset = "") { ++ if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") ++ Sys.getenv(name, unset = unset) ++ else ++ unset ++} +\ No newline at end of file +diff --git a/R/tbb.R b/R/tbb.R +index 3d8fbb3..9560165 100644 +--- a/R/tbb.R ++++ b/R/tbb.R +@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { + sysname <- Sys.info()[["sysname"]] + + # find root for TBB install +- tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) ++ tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) + if (is.null(name)) + return(tbbRoot) + +@@ -60,7 +60,7 @@ tbbCxxFlags <- function() { + flags <- c("-DRCPP_PARALLEL_USE_TBB=1") + + # if TBB_INC is set, apply those library paths +- tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) ++ tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) + if (!file.exists(tbbInc)) { + tbbInc <- system.file("include", package = "RcppParallel") + } +@@ -100,7 +100,7 @@ tbbLdFlags <- function() { + } + + # shortcut if TBB_LIB defined +- tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) ++ tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) + if (nzchar(tbbLib)) { + if (R.version$os == "emscripten") { + fmt <- "-L%1$s -l%2$s" +diff --git a/src/install.libs.R b/src/install.libs.R +index 4dcbac2..57dbc74 100644 +--- a/src/install.libs.R ++++ b/src/install.libs.R +@@ -258,6 +258,18 @@ prependFlags <- function(prependFlags, toFlags) { + tbbLib <- Sys.getenv("TBB_LIB") + tbbInc <- Sys.getenv("TBB_INC") + ++# bincraft patch: the companion change in tools/config/configure.R stops an ++# ambient TBB_LIB / TBB_INC from selecting a system TBB, but this script is ++# also run directly by `R CMD INSTALL` (not only through the `tbb` rule in ++# src/Makevars, which passes the configured values), so at install time it ++# still sees the image's environment and would symlink the system libraries ++# into RcppParallel/lib. Drop them here for the same reason, under the same ++# opt-out. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbLib <- "" ++ tbbInc <- "" ++} ++ + args <- commandArgs(trailingOnly = TRUE) + if (identical(args, "build")) { + if (nzchar(tbbLib) && nzchar(tbbInc)) { +diff --git a/tools/config/configure.R b/tools/config/configure.R +index a001e63..bfe8c8e 100644 +--- a/tools/config/configure.R ++++ b/tools/config/configure.R +@@ -144,6 +144,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) + tbbLib <- Sys.getenv("TBB_LIB", unset = NA) + tbbInc <- Sys.getenv("TBB_INC", unset = NA) + ++# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Our build ++# images exported these until build-env-images c51ed09 (a leftover from ++# RcppParallel 5.x, whose bundled Intel TBB would not build on musl or with ++# modern g++), and any of them switches the branches below to a system TBB. ++# The published binary then records NEEDED libtbb.so.12 (or libtbb.so.2 for ++# the classic Intel TBB) and gets a RcppParallel/lib full of absolute symlinks ++# into the image's library dir, so it cannot dyn.load on a consumer machine ++# without that exact TBB. 6.x bundles oneTBB 2022 and builds it with cmake on ++# every platform we ship, so the bundled copy is always the right choice here; ++# forcing it in the package rather than relying on the image environment keeps ++# the binary correct whichever image version CI happens to pull. Set ++# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbRoot <- NA ++ tbbLib <- NA ++ tbbInc <- NA ++} ++ + tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") + tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") + +@@ -224,10 +242,21 @@ define( + ) + + # set PKG_LIBS ++# ++# bincraft patch: the library directories below are passed as plain '-L', not ++# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) ++# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', ++# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build ++# host with a distro TBB installed, '-ltbb' resolves to that library and ++# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the ++# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads ++# the system TBB rather than the copy shipped in RcppParallel/lib, and fails ++# outright on a machine that has no system TBB. gcc places a plain '-L' before ++# its built-in dirs, so the intended library is found first. + pkgLibs <- if (!is.na(tbbLib)) { + + c( +- "-Wl,-L\"$(TBB_LIB)\"", ++ "-L\"$(TBB_LIB)\"", + sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" +@@ -240,14 +269,14 @@ pkgLibs <- if (!is.na(tbbLib)) { + } else if (R.version$os == "emscripten") { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)" + ) + + } else { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" + ) diff --git a/local/patches/RcppParallel/force-bundled-tbb-6.1.x.patch b/local/patches/RcppParallel/force-bundled-tbb-6.1.x.patch new file mode 100644 index 0000000..948643c --- /dev/null +++ b/local/patches/RcppParallel/force-bundled-tbb-6.1.x.patch @@ -0,0 +1,157 @@ +diff --git a/R/aaa.R b/R/aaa.R +index 568a2aa..2744059 100644 +--- a/R/aaa.R ++++ b/R/aaa.R +@@ -5,4 +5,21 @@ TBB_LIB <- "" + TBB_INC <- "" + + TBB_NAME <- "tbb" +-TBB_MALLOC_NAME <- "tbbmalloc" +\ No newline at end of file ++TBB_MALLOC_NAME <- "tbbmalloc" ++ ++# bincraft patch: build images (ours did until build-env-images c51ed09) and ++# plenty of user environments export TBB_ROOT / TBB_LIB / TBB_INC. A binary we ++# publish always carries its own oneTBB in RcppParallel/lib -- see the ++# companion changes in tools/config/configure.R and src/install.libs.R -- so ++# honouring those variables at run time is actively harmful: .onLoad() would ++# dyn.load() a second, unrelated TBB into the process next to the bundled one ++# (two copies of the same symbols in the global scope, which segfaults R on ++# load), and RcppParallelLibs() / CxxFlags() would hand that system TBB to ++# dependents such as rstan, putting NEEDED libtbb.so.12 back into their ++# binaries. Read the variables only when explicitly opted back in. ++bincraftGetenv <- function(name, unset = "") { ++ if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") ++ Sys.getenv(name, unset = unset) ++ else ++ unset ++} +\ No newline at end of file +diff --git a/R/tbb.R b/R/tbb.R +index 152fd16..4b06aa5 100644 +--- a/R/tbb.R ++++ b/R/tbb.R +@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { + sysname <- Sys.info()[["sysname"]] + + # find root for TBB install +- tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) ++ tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) + if (is.null(name)) + return(tbbRoot) + +@@ -58,7 +58,7 @@ tbbCxxFlags <- function() { + flags <- c("-DRCPP_PARALLEL_USE_TBB=1") + + # if TBB_INC is set, apply those library paths +- tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) ++ tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) + if (!file.exists(tbbInc)) { + tbbInc <- system.file("include", package = "RcppParallel") + } +@@ -96,7 +96,7 @@ tbbLdFlags <- function() { + } + + # shortcut if TBB_LIB defined +- tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) ++ tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) + if (nzchar(tbbLib)) { + if (R.version$os == "emscripten") { + fmt <- "-L%1$s -l%2$s" +diff --git a/src/install.libs.R b/src/install.libs.R +index e770838..1dd2fc7 100644 +--- a/src/install.libs.R ++++ b/src/install.libs.R +@@ -492,6 +492,18 @@ prependFlags <- function(prependFlags, toFlags) { + tbbLib <- Sys.getenv("TBB_LIB") + tbbInc <- Sys.getenv("TBB_INC") + ++# bincraft patch: the companion change in tools/config/configure.R stops an ++# ambient TBB_LIB / TBB_INC from selecting a system TBB, but this script is ++# also run directly by `R CMD INSTALL` (not only through the `tbb` rule in ++# src/Makevars, which passes the configured values), so at install time it ++# still sees the image's environment and would symlink the system libraries ++# into RcppParallel/lib. Drop them here for the same reason, under the same ++# opt-out. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbLib <- "" ++ tbbInc <- "" ++} ++ + args <- commandArgs(trailingOnly = TRUE) + if (identical(args, "build")) { + if (nzchar(tbbLib) && nzchar(tbbInc)) { +diff --git a/tools/config/configure.R b/tools/config/configure.R +index c3842f9..adf7876 100644 +--- a/tools/config/configure.R ++++ b/tools/config/configure.R +@@ -66,6 +66,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) + tbbLib <- Sys.getenv("TBB_LIB", unset = NA) + tbbInc <- Sys.getenv("TBB_INC", unset = NA) + ++# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Our build ++# images exported these until build-env-images c51ed09 (a leftover from ++# RcppParallel 5.x, whose bundled Intel TBB would not build on musl or with ++# modern g++), and any of them switches the branches below to a system TBB. ++# The published binary then records NEEDED libtbb.so.12 (or libtbb.so.2 for ++# the classic Intel TBB) and gets a RcppParallel/lib full of absolute symlinks ++# into the image's library dir, so it cannot dyn.load on a consumer machine ++# without that exact TBB. 6.x bundles oneTBB 2022 and builds it with cmake on ++# every platform we ship, so the bundled copy is always the right choice here; ++# forcing it in the package rather than relying on the image environment keeps ++# the binary correct whichever image version CI happens to pull. Set ++# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbRoot <- NA ++ tbbLib <- NA ++ tbbInc <- NA ++} ++ + tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") + tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") + +@@ -146,6 +164,18 @@ define( + ) + + # set PKG_LIBS ++# ++# bincraft patch: the non-Windows library directories below are passed as ++# plain '-L', not '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, ++# /usr/lib/) into explicit '-L' options ahead of anything forwarded ++# verbatim with '-Wl,', so with '-Wl,-L' a system libtbb.so wins over the one ++# named here: on a build host with a distro TBB installed, '-ltbb' resolves to ++# that library and RcppParallel.so records its SONAME (libtbb.so.12, or ++# libtbb.so.2 for the classic Intel TBB) instead of the bundled 'libtbb.so'. ++# The binary then loads the system TBB rather than the copy shipped in ++# RcppParallel/lib, and fails outright on a machine that has no system TBB. ++# gcc places a plain '-L' before its built-in dirs, so the intended library is ++# found first. + pkgLibs <- if (.Platform$OS.type == "windows") { + + if (!is.na(tbbLib) && file.exists(file.path(tbbInc, "oneapi"))) { +@@ -183,7 +213,7 @@ pkgLibs <- if (.Platform$OS.type == "windows") { + } else if (!is.na(tbbLib)) { + + c( +- "-Wl,-L\"$(TBB_LIB)\"", ++ "-L\"$(TBB_LIB)\"", + sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" +@@ -192,14 +222,14 @@ pkgLibs <- if (.Platform$OS.type == "windows") { + } else if (R.version$os == "emscripten") { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)" + ) + + } else { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" + ) diff --git a/local/patches/RcppParallel/force-bundled-tbb.patch b/local/patches/RcppParallel/force-bundled-tbb-6.2.x.patch similarity index 67% rename from local/patches/RcppParallel/force-bundled-tbb.patch rename to local/patches/RcppParallel/force-bundled-tbb-6.2.x.patch index 5c37fa6..d88cfc5 100644 --- a/local/patches/RcppParallel/force-bundled-tbb.patch +++ b/local/patches/RcppParallel/force-bundled-tbb-6.2.x.patch @@ -1,5 +1,5 @@ diff --git a/R/aaa.R b/R/aaa.R -index 568a2aa..bfdab4c 100644 +index 568a2aa..2744059 100644 --- a/R/aaa.R +++ b/R/aaa.R @@ -5,4 +5,21 @@ TBB_LIB <- "" @@ -10,16 +10,16 @@ index 568a2aa..bfdab4c 100644 \ No newline at end of file +TBB_MALLOC_NAME <- "tbbmalloc" + -+# bincraft patch: our build images (and plenty of user environments) export -+# TBB_ROOT / TBB_LIB / TBB_INC. A binary we publish always carries its own -+# oneTBB in RcppParallel/lib -- see the companion changes in -+# tools/config/configure.R and src/install.libs.R -- so honouring those -+# variables at run time is actively harmful: .onLoad() would dyn.load() a -+# second, unrelated TBB into the process next to the bundled one (two copies -+# of the same symbols in the global scope, which segfaults R on load), and -+# RcppParallelLibs() / CxxFlags() would hand that system TBB to dependents -+# such as rstan, putting NEEDED libtbb.so.12 back into their binaries. Read -+# the variables only when explicitly opted back in. ++# bincraft patch: build images (ours did until build-env-images c51ed09) and ++# plenty of user environments export TBB_ROOT / TBB_LIB / TBB_INC. A binary we ++# publish always carries its own oneTBB in RcppParallel/lib -- see the ++# companion changes in tools/config/configure.R and src/install.libs.R -- so ++# honouring those variables at run time is actively harmful: .onLoad() would ++# dyn.load() a second, unrelated TBB into the process next to the bundled one ++# (two copies of the same symbols in the global scope, which segfaults R on ++# load), and RcppParallelLibs() / CxxFlags() would hand that system TBB to ++# dependents such as rstan, putting NEEDED libtbb.so.12 back into their ++# binaries. Read the variables only when explicitly opted back in. +bincraftGetenv <- function(name, unset = "") { + if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") + Sys.getenv(name, unset = unset) @@ -28,32 +28,24 @@ index 568a2aa..bfdab4c 100644 +} \ No newline at end of file diff --git a/R/tbb.R b/R/tbb.R -index 6f6a745..e407986 100644 +index b071833..e3bf7a5 100644 --- a/R/tbb.R +++ b/R/tbb.R -@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { - sysname <- Sys.info()[["sysname"]] - +@@ -19,3 +19,3 @@ tbbLibraryPath <- function(name = NULL) { # find root for TBB install - tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) + tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) if (is.null(name)) - return(tbbRoot) - -@@ -58,3 +58,3 @@ tbbCxxFlags <- function() { +@@ -63,3 +63,3 @@ tbbCxxFlags <- function() { # if TBB_INC is set, apply those library paths - tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) + tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) if (!file.exists(tbbInc)) { -@@ -117,7 +117,7 @@ tbbLdFlags <- function() { - } - +@@ -122,3 +122,3 @@ tbbLdFlags <- function() { # shortcut if TBB_LIB defined - tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) + tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) if (nzchar(tbbLib)) { - if (R.version$os == "emscripten") { - fmt <- "-L%1$s -l%2$s" diff --git a/src/install.libs.R b/src/install.libs.R index 3b3cfda..c0e6f3e 100644 --- a/src/install.libs.R @@ -78,24 +70,24 @@ index 3b3cfda..c0e6f3e 100644 if (identical(args, "build")) { if (nzchar(tbbLib) && nzchar(tbbInc)) { diff --git a/tools/config/configure.R b/tools/config/configure.R -index 6293fe1..eae4aaa 100644 +index c203a1f..0050b5c 100644 --- a/tools/config/configure.R +++ b/tools/config/configure.R -@@ -40,6 +40,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) +@@ -79,6 +79,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) tbbLib <- Sys.getenv("TBB_LIB", unset = NA) tbbInc <- Sys.getenv("TBB_INC", unset = NA) -+# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Several of -+# our build images export these (a leftover from RcppParallel 5.x, whose -+# bundled Intel TBB would not build on musl or with modern g++), and any of -+# them switches the branches below to a system TBB. The published binary then -+# records NEEDED libtbb.so.12 (or libtbb.so.2 for the classic Intel TBB) and -+# gets a RcppParallel/lib full of absolute symlinks into the image's library -+# dir, so it cannot dyn.load on a consumer machine without that exact TBB. -+# 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, -+# so the bundled copy is always the right choice here; forcing it in the -+# package rather than relying on the image environment keeps the binary -+# correct whichever image version CI happens to pull. Set ++# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Our build ++# images exported these until build-env-images c51ed09 (a leftover from ++# RcppParallel 5.x, whose bundled Intel TBB would not build on musl or with ++# modern g++), and any of them switches the branches below to a system TBB. ++# The published binary then records NEEDED libtbb.so.12 (or libtbb.so.2 for ++# the classic Intel TBB) and gets a RcppParallel/lib full of absolute symlinks ++# into the image's library dir, so it cannot dyn.load on a consumer machine ++# without that exact TBB. 6.x bundles oneTBB 2022 and builds it with cmake on ++# every platform we ship, so the bundled copy is always the right choice here; ++# forcing it in the package rather than relying on the image environment keeps ++# the binary correct whichever image version CI happens to pull. Set +# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. +if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { + tbbRoot <- NA @@ -106,7 +98,7 @@ index 6293fe1..eae4aaa 100644 tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") -@@ -186,12 +204,23 @@ define( +@@ -225,12 +243,23 @@ define( ) # set PKG_LIBS @@ -131,7 +123,7 @@ index 6293fe1..eae4aaa 100644 if (.Platform$OS.type != "windows") sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), "-l$(TBB_NAME)", -@@ -201,14 +230,14 @@ pkgLibs <- if (!is.na(tbbLib)) { +@@ -240,14 +269,14 @@ pkgLibs <- if (!is.na(tbbLib)) { } else if (R.version$os == "emscripten") { c( diff --git a/local/patches/registry.json b/local/patches/registry.json index 2457304..1f341b7 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -6,8 +6,48 @@ "env": {}, "configure_args": [], "makevars": {}, - "patch": "RcppParallel/force-bundled-tbb.patch", - "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, so the system-TBB workaround needed for 5.x is gone, but two things still steer the build back to a system TBB. (1) Ambient TBB_ROOT/TBB_LIB/TBB_INC: several build images still export these (build-env-images dropped them, but the images are rebuilt only by cron/manual runs, so a stale image keeps them), and R reads ~/.Renviron *after* the process environment, so no pipeline-side env override can undo it. configure.R then takes the system-TBB branch, install.libs.R symlinks the image's libraries into RcppParallel/lib as absolute paths, and the binary records NEEDED libtbb.so.12 (libtbb.so.2 for the classic Intel TBB on el8/el9) -- it cannot dyn.load on a consumer machine without that exact TBB. Both files are patched to ignore those variables (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE); install.libs.R needs it separately because R CMD INSTALL runs it outside the src/Makevars rule that passes the configured values. (2) Link order: the bundled branch passes its build dir as '-Wl,-Ltbb/build/lib_release', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,', so '-ltbb' would still resolve to a distro TBB when one is installed; a plain '-L' puts the bundled dir first. Verified on the current build-env-ubuntu:jammy image (which still exports TBB_INC/TBB_LIB): NEEDED libtbb.so, RUNPATH $ORIGIN/../lib, real libtbb.so.2 in RcppParallel/lib, and the package loads with every system libtbb moved away." + "patch": "RcppParallel/force-bundled-tbb-6.2.x.patch", + "reason": "RcppParallel 6.x builds its bundled oneTBB 2022 with cmake, but tools/config/configure.R passes the library dir as '-Wl,-L...', and gcc places its own search dirs (/usr/lib64, /usr/lib/) ahead of anything forwarded with '-Wl,'. Every build image installs a distro libtbb, so '-ltbb' resolves to it: the binary records NEEDED libtbb.so.12 (libtbb.so.2 on el8/el9) instead of the bundled libtbb.so and cannot load on a machine without that exact system TBB. The patch passes a plain '-L' so the bundled build dir wins. As a safeguard it also makes configure.R, src/install.libs.R and R/tbb.R ignore an ambient TBB_ROOT/TBB_LIB/TBB_INC (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE): the build images stopped exporting them in build-env-images c51ed09, but a stale image or ~/.Renviron would otherwise switch the build back to the system TBB, and R reads ~/.Renviron after the process environment, so no pipeline env can override it. The read sites move between releases, so each entry pins a patch generated against that version. Upstream still uses '-Wl,-L' in 6.2.1; drop these entries once a release passes a plain '-L'." + }, + { + "package": "RcppParallel", + "versions": "6.2.0", + "platforms": ["*"], + "env": {}, + "configure_args": [], + "makevars": {}, + "patch": "RcppParallel/force-bundled-tbb-6.2.x.patch", + "reason": "RcppParallel 6.x builds its bundled oneTBB 2022 with cmake, but tools/config/configure.R passes the library dir as '-Wl,-L...', and gcc places its own search dirs (/usr/lib64, /usr/lib/) ahead of anything forwarded with '-Wl,'. Every build image installs a distro libtbb, so '-ltbb' resolves to it: the binary records NEEDED libtbb.so.12 (libtbb.so.2 on el8/el9) instead of the bundled libtbb.so and cannot load on a machine without that exact system TBB. The patch passes a plain '-L' so the bundled build dir wins. As a safeguard it also makes configure.R, src/install.libs.R and R/tbb.R ignore an ambient TBB_ROOT/TBB_LIB/TBB_INC (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE): the build images stopped exporting them in build-env-images c51ed09, but a stale image or ~/.Renviron would otherwise switch the build back to the system TBB, and R reads ~/.Renviron after the process environment, so no pipeline env can override it. The read sites move between releases, so each entry pins a patch generated against that version. Upstream still uses '-Wl,-L' in 6.2.1; drop these entries once a release passes a plain '-L'." + }, + { + "package": "RcppParallel", + "versions": "6.1.1", + "platforms": ["*"], + "env": {}, + "configure_args": [], + "makevars": {}, + "patch": "RcppParallel/force-bundled-tbb-6.1.x.patch", + "reason": "RcppParallel 6.x builds its bundled oneTBB 2022 with cmake, but tools/config/configure.R passes the library dir as '-Wl,-L...', and gcc places its own search dirs (/usr/lib64, /usr/lib/) ahead of anything forwarded with '-Wl,'. Every build image installs a distro libtbb, so '-ltbb' resolves to it: the binary records NEEDED libtbb.so.12 (libtbb.so.2 on el8/el9) instead of the bundled libtbb.so and cannot load on a machine without that exact system TBB. The patch passes a plain '-L' so the bundled build dir wins. As a safeguard it also makes configure.R, src/install.libs.R and R/tbb.R ignore an ambient TBB_ROOT/TBB_LIB/TBB_INC (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE): the build images stopped exporting them in build-env-images c51ed09, but a stale image or ~/.Renviron would otherwise switch the build back to the system TBB, and R reads ~/.Renviron after the process environment, so no pipeline env can override it. The read sites move between releases, so each entry pins a patch generated against that version. Upstream still uses '-Wl,-L' in 6.2.1; drop these entries once a release passes a plain '-L'." + }, + { + "package": "RcppParallel", + "versions": "6.1.0", + "platforms": ["*"], + "env": {}, + "configure_args": [], + "makevars": {}, + "patch": "RcppParallel/force-bundled-tbb-6.1.x.patch", + "reason": "RcppParallel 6.x builds its bundled oneTBB 2022 with cmake, but tools/config/configure.R passes the library dir as '-Wl,-L...', and gcc places its own search dirs (/usr/lib64, /usr/lib/) ahead of anything forwarded with '-Wl,'. Every build image installs a distro libtbb, so '-ltbb' resolves to it: the binary records NEEDED libtbb.so.12 (libtbb.so.2 on el8/el9) instead of the bundled libtbb.so and cannot load on a machine without that exact system TBB. The patch passes a plain '-L' so the bundled build dir wins. As a safeguard it also makes configure.R, src/install.libs.R and R/tbb.R ignore an ambient TBB_ROOT/TBB_LIB/TBB_INC (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE): the build images stopped exporting them in build-env-images c51ed09, but a stale image or ~/.Renviron would otherwise switch the build back to the system TBB, and R reads ~/.Renviron after the process environment, so no pipeline env can override it. The read sites move between releases, so each entry pins a patch generated against that version. Upstream still uses '-Wl,-L' in 6.2.1; drop these entries once a release passes a plain '-L'." + }, + { + "package": "RcppParallel", + "versions": "6.0.0", + "platforms": ["*"], + "env": {}, + "configure_args": [], + "makevars": {}, + "patch": "RcppParallel/force-bundled-tbb-6.0.0.patch", + "reason": "RcppParallel 6.x builds its bundled oneTBB 2022 with cmake, but tools/config/configure.R passes the library dir as '-Wl,-L...', and gcc places its own search dirs (/usr/lib64, /usr/lib/) ahead of anything forwarded with '-Wl,'. Every build image installs a distro libtbb, so '-ltbb' resolves to it: the binary records NEEDED libtbb.so.12 (libtbb.so.2 on el8/el9) instead of the bundled libtbb.so and cannot load on a machine without that exact system TBB. The patch passes a plain '-L' so the bundled build dir wins. As a safeguard it also makes configure.R, src/install.libs.R and R/tbb.R ignore an ambient TBB_ROOT/TBB_LIB/TBB_INC (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE): the build images stopped exporting them in build-env-images c51ed09, but a stale image or ~/.Renviron would otherwise switch the build back to the system TBB, and R reads ~/.Renviron after the process environment, so no pipeline env can override it. The read sites move between releases, so each entry pins a patch generated against that version. Upstream still uses '-Wl,-L' in 6.2.1; drop these entries once a release passes a plain '-L'." }, { "package": "fs", From 3426d5f59f28d1a91ef18115284f0eaa5c4de6e7 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sun, 11 Oct 2026 00:32:41 +0000 Subject: [PATCH 116/118] chore(deps): update dependency nbafrank/uvr to v0.4.7 --- local/uvr-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local/uvr-install.sh b/local/uvr-install.sh index ed6b1ce..e103e13 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -30,7 +30,7 @@ set -eu # renovate: datasource=github-releases depName=nbafrank/uvr -UVR_PIN="v0.4.6" +UVR_PIN="v0.4.7" if [ "$#" -eq 0 ]; then echo "usage: $0 ..." >&2 From d992a2b6361439966ffbc70088bacec7b664c78b Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 11 Oct 2026 12:12:28 +0000 Subject: [PATCH 117/118] fix(ci): give each xvfb-run its own display (#230) Every `process-cran-updates-ubuntu-2204-arm64` cron run since pipeline 13744 has failed with `xvfb-run: error: Xvfb failed to start` (e.g. 13829). ## Cause The newer-R loop added in #216 runs two `xvfb-run` calls back to back: the R-minor-sensitive `process_cran_updates` pass, then `local/build-newer-r.R`. Without `-a`, `xvfb-run` always uses display `:99`, and on exit it kills its Xvfb without waiting for it. The second call can therefore start while the old server still holds the `:99` lock, so the new Xvfb refuses to start. Only ubuntu-2204-arm64 hits this because it is the only image with a second R minor installed, so the loop body only runs there. ## Change - Default `XVFB_ARGS` to `-a` (`--auto-servernum`) in `process-updates.yaml`, `build-all-versions.yaml` and `weekly-rebuild-missing.yaml`, so each call picks a free display. - The RHEL 10 `xwfb-run` branch still overrides `XVFB_ARGS` with `-c weston`, so it is unchanged. ## Verification `crow lint .crow/` passes. A local repro was not possible (`docker run` hangs on the dev host), so the next ubuntu-2204-arm64 cron run is the real check. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/230 --- .crow/build-all-versions.yaml | 2 +- .crow/process-updates.yaml | 2 +- .crow/weekly-rebuild-missing.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 7451988..ce3bc09 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -153,7 +153,7 @@ steps: # not visible here. Install the packages needed to recompute the snapshot # and query build metadata in every build job. - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index a1b9698..531a47f 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -192,7 +192,7 @@ steps: - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 955476e..a93a7bd 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -388,7 +388,7 @@ steps: - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 From 5dad51c76a1136c003853bb6c84fbef9dc798e4d Mon Sep 17 00:00:00 2001 From: automation-bot Date: Mon, 12 Oct 2026 00:32:11 +0000 Subject: [PATCH 118/118] chore(deps): update pre-commit hook rbubley/mirrors-prettier to v3.9.10 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dec6844..e0ba9a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.9 + rev: v3.9.10 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit