Compare commits

...
Author SHA1 Message Date
0aeaed4a99
fix(patches): build-one image bincraft v4.3.0 + ship patch registry
The targeted rebuild path pinned bincraft v4.2.1, which predates the patches
argument, so build-one.R's patches= call would error. Bump to v4.3.0 (the
tagged release that adds build_binary_package(patches=)).
Also COPY the patch registry into the image so patches='local/patches'
resolves at /work/local/patches instead of silently no-opping.
2026-06-30 14:11:52 +02:00
1fd8d811e7
ci(patches): wire patches into all build workflows; pin bincraft v4.3.1
Pass patches = 'local/patches' to the daily process_cran_updates() calls
(requires bincraft v4.3.1) and the weekly-rebuild build_binary_package() call,
so every CI build path applies the patch registry like build-all/build-one
already do.
Bump the bincraft pin v4.3.0 -> v4.3.1 across all .crow workflows.
2026-06-30 10:46:18 +02:00
48f39d2b03
ci(patches): bump bincraft pin to v4.3.0
v4.3.0 adds the `patches` argument that build-one.R/build-all.R now pass.
Updates the install pin (and version guard) across all .crow workflows so CI
installs a bincraft that accepts the argument.
2026-06-30 10:21:00 +02:00
e802124ec3
chore(patches): untrack SDD scratch, exclude design docs from markdownlint, fix README table
Remove the accidentally-committed .superpowers/ scratch (briefs/reports) and
gitignore it.
Exclude specs/ and plans/ design docs from markdownlint (like the existing
docs/superpowers/** exclusion).
Pad the registry README table separator so it satisfies MD060.
2026-06-30 09:58:04 +02:00
f3f3bfa20d
fix(patches): validator portability, clearer docs, broader hook trigger 2026-06-30 09:17:24 +02:00
a826a27246
docs(patches): document the package patching workflow 2026-06-30 08:54:59 +02:00
bda1a375d7
feat(patches): pass patch registry to bincraft build calls 2026-06-30 08:53:38 +02:00
912423f9d4
ci(patches): validate patch registry in pre-commit 2026-06-30 08:52:18 +02:00
7e382c7941
fix(patches): report validator errors cleanly instead of crashing on malformed entries 2026-06-30 08:50:44 +02:00
e3e219e095
feat(patches): add registry validator script 2026-06-30 08:49:21 +02:00
14 changed files with 111 additions and 16 deletions

View file

@ -62,7 +62,7 @@ steps:
GIT_USER: pat-s GIT_USER: pat-s
R_VERSION: 4.5.3 R_VERSION: 4.5.3
commands: commands:
- /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1", dependencies = TRUE)'
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
- /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)'
backend_options: backend_options:

View file

@ -75,7 +75,7 @@ steps:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
# Pin the same bincraft version the build steps use, so the precomputed # Pin the same bincraft version the build steps use, so the precomputed
# snapshot and the per-agent library stay consistent across the pipeline. # snapshot and the per-agent library stay consistent across the pipeline.
- /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")'
- /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))"
backend_options: backend_options:
docker: docker:

View file

@ -121,7 +121,7 @@ steps:
# to a zero-length value and breaks every metadata query and the sysdeps # to a zero-length value and breaks every metadata query and the sysdeps
# install). Pin bincraft here, exactly like the R-minor pass below. # install). Pin bincraft here, exactly like the R-minor pass below.
- rm -rf /mnt/cache/R-pkgs/00LOCK-* - rm -rf /mnt/cache/R-pkgs/00LOCK-*
- /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")'
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
- $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1
- | - |
@ -133,7 +133,7 @@ steps:
echo "=== R-minor-sensitive pass under R $RV ===" echo "=== R-minor-sensitive pass under R $RV ==="
LIB="/mnt/cache/R-pkgs-$RMINOR" LIB="/mnt/cache/R-pkgs-$RMINOR"
mkdir -p "$LIB" mkdir -p "$LIB"
R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' || true R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' || true
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true
done done
# archive missed packages; first arg is the codename (e.g. "alpine324"), # archive missed packages; first arg is the codename (e.g. "alpine324"),

View file

@ -192,13 +192,13 @@ steps:
commands: commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache
- /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")'
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
# rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"
- | - |
PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2)
for RBIN in /opt/R/[0-9]*/bin/R; do for RBIN in /opt/R/[0-9]*/bin/R; do
@ -208,8 +208,8 @@ steps:
echo "=== R-minor-sensitive update pass under R $RV ===" echo "=== R-minor-sensitive update pass under R $RV ==="
LIB="/mnt/cache/R-pkgs-$RMINOR" LIB="/mnt/cache/R-pkgs-$RMINOR"
mkdir -p "$LIB" mkdir -p "$LIB"
R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' || true R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' || true
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true
done done
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
- | - |

View file

@ -129,12 +129,12 @@ steps:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
- rm -rf /mnt/cache/R-pkgs/00LOCK-* - rm -rf /mnt/cache/R-pkgs/00LOCK-*
- /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")'
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")'
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1
backend_options: backend_options:
docker: docker:
resources: resources:

3
.gitignore vendored
View file

@ -99,3 +99,6 @@ docs/
local/test.R local/test.R
.DS_Store .DS_Store
docs/ docs/
# Superpowers SDD scratch (briefs, reports, ledger)
.superpowers/

View file

@ -2,3 +2,6 @@
ignores: ignores:
- LICENSE.md - LICENSE.md
- docs/superpowers/** - docs/superpowers/**
# Internal design docs (specs/plans) are not user-facing reference material.
- specs/**
- plans/**

View file

@ -43,3 +43,9 @@ repos:
entry: YAML filenames must have .yaml extension. entry: YAML filenames must have .yaml extension.
language: fail language: fail
files: .yml$ files: .yml$
- id: validate-patches
name: validate patch registry
entry: Rscript local/validate-patches.R
language: system
files: ^local/(patches/|validate-patches\.R$)
pass_filenames: false

View file

@ -53,6 +53,28 @@ For every package+tag combination:
1. Archive old package versions and keep the latest one in the root 1. Archive old package versions and keep the latest one in the root
1. Delete local binaries after successful upload 1. Delete local binaries after successful upload
## Patching packages
Some CRAN packages fail to compile on specific platforms due to compiler- or OS-specific issues unrelated to the package itself.
The canonical example is `RcppParallel`, whose bundled TBB sources fail on musl (Alpine) and newer compiler/OS combinations.
Because such packages are often transitive dependencies of many others, a single failure cascades: all dependents fail even though nothing is wrong with the dependent itself.
To address this, frequently-failing packages can be "patched" before they are installed — whether as a direct build target or a transitive dependency pulled in by `pak`.
The patch registry lives in `local/patches/registry.json`.
Each entry specifies a package and the platforms/versions it applies to, along with either lightweight build-time overrides (environment variables, configure arguments, Makevars) or a source diff (for deeper fixes).
See `local/patches/README.md` for the complete schema.
Patching uses a two-tier approach:
1. **Lightweight overrides:** environment variables, configure arguments, or Makevars settings applied during build — typically version-independent and fast.
2. **Source diffs:** unified diff patches applied to the unpacked source before building — more powerful but version-pinned.
The system is implemented in `bincraft`: when a package needs patching, `bincraft` pre-builds it with the patch and serves the patched binary to `pak`, ensuring transitive dependents receive the fixed package.
This way, the fix cascades to all packages that depend on it.
For the design rationale and architecture, see `specs/2026-06-30-package-patching-design.md`.
## Build Environment ## Build Environment
Binaries are built on a mixed-architecture Kubernetes cluster using CI. Binaries are built on a mixed-architecture Kubernetes cluster using CI.

View file

@ -16,6 +16,9 @@ ARG CACHEBUST
WORKDIR /work WORKDIR /work
COPY build-one.R /work/build-one.R COPY build-one.R /work/build-one.R
# Ship the patch registry so build-one.R's `patches = "local/patches"` resolves
# (build context is `local/`, CWD is /work).
COPY patches /work/local/patches
RUN --mount=type=secret,id=b2_access,required=true \ RUN --mount=type=secret,id=b2_access,required=true \
--mount=type=secret,id=b2_secret,required=true \ --mount=type=secret,id=b2_secret,required=true \
@ -37,7 +40,7 @@ RUN --mount=type=secret,id=b2_access,required=true \
echo "No working virtual display; building without xvfb" >&2; \ echo "No working virtual display; building without xvfb" >&2; \
fi; \ fi; \
run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \ run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \
ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")'; }; \ ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.0")'; }; \
PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \
seen=" $PRIMARY_MINOR "; \ seen=" $PRIMARY_MINOR "; \
prc=0; failed=""; \ prc=0; failed=""; \

View file

@ -140,6 +140,7 @@ mapply(
metadata_db_sslmode = "require", metadata_db_sslmode = "require",
metadata_db_port = 15432, metadata_db_port = 15432,
archive = TRUE, archive = TRUE,
patches = "local/patches",
upload = TRUE, upload = TRUE,
store_build_metadata = TRUE store_build_metadata = TRUE
) )

View file

@ -103,6 +103,7 @@ for (ver in versions) {
force = TRUE, force = TRUE,
upload = TRUE, upload = TRUE,
archive = TRUE, archive = TRUE,
patches = "local/patches",
store_build_metadata = TRUE, store_build_metadata = TRUE,
s3_endpoint = s3$s3_endpoint, s3_endpoint = s3$s3_endpoint,
s3_region = s3$s3_region, s3_region = s3$s3_region,

View file

@ -9,7 +9,7 @@ The registry is defined in `registry.json` as an array of patch entries. Each en
### Field semantics ### Field semantics
| Field | Type | Required | Description | | Field | Type | Required | Description |
|-------|------|----------|-------------| | --- | --- | --- | --- |
| `package` | string | yes | CRAN package name. | | `package` | string | yes | CRAN package name. |
| `versions` | string | yes | `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. | | `versions` | string | yes | `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. |
| `platforms` | array of strings | yes | Matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). An entry matches if any listed token matches any build token. `["*"]` matches all platforms. | | `platforms` | array of strings | yes | Matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). An entry matches if any listed token matches any build token. `["*"]` matches all platforms. |
@ -34,10 +34,10 @@ To add a new patch entry:
## Validation ## Validation
The registry is validated and applied by bincraft during the build process. The registry is validated and applied by bincraft during the build process.
For manual validation, use: For manual validation, run the validator from the repo root:
```r ```bash
x <- jsonlite::fromJSON("local/patches/registry.json", simplifyVector = FALSE) Rscript local/validate-patches.R
``` ```
This loads the registry; inspect the structure to verify correctness. This validates the schema, referenced patch-file existence, and checks for duplicate entries across platforms and versions.

56
local/validate-patches.R Normal file
View file

@ -0,0 +1,56 @@
#!/usr/bin/env Rscript
# Validate local/patches/registry.json: schema, referenced patch files, and
# ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI.
dir <- "local/patches"
registry_file <- file.path(dir, "registry.json")
if (!file.exists(registry_file)) {
cat("No registry.json found; nothing to validate.\n")
quit(status = 0L)
}
or_q <- function(x) if (is.null(x)) "?" else x
reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE)
required <- c("package", "versions", "platforms", "reason")
errs <- character(0L)
for (i in seq_along(reg)) {
e <- reg[[i]]
missing <- setdiff(required, names(e))
if (length(missing) > 0L) {
errs <- c(errs, sprintf(
"entry %d (%s): missing %s", i,
if (is.null(e$package)) "?" else e$package, toString(missing)
))
}
if (!is.null(e$patch)) {
p <- file.path(dir, e$patch)
if (!file.exists(p)) {
errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing",
i, e$package, p))
}
}
}
# Ambiguous overlap: two entries for the same package with identical platforms
# and versions.
keys <- vapply(reg, function(e) {
sprintf(
"%s|%s|%s",
or_q(e$package),
paste(sort(as.character(unlist(e$platforms))), collapse = ","),
or_q(e$versions)
)
}, character(1L))
dups <- keys[duplicated(keys)]
if (length(dups) > 0L) {
errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups))))
}
if (length(errs) > 0L) {
cat("Patch registry validation FAILED:\n")
cat(paste0(" - ", errs, "\n"))
quit(status = 1L)
}
cat(sprintf("Patch registry OK (%d %s).\n", length(reg), if (length(reg) == 1L) "entry" else "entries"))