diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index f1fe920..71f808b 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,9 +62,9 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), workers = 2L)' + - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' backend_options: kubernetes: resources: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 389b57b..e0cca86 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -10,11 +10,22 @@ variables: - arm64 default: amd64 OS: - description: 'Base OS image name (e.g. alpine, redhat, ubuntu).' + description: "Base OS image name." + options: + - alpine + - redhat + - ubuntu default: alpine OS_VERSION: - description: 'OS version / image tag (e.g. 3.24, 9, jammy, noble).' - default: '3.24' + description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + options: + - "3.24" + - "8" + - "9" + - "10" + - "jammy" + - "noble" + default: "3.24" R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -42,6 +53,9 @@ steps: image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN GITHUB_PAT: @@ -66,8 +80,9 @@ steps: - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + # Pin the same bincraft version the build steps use, so the precomputed + # snapshot and the per-agent library stay consistent across the pipeline. + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 1d4feee..3932811 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -11,11 +11,25 @@ variables: - arm64 default: amd64 OS: - description: 'Base OS image name (e.g. alpine, redhat, ubuntu).' + description: "Base OS image name." + options: + - alpine + - redhat + - ubuntu default: alpine OS_VERSION: - description: 'OS version / image tag (e.g. 3.24, 9, jammy, noble).' - default: '3.24' + description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + options: + - "3.22" + - "3.23" + - "3.24" + - "8" + - "9" + - "10" + - "jammy" + - "noble" + - "resolute" + default: "3.24" R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -76,6 +90,9 @@ steps: image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW B2_S3_ACCESS_KEY: @@ -105,6 +122,14 @@ steps: # growth is bounded separately by trim_pkgcache_metadata() in build-all.R. - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # The primary pass must not rely on build-all-versions-install-deps having + # run on *this* agent: depends_on only orders the steps, but the cache + # volume is per-agent, so a job landing on an agent where install-deps did + # not run would otherwise use a stale bincraft (which resolves `platform` + # to a zero-length value and breaks every metadata query and the sysdeps + # install). Pin bincraft here, exactly like the R-minor pass below. + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -116,11 +141,12 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done - # archive missed packages - - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" + # archive missed packages; first arg is the codename (e.g. "alpine324"), + # derived via bincraft like the upload step, not paste(OS, OS_VERSION). + - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(bincraft::set_codename(NULL), Sys.getenv('ARCH'), s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), workers = $NCPUS)" backend_options: docker: resources: @@ -134,6 +160,9 @@ steps: image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none B2_S3_ACCESS_KEY: from_secret: B2_S3_ACCESS_KEY B2_S3_SECRET_KEY: diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 9d40c67..3da41fd 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -15,6 +15,8 @@ variables: - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -25,6 +27,8 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 default: all when: @@ -64,6 +68,18 @@ matrix: IMG: alpine:3.24 OS_ID: alpine323 PROCESS_NEW: "FALSE" + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + OS_ID: alpine324 + PROCESS_NEW: "FALSE" + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + OS_ID: alpine324 + PROCESS_NEW: "FALSE" - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 @@ -124,12 +140,27 @@ matrix: IMG: ubuntu:noble OS_ID: noble PROCESS_NEW: "TRUE" + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + OS_ID: resolute + PROCESS_NEW: "TRUE" + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + OS_ID: resolute + PROCESS_NEW: "TRUE" steps: - name: 'Processing Updates' image: reg.devxy.io/rpkgs/build-env-${IMG} pull: true environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW B2_S3_ACCESS_KEY: @@ -164,13 +195,13 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) for RBIN in /opt/R/[0-9]*/bin/R; do @@ -180,8 +211,8 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | @@ -202,6 +233,9 @@ steps: - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY SUBDOMAIN1: 'cran.devxy.io' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 843429e..934c518 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -103,6 +103,9 @@ steps: image: reg.devxy.io/rpkgs/build-env-${IMG} pull: true environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW B2_S3_ACCESS_KEY: @@ -129,12 +132,12 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: docker: resources: diff --git a/.gitignore b/.gitignore index 94e2c2f..f4124a2 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,6 @@ docs/ local/test.R .DS_Store docs/ + +# Superpowers SDD scratch (briefs, reports, ledger) +.superpowers/ diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml index 66e7cc8..032c0ea 100644 --- a/.markdownlint-cli2.yaml +++ b/.markdownlint-cli2.yaml @@ -2,3 +2,6 @@ ignores: - LICENSE.md - docs/superpowers/** + # Internal design docs (specs/plans) are not user-facing reference material. + - specs/** + - plans/** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6784ad2..a0f4422 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,19 +16,19 @@ repos: args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.22.1 + rev: v0.23.0 hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.8.4 + rev: v3.9.4 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit - rev: 0.9.0 + rev: 0.10.0 hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.7.0 + rev: v3.8.0 hooks: - id: editorconfig-checker - repo: https://github.com/adrienverge/yamllint.git @@ -43,3 +43,9 @@ repos: entry: YAML filenames must have .yaml extension. language: fail files: .yml$ + - id: validate-patches + name: validate patch registry + entry: Rscript local/validate-patches.R + language: system + files: ^local/(patches/|validate-patches\.R$) + pass_filenames: false diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 2ae039e..6ef10fa 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.14.3" - constraints = "~> 0.14" + version = "0.15.1" + constraints = "~> 0.15" hashes = [ - "h1:2VmbbvV/3fVb/dHZ5m3CxSPoptpn/PuhvSgV/RA9Ae8=", - "h1:4MANuttWPyJGZlfbx5P4i2Jpbmvda1tfWgd3olK2nhY=", - "h1:AsSfDvm6flYQgjFnFVaRPv7v1/P5pj36dQ4N103Nw+A=", - "h1:D2AK5psWQmZqrOHsbeV70OP/zcG92i++AmGKBz/HVMA=", - "h1:DpJWi/bhHoZa/ccy2YB/c7HJjTlR6CuZBxGFCxU2uJk=", - "h1:Iz47FCzHBOZC1RrTxKwJwKsH6bdEvdYtO0+RBpbmz8I=", - "h1:RJQm4qEfljAmcJaFf45uXwUhtbRTwfDuiUWZ6Q7X+Cw=", - "h1:YQBi0MRW+dOfkxQYbjKkzDm7UIcc2FeFm8Cck+W3kgs=", - "h1:bWq/zJ2Chr5TNuz9y3eaGmzrmXsX30pl5HkGpxE1kVM=", - "h1:hwzEskDnFHaJFvKNAGv/wIxRT2SHcXLfwmBb8XNspq4=", - "h1:jGggry0lZmG0S0inM7Odae3275na9Gp3O97ZlvBiA2s=", - "h1:mJdpVc9qOpqXcPpptWfAET8qloo6qFMb+NzHesazybk=", - "h1:nbpkaopSGlmxkKLtNe9sKx9IqojoicD1+hNRdaYd5ok=", - "h1:pwuoUqikuVOut3RtPTnu6pNoDnyeXlEX7Rfb2Hr1Ch0=", - "h1:u0naFiWGrBVhI3AsUb/ZfS5N/GeiRk0ZkxjHtA10lF0=", - "h1:uozoSPzZ7MJIHj5O62W8n86NIfH4JtF7GvMNI6rKlrM=", - "h1:vDS308OeXVVgjojMCAAEQ1VshmOLjwra7obliKB5fv8=", - "zh:23200ad919df4c44a720c667c536d3a0ae0c235e1f7b53b9e56f88005bf01891", - "zh:2fe5445f578c6a244995538fde6dd7ef0dd5f3969686cba3d91fcf28ea31278d", - "zh:3a5a8139ed177ccbee74feb4d19c881e18723e4bf3a64b6d35931420402a2a1a", - "zh:5189bd6c03ec3c428185992f519f3bf520d4e9eba8771c240814adbecdf60140", - "zh:532ac8eb94a84ccf046d4cb4eded825269effdbb7597f392f848fbfad5dcae1d", - "zh:5f100279d73ce998c7bf4ffa2bbcd617ee307bf51c4a94146f210aeec268c4d0", - "zh:80cb58abd2e431386ab1b6ad186a6e6e5347448993bacc831c0251b9016445b9", + "h1:/2NUpbtjkc+w6n5V3kGP0rSzGjN0K2Wdfe2K+CZdmhU=", + "h1:1TOrpmCR0aT5xX1sjt70zqlYkMJnVe6r0nx2B0DS/mE=", + "h1:4uC8r8ILr+vZJ230uGqIUaWQ7uORCrIzoEuYrAq2JuE=", + "h1:6b+2osJUfwcaYZ6mathLPf/58sr/4XkLXQrcSufnkMk=", + "h1:CPldfjf79QS8mIP+GWoS0FVhrFlyjvN2+39zfyP76Ik=", + "h1:EvXICHyGIKpoYlDeKHOPzfmpvdRdhgsOMcR2nb6+tKY=", + "h1:GZLq+nDxS1CyBH0ELGTSQj9X7ozemJ1jpPA4KwbtR+c=", + "h1:ISNFqL745IQgZ6yMLy8ofV8ixbYqZYa9JKdi2W3pmNk=", + "h1:IrNrEuvFd0nYDGQefwmT8d1CSJb9e8LN5w9vw1ODp7E=", + "h1:Ms79slY9bZ94+n4cwIHvI9+/cvbucwo6S1+z5KAiznw=", + "h1:NWA9XSEBcpSkgwwIvl6tHrxGQY3uYhqNS4Vnb9RLyLQ=", + "h1:UjvxxxggicLtiE3yTe1Gx0oLUTeZpWmgIfXuHzWHn1c=", + "h1:VgJjo14DGkU4Jwo4D3GT4/5sq1tdjiZscKS5l3cb890=", + "h1:dBu3AW5YNLIvbBIMNk3wUHKw4TW+BDbj34a+mCqYhWE=", + "h1:i5oGD06nQ3JRsBIa2u4wCej+ETgp970CFl75dOKkHno=", + "h1:mTqR+vD1AWPx+mu7S0/pzBy71z7WOrjH7arOP77PXh4=", + "h1:nmTM61G8vYjpofeEqspMORpsNvTGCNZySGfjdXardL0=", + "zh:0f9bf5aaa47164a4d6ae4433d5e285a9456a5053401b2bad4ed68622f574ddee", + "zh:3039bee421fb8855a919f449fc731d145254371f5f0c39cc4660f3aeed6a8b10", + "zh:36664b08186e0c194747b18dee24ed97327c6e704133d4cf0df27abef1652f86", + "zh:3c7eae99d8c5ff65dfb99c8b9c1980147282d68971e1ef1ff0f126a6bff59d8f", + "zh:5293cc21abf54f4e5745437ca2d40d206aae323b2e1d41cf45dcc63a8868cbb3", + "zh:5994e5145e616e7e881010717e4c7def2945eb6d933f62db4ec3167732ccac84", + "zh:7994db9ed3fdb6cbf21f2154ea962cf82f04e425988bedc6657de2497d6cb6c3", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9ac734c6aae19ea0832269f1f1dc3c3e100855c09b6168eecd8693958515b228", - "zh:a0e3b75176c71e5f41a6552cef57ad67d11778cb59fd2eb792e69f662b67ae01", - "zh:b4d0e1dd9a80d1ac9a3f1beaefb58d4b944cb0a30bdc672b9037a69b4dc20ec3", - "zh:b8309337e966528991cc2dd61ff0247864e4c3c7bef7dcf225f066edb6171050", - "zh:c34ce5fcdd481214697505bceb616a7dfcda7702f1afefa7dc5459965b8e01ca", - "zh:cb3f662ad06bb86aa763a5644ba318bc3db0ed30b04eb810c478e81f4b012d55", - "zh:cf0f0a459e2b6b1e014af9f53f32b5f555a249b61a0dcaccd3e4fe89178d8aa5", - "zh:d4dfb653a0295bfbbebcb4bb022b490f2c24ca31287ebf77fe9dcaee2e5ea658", - "zh:e49ba31aa092a30ea8a1091f7b26dbe41c3998f924f5b0d82810f27fc2f63501", - "zh:ea4d7d5688caf2f9cda3a1721779b61a051db6b43ca7267ae2712bd1262cfcfd", + "zh:93aa863e536ba9376ccf9e614e9edc9b214a2ce8c4316d416a8e249b436f52d2", + "zh:ab5cb4baeda57559686a0ccf0e09158aa64624ee6ba0ef32b769f13b11a43068", + "zh:ae9388b62eede8fd9272407bf75f8241a965bd489d45ec9dd3f9fac696d500e1", + "zh:caa5befd16960e2f69c7ec483e228e5ff43ab0979c17f1b874c9ffaa1c7c0e43", + "zh:cddd3e1067defa06a4e4ad5cb3940c7943e29c42417de57236aa7d3e2aeaae13", + "zh:cdfa44d591d0805116159556947904d70f534c6816188c45cf3a7544d2722ac9", + "zh:d607e9f1f3e09f13404f219e1893e3b3c77aece4afb54e999f934c021f41f576", + "zh:d8a397aca95125c6a0c0c78d2ded5843b9204effa9f5cf7419f017b500ad9228", + "zh:f5499eaff0d221725ad209d27d87c5b46d5c554caad7dc42947c760377abe3b0", + "zh:fc5f5cf433abc83e5169fa222992ec521c0c802075970251e3dd2c1d50c4f5c1", ] } diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..faae5f1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,14 @@ +# CLAUDE.md + +Crow CI pipelines (`.crow/`) and local tooling (`local/`) that build CRAN binary packages (incl. Alpine/musl) and upload them to Backblaze B2. + +## Conventions + +- **PRs:** the remote is Forgejo on `codefloe.com`; use `fj -H codefloe.com` (not `gh`). +- **Storage:** Backblaze B2 bucket `devxy-r-builds` (endpoints configured in the `.crow/` pipelines). + +## Gotchas + +- **B2 requires authentication for the list-bucket API.** Anonymous GET only works for individual public-read objects — an empty listing means missing credentials, not an empty bucket. +- Weekly-rebuild pipelines run for hours; watch them as background tasks and fetch Crow logs yourself instead of having the user paste progress. +- musl builds of packages with bundled native deps (e.g. RcppParallel/TBB) recur as failures; check for an existing patch before re-deriving a fix. diff --git a/README.md b/README.md index 278a475..28541a1 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,28 @@ For every package+tag combination: 1. Archive old package versions and keep the latest one in the root 1. Delete local binaries after successful upload +## Patching packages + +Some CRAN packages fail to compile on specific platforms due to compiler- or OS-specific issues unrelated to the package itself. +The canonical example is `RcppParallel`, whose bundled TBB sources fail on musl (Alpine) and newer compiler/OS combinations. +Because such packages are often transitive dependencies of many others, a single failure cascades: all dependents fail even though nothing is wrong with the dependent itself. + +To address this, frequently-failing packages can be "patched" before they are installed — whether as a direct build target or a transitive dependency pulled in by `pak`. + +The patch registry lives in `local/patches/registry.json`. +Each entry specifies a package and the platforms/versions it applies to, along with either lightweight build-time overrides (environment variables, configure arguments, Makevars) or a source diff (for deeper fixes). +See `local/patches/README.md` for the complete schema. + +Patching uses a two-tier approach: + +1. **Lightweight overrides:** environment variables, configure arguments, or Makevars settings applied during build — typically version-independent and fast. +2. **Source diffs:** unified diff patches applied to the unpacked source before building — more powerful but version-pinned. + +The system is implemented in `bincraft`: when a package needs patching, `bincraft` pre-builds it with the patch and serves the patched binary to `pak`, ensuring transitive dependents receive the fixed package. +This way, the fix cascades to all packages that depend on it. + +For the design rationale and architecture, see `specs/2026-06-30-package-patching-design.md`. + ## Build Environment Binaries are built on a mixed-architecture Kubernetes cluster using CI. @@ -199,7 +221,7 @@ Tag does not have a NAMESPACE file and hence cannot be built. Dependency not available: Either because the dependency was not declared or errored itself during installation. ```text - In function '\033[01m\033[KRcpp::List solveRRBLUP(const mat&, const mat&, const mat&)\033[m\033[K':\n\033[01m\033[KMME.cpp:162:61:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope\n 162 | double ll = -0.5*(double(optRes[\"objective\"])+df+df*log(2*\033[01;31m\033[KPI\033[m\033[K/df));\n | \033[01;31m\033[K^~\033[m\033[K\n\033[01m\033[KMME.cpp:\033[m\033[K In function '\033[01m\033[KRcpp::List solveRRBLUPMV(const mat&, const mat&, const mat&, int, double)\033[m\033[K':\n\033[01m\033[KMME.cpp:277:31:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope; did you mean '\033[01m\033[KHI\033[m\033[K'?\n 277 | ll -= double(n*m)/2.0*log(2*\033[01;31m\033[KPI\033[m\033[K);\n | \033[01;31m\033[K^~\033[m\033[K\n | \033[32m\033[KHI\033[m\033[K\nmake: *** [/opt/R/4.4.1/lib/R/etc/Makeconf:204: MME.o] Error 1\nERROR: compilation failed for package 'AlphaSimR'\n* removing '/tmp/RtmpclI5CE/temp_libpath11135d215d5/AlphaSimR'\n + In function '\033[01m\033[KRcpp::List solveRRBLUP(const mat&, const mat&, const mat&)\033[m\033[K':\n\033[01m\033[KMME.cpp:162:61:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope\n 162 | double ll = -0.5*(double(optRes[\"objective\"])+df+df*log(2*\033[01;31m\033[KPI\033[m\033[K/df));\n | \033[01;31m\033[K^~\033[m\033[K\n\033[01m\033[KMME.cpp:\033[m\033[K In function '\033[01m\033[KRcpp::List solveRRBLUPMV(const mat&, const mat&, const mat&, int, double)\033[m\033[K':\n\033[01m\033[KMME.cpp:277:31:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope; did you mean '\033[01m\033[KHI\033[m\033[K'?\n 277 | ll -= double(n*m)/2.0*log(2*\033[01;31m\033[KPI\033[m\033[K);\n | \033[01;31m\033[K^~\033[m\033[K\n | \033[32m\033[KHI\033[m\033[K\nmake: *** [/opt/R/4.4.2/lib/R/etc/Makeconf:204: MME.o] Error 1\nERROR: compilation failed for package 'AlphaSimR'\n* removing '/tmp/RtmpclI5CE/temp_libpath11135d215d5/AlphaSimR'\n ``` Compiler error: Possible reasons: too old CXX code which cannot be compiled anymore with CXX14 or CXX17. diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 2cd4d51..5bc107a 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -16,6 +16,11 @@ ARG CACHEBUST WORKDIR /work COPY build-one.R /work/build-one.R +# Resolve and install the latest bincraft release dynamically (no hardcoded pin). +COPY install-bincraft.R /work/install-bincraft.R +# Ship the patch registry so build-one.R's `patches = "local/patches"` resolves +# (build context is `local/`, CWD is /work). +COPY patches /work/local/patches RUN --mount=type=secret,id=b2_access,required=true \ --mount=type=secret,id=b2_secret,required=true \ @@ -27,6 +32,9 @@ RUN --mount=type=secret,id=b2_access,required=true \ export GITHUB_PAT="$(cat /run/secrets/github_pat 2>/dev/null || true)" && \ export GIT_TERMINAL_PROMPT=0 && \ export OTEL_SDK_DISABLED=true && \ + export OTEL_R_TRACES_EXPORTER=none && \ + export OTEL_R_LOGS_EXPORTER=none && \ + export OTEL_R_METRICS_EXPORTER=none && \ XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run || true); \ XVFB_ARGS=""; \ if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi; \ @@ -37,7 +45,7 @@ RUN --mount=type=secret,id=b2_access,required=true \ echo "No working virtual display; building without xvfb" >&2; \ fi; \ run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \ - ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")'; }; \ + ensure_bincraft() { "$1" -q -e 'source("/work/install-bincraft.R")'; }; \ PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ seen=" $PRIMARY_MINOR "; \ prc=0; failed=""; \ diff --git a/docker/buildkitd.toml b/docker/buildkitd.toml new file mode 100644 index 0000000..ebbeb38 --- /dev/null +++ b/docker/buildkitd.toml @@ -0,0 +1,41 @@ +# BuildKit GC config for the remote buildx builders (artemis/amd64, gaia/arm64). +# +# Apply when creating the docker-container builders: +# docker buildx create --name artemis --driver docker-container \ +# --config docker/buildkitd.toml ssh:// +# docker buildx create --name gaia --driver docker-container \ +# --config docker/buildkitd.toml ssh:// +# +# Why: BuildKit's default GC caps the ephemeral cache tier — RUN +# --mount=type=cache mounts, local build context, git checkouts — at a +# hardcoded 512 MB (shown as "488.3 MiB" in `buildx inspect`). Across our +# 7-distro build matrix that fills instantly and forces re-downloads of +# system + R packages every rebuild. The first rule below raises that tier. +# +# Limits are absolute (not %) on purpose: artemis and gaia have very +# different free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB), so a +# percentage would mean wildly different real budgets. minFreeSpace = 20 GB +# keeps the tight Hetzner host safe while staying modest on the Mac mini. + +[worker.oci] + gc = true + + # Tier 1 — ephemeral caches (cache mounts, local context, git checkouts). + # Raised from the 512 MB default to 8 GB, retained for 7 days so weekly + # rebuilds reuse downloaded packages instead of re-fetching them. + [[worker.oci.gcpolicy]] + filters = [ + "type==source.local", + "type==exec.cachemount", + "type==source.git.checkout", + ] + keepDuration = "168h" + maxUsedSpace = "8GB" + + # Tier 2 — everything else (image layers, RUN exec results). Bounds the + # whole buildkit cache and always leaves 20 GB free on the host disk. + [[worker.oci.gcpolicy]] + all = true + reservedSpace = "2GB" + maxUsedSpace = "40GB" + minFreeSpace = "20GB" diff --git a/justfile b/justfile index 2d9388d..a9a705e 100644 --- a/justfile +++ b/justfile @@ -10,8 +10,10 @@ # - buildx builders named `artemis` (amd64) and `gaia` (arm64), created with the # docker-container driver (runs BuildKit on the remote host's docker daemon over # SSH). The default `remote` driver does NOT work with an ssh:// docker host. -# docker buildx create --name artemis --driver docker-container ssh:// -# docker buildx create --name gaia --driver docker-container ssh:// +# Pass --config docker/buildkitd.toml so BuildKit's GC keeps a usable cache +# (the default caps the cache-mount tier at 512 MB, forcing re-downloads). +# docker buildx create --name artemis --driver docker-container --config docker/buildkitd.toml ssh:// +# docker buildx create --name gaia --driver docker-container --config docker/buildkitd.toml ssh:// # - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional) # # Overridable (env or `just VAR=… rebuild …`): diff --git a/local/build-all.R b/local/build-all.R index 56f1fbc..1df1f83 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -64,12 +64,38 @@ sprintf("# of package versions for this job: %s", nrow(chunk)) exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] chunk <- chunk[!chunk$Package %in% exclude, ] -# Skip package versions already built in a previous run. +# Skip package versions already attempted in a previous run (built or errored). # pkgs_to_build.rds is a static snapshot from the install-deps step, so on a # restart it still lists everything an interrupted run already produced. The # metadata DB reflects that progress, so we re-derive the remaining set here. -platform <- paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-") -arch <- Sys.getenv("ARCH") +# We exclude *all* attempted versions, not just successful ones: a previously +# errored version is skipped by build_binary_package() anyway, so leaving it in +# the chunk only makes the job cycle through it one-by-one for no benefit. +# Derive platform + arch from the running container, mirroring the codename -> +# platform mapping bincraft uses internally. The OS/OS_VERSION selectors are +# workflow-level CI variables that are not injected into the container +# environment, so Sys.getenv() would return "" and this pre-filter would query +# platform "-" and skip nothing. +codename <- bincraft::set_codename(NULL) +platform <- switch( + codename, + jammy = "ubuntu-2204", + noble = "ubuntu-2404", + resolute = "ubuntu-2604", + rhel10 = "redhat-10", + rhel9 = "redhat-9", + rhel8 = "redhat-8", + alpine320 = "alpine-320", + alpine321 = "alpine-321", + alpine322 = "alpine-322", + alpine323 = "alpine-323", + alpine324 = "alpine-324", + alpine325 = "alpine-325", + alpine326 = "alpine-326", + NA_character_ +) +local_machine <- Sys.info()[["machine"]] +arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" con <- DBI::dbConnect( RPostgres::Postgres(), dbname = "build_metadata", @@ -81,13 +107,13 @@ con <- DBI::dbConnect( ) built <- DBI::dbGetQuery( con, - "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2 AND error_occurred = FALSE", + "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2", params = list(platform, arch) ) DBI::dbDisconnect(con) before <- nrow(chunk) chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] -sprintf("Skipped %d already-built package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) +sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, @@ -121,6 +147,7 @@ mapply( metadata_db_sslmode = "require", metadata_db_port = 15432, archive = TRUE, + patches = "local/patches", upload = TRUE, store_build_metadata = TRUE ) diff --git a/local/build-one.R b/local/build-one.R index f2d6ebd..07c0b6a 100644 --- a/local/build-one.R +++ b/local/build-one.R @@ -103,6 +103,7 @@ for (ver in versions) { force = TRUE, upload = TRUE, archive = TRUE, + patches = "local/patches", store_build_metadata = TRUE, s3_endpoint = s3$s3_endpoint, s3_region = s3$s3_region, diff --git a/local/install-bincraft.R b/local/install-bincraft.R new file mode 100644 index 0000000..4665b3b --- /dev/null +++ b/local/install-bincraft.R @@ -0,0 +1,52 @@ +#!/usr/bin/env Rscript + +# Install the latest tagged bincraft release, resolved dynamically, so the CI +# workflows and the build-one image never pin a hardcoded version (no more +# editing `@vX.Y.Z` in many places on every release). +# +# Run with the R whose library should receive bincraft: +# Rscript local/install-bincraft.R +# or, to target a specific R from a shell loop: +# "$RBIN" -q -e 'source("local/install-bincraft.R")' +# +# How it works: list the remote tags with `git ls-remote` (no token needed for +# the public repo), keep the `vX.Y.Z` release tags, pick the highest version, +# and install it with pak. pak is idempotent on the git ref, so re-running keeps +# the package when it is already current and only updates when a newer tag ships. +# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is +# identical across git versions and `system2()` argument handling. + +repo_url <- Sys.getenv( + "BINCRAFT_GIT_URL", + unset = "https://codefloe.com/rpkgs/bincraft.git" +) + +# GIT_TERMINAL_PROMPT=0 keeps a non-interactive run from hanging on auth. +refs <- system2( + "git", + c("ls-remote", "--tags", repo_url), + stdout = TRUE, + stderr = FALSE, + env = "GIT_TERMINAL_PROMPT=0" +) +tags <- sub(".*refs/tags/", "", refs) +tags <- tags[!grepl("\\^\\{\\}$", tags)] # drop dereferenced "...^{}" lines +tags <- grep("^v[0-9]", tags, value = TRUE) # only vX.Y.Z release tags +if (length(tags) == 0L) { + stop( + "Could not resolve any bincraft release tag from ", + repo_url, + call. = FALSE + ) +} +latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][ + 1L +] + +message(sprintf("Installing latest bincraft release: %s", latest)) +pak::pak(sprintf("git::%s@%s", repo_url, latest)) +message(sprintf( + "bincraft %s installed (%s)", + as.character(utils::packageVersion("bincraft")), + latest +)) diff --git a/local/patches/README.md b/local/patches/README.md new file mode 100644 index 0000000..af24b2b --- /dev/null +++ b/local/patches/README.md @@ -0,0 +1,43 @@ +# Patch Registry + +This directory contains the curated registry of per-package build-time patches consumed by bincraft's `patches` argument. + +## Schema + +The registry is defined in `registry.json` as an array of patch entries. Each entry specifies lightweight build-time overrides (environment variables, configure arguments, Makevars) and optionally a source diff to apply before building. + +### Field semantics + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `package` | string | yes | CRAN package name. | +| `versions` | string | yes | `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. | +| `platforms` | array of strings | yes | Matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). An entry matches if any listed token matches any build token. `["*"]` matches all platforms. | +| `env` | object | no | Environment variables exported only for this package's isolated build. | +| `configure_args` | array | no | Arguments passed as `--configure-args` to the isolated build. | +| `makevars` | object | no | Key/value pairs written into a package-local Makevars for the isolated build. | +| `patch` | string or null | no | Path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. | +| `reason` | string | yes | Human explanation, surfaced in logs and metadata. | + +## Adding an entry + +To add a new patch entry: + +1. Add an object to the array in `registry.json` with the fields documented above. + Start with lightweight overrides (environment variables, configure arguments, Makevars) before resorting to source diffs. + +2. If a source diff is needed, place it in `local/patches//.patch` and reference its path in the `patch` field. + For example, a diff for `RcppParallel` would go in `local/patches/RcppParallel/fix.patch` and be referenced as `"patch": "RcppParallel/fix.patch"`. + +3. The `reason` field should clearly explain why the patch is needed and what problem it solves. + +## Validation + +The registry is validated and applied by bincraft during the build process. +For manual validation, run the validator from the repo root: + +```bash +Rscript local/validate-patches.R +``` + +This validates the schema, referenced patch-file existence, and checks for duplicate entries across platforms and versions. diff --git a/local/patches/RcppParallel/disable-tbb.patch b/local/patches/RcppParallel/disable-tbb.patch new file mode 100644 index 0000000..0fe8ad9 --- /dev/null +++ b/local/patches/RcppParallel/disable-tbb.patch @@ -0,0 +1,16 @@ +diff --git a/src/Makevars.in b/src/Makevars.in +index be8445f..faee771 100644 +--- a/src/Makevars.in ++++ b/src/Makevars.in +@@ -60,7 +60,10 @@ else + endif + + ifeq ($(UNAME), Linux) +- USE_TBB=Linux ++ # bincraft patch: the bundled Intel TBB build hangs/fails on musl (Alpine) ++ # and newer toolchains (g++ 15). Skip it (leave USE_TBB unset) and force the ++ # TinyThread backend so RcppParallel still builds. ++ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=0 + endif + + ifeq ($(UNAME), SunOS) diff --git a/local/patches/fs/force-vendored-libuv.patch b/local/patches/fs/force-vendored-libuv.patch new file mode 100644 index 0000000..c69e8b9 --- /dev/null +++ b/local/patches/fs/force-vendored-libuv.patch @@ -0,0 +1,19 @@ +diff --git a/configure b/configure +--- a/configure ++++ b/configure +@@ -11,6 +11,15 @@ + PKG_TEST_HEADER="" + PKG_LIBS="-luv" + ++# bincraft patch: force the vendored static libuv so the resulting binary ++# is self-contained. fs configure otherwise links system libuv whenever ++# pkg-config finds libuv-devel (installed as a build-time sysreq), yielding ++# an fs.so with NEEDED libuv.so.1 that fails to dyn.load on machines lacking ++# runtime libuv (install.packages/renv do not install SystemRequirements). ++echo "Building static libuv (bincraft: forced vendored)" 1>&2 ++cp -f src/Makevars.vendor src/Makevars ++exit 0 ++ + # Use pkg-config if available + if [ `command -v pkg-config` ]; then + PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` diff --git a/local/patches/registry.json b/local/patches/registry.json new file mode 100644 index 0000000..7a197d7 --- /dev/null +++ b/local/patches/registry.json @@ -0,0 +1,22 @@ +[ + { + "package": "RcppParallel", + "versions": "*", + "platforms": ["alpine", "ubuntu-2604"], + "env": {}, + "configure_args": [], + "makevars": {}, + "patch": "RcppParallel/disable-tbb.patch", + "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); patch unsets USE_TBB and forces -DRCPP_PARALLEL_USE_TBB=0 so RcppParallel skips the bundled build and uses the TinyThread backend" + }, + { + "package": "fs", + "versions": "*", + "platforms": ["*"], + "env": {}, + "configure_args": [], + "makevars": {}, + "patch": "fs/force-vendored-libuv.patch", + "reason": "fs 2.x configure links system libuv whenever pkg-config finds libuv-devel (installed as a build-time sysreq), producing an fs.so with NEEDED libuv.so.1. That binary fails to dyn.load on consumer machines lacking runtime libuv, because install.packages/renv do not install SystemRequirements (only pak does, and only in the build container). The patch short-circuits configure to copy src/Makevars.vendor and build the bundled static libuv (needs cmake) so the binary is self-contained on every platform. An env/pkg-config override was tried first but the rebuilt binary still linked libuv.so.1, so a source patch is used instead." + } +] diff --git a/local/validate-patches.R b/local/validate-patches.R new file mode 100644 index 0000000..5f5aace --- /dev/null +++ b/local/validate-patches.R @@ -0,0 +1,56 @@ +#!/usr/bin/env Rscript +# Validate local/patches/registry.json: schema, referenced patch files, and +# ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. + +dir <- "local/patches" +registry_file <- file.path(dir, "registry.json") +if (!file.exists(registry_file)) { + cat("No registry.json found; nothing to validate.\n") + quit(status = 0L) +} + +or_q <- function(x) if (is.null(x)) "?" else x + +reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +required <- c("package", "versions", "platforms", "reason") +errs <- character(0L) + +for (i in seq_along(reg)) { + e <- reg[[i]] + missing <- setdiff(required, names(e)) + if (length(missing) > 0L) { + errs <- c(errs, sprintf( + "entry %d (%s): missing %s", i, + if (is.null(e$package)) "?" else e$package, toString(missing) + )) + } + if (!is.null(e$patch)) { + p <- file.path(dir, e$patch) + if (!file.exists(p)) { + errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing", + i, e$package, p)) + } + } +} + +# Ambiguous overlap: two entries for the same package with identical platforms +# and versions. +keys <- vapply(reg, function(e) { + sprintf( + "%s|%s|%s", + or_q(e$package), + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + or_q(e$versions) + ) +}, character(1L)) +dups <- keys[duplicated(keys)] +if (length(dups) > 0L) { + errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups)))) +} + +if (length(errs) > 0L) { + cat("Patch registry validation FAILED:\n") + cat(paste0(" - ", errs, "\n")) + quit(status = 1L) +} +cat(sprintf("Patch registry OK (%d %s).\n", length(reg), if (length(reg) == 1L) "entry" else "entries")) diff --git a/plans/2026-06-30-package-patching-implementation.md b/plans/2026-06-30-package-patching-implementation.md new file mode 100644 index 0000000..3a743d7 --- /dev/null +++ b/plans/2026-06-30-package-patching-implementation.md @@ -0,0 +1,1477 @@ +# Package Patching Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let a curated registry of packages be "patched" (env/configure/Makevars overrides and/or source diffs) before `pak` installs them, including when they are transitive dependencies, so compiler-/OS-specific failures like `RcppParallel` stop cascading. + +**Architecture:** The mechanism lives in `bincraft`. Before `pak::local_install_deps()` runs, bincraft pre-builds each registry-matched package as a binary (reusing `pkgbuild::build(binary=TRUE)` + `cranlike::add_PACKAGES`), caches it, and serves it from a local `file://` repo prepended to `options("repos")` so `pak` installs the patched binary — direct or transitive — without recompiling. The curated registry (`registry.json` + diff files) lives in this repo and is passed in via a new `patches` argument. + +**Tech Stack:** R, `pak`, `pkgbuild`, `cranlike` (fork `pat-s/cranlike@s3`), `pkgsearch`, `withr`, `jsonlite`, `testthat` (3e), `mockery`. Two repos: `bincraft` at `/Users/pjs/git/codefloe.com/rpkgs/bincraftr`, and this repo `build-cran-binaries`. + +## Global Constraints + +- bincraft package name is `bincraft`; repo dir is `…/rpkgs/bincraftr`. Current version `4.2.2.9999`. +- Logging uses bincraft's wrappers only: `log_info`, `log_warn`, `log_success`, `log_debug`, `log_error`, `log_header` (never bare `message`/`cat` in package code). +- `cli`-style inline markup is allowed in log messages (e.g. `{.pkg %s}`, `{.path %s}`); curly braces in dynamic/error text must be escaped as already done in `install_helpers.R`. +- Tests: `testthat` 3rd edition, files at `tests/testthat/test-.R`, network/build tests guarded with `skip_on_cran()` / `skip_if_offline()`; end-to-end build tests guarded behind `skip_if_not(nzchar(Sys.getenv("BINCRAFT_PATCH_E2E")))`. +- New exported functions need roxygen with `@keywords internal` for non-user helpers; run `devtools::document()` after adding roxygen. +- Patched-binary cache dir default: `/mnt/cache/patched-binaries`. +- Platform tokens for matching = `c(, , )`, e.g. `ubuntu-2604` → `c("ubuntu-2604","ubuntu","amd64")`. +- Registry entry required fields: `package`, `versions`, `platforms`, `reason`. Optional: `env`, `configure_args`, `makevars`, `patch`. +- One sentence per line in prose/commit messages; do not hard-wrap at 80 columns. +- Use `fj -H codefloe.com` for any PR operations (Forgejo), not `gh`. + +--- + +## Phase A — bincraft mechanism + +All Phase A paths are relative to `/Users/pjs/git/codefloe.com/rpkgs/bincraftr`. + +### Task A0: Proof of mechanism — pak installs a patched binary from a prepended `file://` repo + +This de-risks the core assumption before building anything on top: that `pak` installs a binary from a local `file://` repo in preference to CRAN for an equal version, and does so without recompiling. If this fails, the contingency (documented in Step 4) is to serve patched *source* and rely on `pkgcache` build-caching — the rest of the plan changes only inside `build_patched_binary()`. + +**Files:** +- Create: `tools/verify-patch-mechanism.R` + +**Interfaces:** +- Produces: a runnable script proving `pak::pkg_install()` resolves a local patched binary over CRAN. No package API. + +- [ ] **Step 1: Write the verification script** + +```r +# tools/verify-patch-mechanism.R +# Proves pak installs a patched binary from a prepended file:// repo instead of +# CRAN's, without recompiling. Run inside a Linux build-env container: +# Rscript tools/verify-patch-mechanism.R +# Exits 0 on success, 1 on failure. + +pkg <- "glue" # small, pure-R CRAN package +sentinel <- "PatchMechanismProof" + +work <- tempfile("verify_") +repo <- file.path(work, "repo", "src", "contrib") +lib <- file.path(work, "lib") +dir.create(repo, recursive = TRUE) +dir.create(lib, recursive = TRUE) + +# 1. Download CRAN source for the current version. +ap <- available.packages(repos = "https://cloud.r-project.org") +ver <- ap[pkg, "Version"] +src <- file.path(work, sprintf("%s_%s.tar.gz", pkg, ver)) +download.file( + sprintf("https://cloud.r-project.org/src/contrib/%s_%s.tar.gz", pkg, ver), + src, mode = "wb" +) + +# 2. Unpack, inject a sentinel field into DESCRIPTION, build a binary. +untar(src, exdir = work) +desc <- file.path(work, pkg, "DESCRIPTION") +writeLines(c(readLines(desc), sprintf("%s: yes", sentinel)), desc) +pkgbuild::build( + file.path(work, pkg), binary = TRUE, vignettes = FALSE, + dest_path = repo, quiet = TRUE +) +built <- list.files(repo, pattern = sprintf("^%s_.*\\.tar\\.gz$", pkg), full.names = TRUE) +file.rename(built[1L], file.path(repo, sprintf("%s_%s.tar.gz", pkg, ver))) +cranlike::add_PACKAGES(sprintf("%s_%s.tar.gz", pkg, ver), repo) + +# 3. Install with the local repo prepended; assert our patched build won. +withr::with_options( + list(repos = c(patched = sprintf("file://%s", dirname(dirname(repo))), + CRAN = "https://cloud.r-project.org")), + pak::pkg_install(pkg, lib = lib, ask = FALSE, upgrade = FALSE) +) + +installed_desc <- file.path(lib, pkg, "DESCRIPTION") +ok <- file.exists(installed_desc) && + any(grepl(sentinel, readLines(installed_desc))) + +if (ok) { + cat("PROOF PASSED: pak installed the patched local binary.\n") + quit(status = 0L) +} else { + cat("PROOF FAILED: pak did not install the patched local binary.\n") + quit(status = 1L) +} +``` + +- [ ] **Step 2: Run the proof in a build-env container** + +Run (amd64 example; use any supported build-env image): + +```bash +just build-single ubuntu 2604 amd64 4.5.0 glue 1.0.0 1 || true # warms the env +docker run --rm -v "$PWD":/work -w /work reg.devxy.io/rpkgs/build-env-ubuntu:2604 \ + Rscript tools/verify-patch-mechanism.R +``` + +Expected: final line `PROOF PASSED: pak installed the patched local binary.` and exit status 0. + +- [ ] **Step 3: Commit** + +```bash +git add tools/verify-patch-mechanism.R +git commit -m "test(patches): prove pak installs a patched binary from a local file:// repo" +``` + +- [ ] **Step 4: Record the outcome / contingency** + +If the proof PASSED, proceed to Task A1 unchanged. +If it FAILED (pak recompiled or picked CRAN's), the mechanism switches to serving patched *source*: in Task A4 `build_patched_binary()` skips `pkgbuild::build()` and instead repackages the patched source tree with `pkgbuild::build(binary = FALSE)`; everything else (registry, matching, cache, repo prepend) is unchanged because `pak` build-caches the compiled result via `pkgcache`. Note the chosen path in the commit message and continue. + +--- + +### Task A1: Registry loading and normalization + +**Files:** +- Create: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Produces: `load_patch_registry(patches_dir)` → `list()` of normalized entries; each entry is a named list with `package`, `versions`, `platforms` (character vector), `env` (named list), `configure_args` (character), `makevars` (named list), `reason`, and `patch_path` (absolute path or `NULL`). `normalize_patch_entry(entry, patches_dir)` → one normalized entry; errors on missing required field or missing patch file. + +- [ ] **Step 1: Write the failing test** + +```r +# tests/testthat/test-patches.R +test_that("load_patch_registry parses and normalizes entries", { + dir <- withr::local_tempdir() + writeLines("--- a patch ---", file.path(dir, "fix.patch")) + jsonlite::write_json( + list(list( + package = "RcppParallel", versions = "*", + platforms = list("alpine", "ubuntu-2604"), + env = list(RCPP_PARALLEL_USE_TBB = "0"), + patch = "fix.patch", reason = "bundled TBB fails" + )), + file.path(dir, "registry.json"), auto_unbox = TRUE + ) + + reg <- load_patch_registry(dir) + + expect_length(reg, 1L) + expect_identical(reg[[1L]]$package, "RcppParallel") + expect_identical(reg[[1L]]$platforms, c("alpine", "ubuntu-2604")) + expect_identical(reg[[1L]]$env$RCPP_PARALLEL_USE_TBB, "0") + expect_identical(reg[[1L]]$configure_args, character(0L)) + expect_true(file.exists(reg[[1L]]$patch_path)) +}) + +test_that("load_patch_registry returns empty list when no registry", { + expect_identical(load_patch_registry(NULL), list()) + expect_identical(load_patch_registry(withr::local_tempdir()), list()) +}) + +test_that("normalize_patch_entry errors on missing required field", { + expect_error( + normalize_patch_entry(list(package = "x"), tempdir()), + "missing required field" + ) +}) + +test_that("normalize_patch_entry errors on missing patch file", { + expect_error( + normalize_patch_entry( + list(package = "x", versions = "*", platforms = "alpine", + reason = "r", patch = "nope.patch"), + tempdir() + ), + "does not exist" + ) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function load_patch_registry". + +- [ ] **Step 3: Write minimal implementation** + +```r +# R/patches.R + +#' Load and validate the patch registry +#' +#' Reads `registry.json` from `patches_dir` and returns normalized entries. +#' +#' @param patches_dir Directory containing `registry.json` and any patch files, +#' or `NULL` to disable patching. +#' @return A list of normalized patch entries (possibly empty). +#' @keywords internal +load_patch_registry <- function(patches_dir) { + if (is.null(patches_dir)) { + return(list()) + } + registry_file <- file.path(patches_dir, "registry.json") + if (!file.exists(registry_file)) { + log_warn(sprintf( + "Patch directory {.path %s} has no registry.json; patching disabled.", + patches_dir + )) + return(list()) + } + raw <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) + lapply(raw, normalize_patch_entry, patches_dir = patches_dir) +} + +#' Normalize and validate a single patch registry entry +#' +#' @param entry A list parsed from `registry.json`. +#' @param patches_dir Directory used to resolve a relative `patch` path. +#' @return The entry with defaults filled and `patch_path` resolved. +#' @keywords internal +normalize_patch_entry <- function(entry, patches_dir) { + required <- c("package", "versions", "platforms", "reason") + missing <- setdiff(required, names(entry)) + if (length(missing) > 0L) { + stop( + sprintf("Patch entry is missing required field(s): %s", toString(missing)), + call. = FALSE + ) + } + entry$platforms <- as.character(unlist(entry$platforms)) + entry$env <- if (is.null(entry$env)) list() else entry$env + entry$configure_args <- if (is.null(entry$configure_args)) { + character(0L) + } else { + as.character(unlist(entry$configure_args)) + } + entry$makevars <- if (is.null(entry$makevars)) list() else entry$makevars + if (!is.null(entry$patch)) { + patch_path <- file.path(patches_dir, entry$patch) + if (!file.exists(patch_path)) { + stop( + sprintf( + "Patch file '%s' for package '%s' does not exist.", + patch_path, entry$package + ), + call. = FALSE + ) + } + entry$patch_path <- patch_path + } else { + entry$patch_path <- NULL + } + entry +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): load and validate the patch registry" +``` + +--- + +### Task A2: Platform matching and version-constraint satisfaction + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: normalized entries from Task A1. +- Produces: `build_platform_tokens(platform, arch)` → character vector; `entry_matches_platform(entry, tokens)` → logical; `match_patch_entries(registry, platform, arch)` → filtered list; `version_satisfies(version, constraint)` → logical (constraint forms: `"*"` handled by caller, `"x.y.z"` exact, `">=x"`, `"<=x"`, `">x"`, `"=5.1.0")) + expect_false(version_satisfies("5.0.0", ">=5.1.0")) + expect_true(version_satisfies("5.1.11-2", "<=5.1.11-2")) + expect_false(version_satisfies("5.1.12", "<=5.1.11-2")) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function build_platform_tokens". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Build platform tokens for patch matching +#' @keywords internal +build_platform_tokens <- function(platform, arch) { + family <- sub("-.*$", "", platform) + unique(c(platform, family, arch)) +} + +#' Does a patch entry apply to the current platform tokens? +#' @keywords internal +entry_matches_platform <- function(entry, tokens) { + any(entry$platforms == "*") || + length(intersect(entry$platforms, tokens)) > 0L +} + +#' Filter registry entries applicable to the current build +#' @keywords internal +match_patch_entries <- function(registry, platform, arch) { + if (length(registry) == 0L) { + return(list()) + } + tokens <- build_platform_tokens(platform, arch) + Filter(function(e) entry_matches_platform(e, tokens), registry) +} + +#' Test whether a version satisfies a single constraint +#' +#' @param version A version string (CRAN style, may contain `-`). +#' @param constraint One of `"x.y.z"`, `"==x"`, `">=x"`, `"<=x"`, `">x"`, `"=|<=|==|>|<)?\\s*(.+)$", constraint) + )[[1L]] + op <- parts[2L] + target <- parts[3L] + v <- package_version(version) + t <- package_version(target) + if (op == "" || op == "==") { + return(v == t) + } + switch( + op, + ">=" = v >= t, + "<=" = v <= t, + ">" = v > t, + "<" = v < t, + FALSE + ) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): platform matching and version-constraint checks" +``` + +--- + +### Task A3: Cache key and version resolution + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: normalized entries. +- Produces: `patch_cache_key(entry, version, platform, arch, r_minor)` → string `"_____"`, where `hash12` covers `env`/`configure_args`/`makevars`/patch bytes; `resolve_patch_version(entry)` → latest CRAN version satisfying `entry$versions`, or `NA_character_`; `describe_patch(entry)` → short human label. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("patch_cache_key is stable and sensitive to env/patch changes", { + e1 <- list(package = "P", env = list(A = "1"), + configure_args = character(0L), makevars = list(), + patch_path = NULL) + e2 <- e1; e2$env <- list(A = "2") + + k1 <- patch_cache_key(e1, "1.0", "alpine-324", "amd64", "4.5") + expect_identical(k1, patch_cache_key(e1, "1.0", "alpine-324", "amd64", "4.5")) + expect_false(identical( + k1, patch_cache_key(e2, "1.0", "alpine-324", "amd64", "4.5") + )) + expect_match(k1, "^P_1.0_alpine-324_amd64_4.5_[0-9a-f]{12}$") +}) + +test_that("resolve_patch_version returns latest for wildcard, NA when unmet", { + local_mocked_bindings( + cran_package = function(pkg) list(Version = "5.1.12"), + .package = "pkgsearch" + ) + expect_identical( + resolve_patch_version(list(package = "RcppParallel", versions = "*")), + "5.1.12" + ) + expect_identical( + resolve_patch_version(list(package = "RcppParallel", versions = ">=9.0")), + NA_character_ + ) +}) + +test_that("describe_patch summarizes the active overrides", { + expect_match( + describe_patch(list(env = list(RCPP_PARALLEL_USE_TBB = "0"), + configure_args = character(0L), makevars = list(), + patch_path = NULL)), + "env: RCPP_PARALLEL_USE_TBB=0" + ) + expect_match( + describe_patch(list(env = list(), configure_args = character(0L), + makevars = list(), patch_path = "/x/fix.patch")), + "source patch" + ) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function patch_cache_key". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Compute the cache key for a patched binary +#' @keywords internal +patch_cache_key <- function(entry, version, platform, arch, r_minor) { + payload <- list( + env = entry$env, + configure_args = entry$configure_args, + makevars = entry$makevars, + patch = if (!is.null(entry$patch_path)) { + readBin(entry$patch_path, "raw", file.size(entry$patch_path)) + } else { + raw(0L) + } + ) + tmp <- tempfile() + on.exit(unlink(tmp), add = TRUE) + saveRDS(payload, tmp) + hash <- substr(unname(tools::md5sum(tmp)), 1L, 12L) + sprintf( + "%s_%s_%s_%s_%s_%s", + entry$package, version, platform, arch, r_minor, hash + ) +} + +#' Resolve the CRAN version to build for a patch entry +#' +#' Returns the latest CRAN version satisfying the entry's `versions` constraint, +#' or `NA_character_` when CRAN's latest does not satisfy it or lookup fails. +#' @keywords internal +resolve_patch_version <- function(entry) { + latest <- tryCatch( + pkgsearch::cran_package(entry$package)$Version, + error = function(e) NA_character_ + ) + if (is.na(latest)) { + return(NA_character_) + } + if (identical(entry$versions, "*") || version_satisfies(latest, entry$versions)) { + return(latest) + } + NA_character_ +} + +#' Short human label describing a patch entry's overrides +#' @keywords internal +describe_patch <- function(entry) { + bits <- character(0L) + if (length(entry$env) > 0L) { + bits <- c(bits, sprintf( + "env: %s", + paste( + names(entry$env), + unlist(entry$env), + sep = "=", collapse = "," + ) + )) + } + if (length(entry$configure_args) > 0L) { + bits <- c(bits, sprintf("configure: %s", toString(entry$configure_args))) + } + if (length(entry$makevars) > 0L) { + bits <- c(bits, "makevars") + } + if (!is.null(entry$patch_path)) { + bits <- c(bits, "source patch") + } + if (length(bits) == 0L) "no-op" else paste(bits, collapse = "; ") +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. (Note: `local_mocked_bindings` requires testthat >= 3.1.7; bincraft uses 3e.) + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): cache key, version resolution, and patch description" +``` + +--- + +### Task A4: Build a patched binary in isolation + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: a normalized entry, a resolved `version`, a `dest_dir`. +- Produces: `download_cran_source(package, version, dest_dir, cran)` → path or `NULL`; `apply_source_patch(patch_path, pkg_src)` → logical; `configure_args_to_build_args(configure_args)` → character; `build_patched_binary(entry, version, dest_dir)` → path to built binary tarball or `NULL`. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("configure_args_to_build_args formats configure args", { + expect_identical(configure_args_to_build_args(character(0L)), character(0L)) + expect_identical( + configure_args_to_build_args(c("--with-foo", "--no-bar")), + "--configure-args=--with-foo --no-bar" + ) +}) + +test_that("apply_source_patch returns FALSE when patch does not apply", { + src <- withr::local_tempdir() + writeLines("unrelated content", file.path(src, "file.txt")) + bad_patch <- tempfile(fileext = ".patch") + writeLines(c( + "--- a/missing.txt", "+++ b/missing.txt", + "@@ -1 +1 @@", "-nope", "+nope2" + ), bad_patch) + expect_false(apply_source_patch(bad_patch, src)) +}) + +test_that("build_patched_binary returns NULL when download fails", { + local_mocked_bindings(download_cran_source = function(...) NULL) + expect_null( + build_patched_binary( + list(package = "P", env = list(), configure_args = character(0L), + makevars = list(), patch_path = NULL), + "1.0", withr::local_tempdir() + ) + ) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function configure_args_to_build_args". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Download a CRAN source tarball for an exact version +#' @keywords internal +download_cran_source <- function( + package, + version, + dest_dir, + cran = "https://cloud.r-project.org" +) { + fname <- sprintf("%s_%s.tar.gz", package, version) + urls <- c( + sprintf("%s/src/contrib/%s", cran, fname), + sprintf("%s/src/contrib/Archive/%s/%s", cran, package, fname) + ) + dest <- file.path(dest_dir, fname) + for (u in urls) { + ok <- tryCatch( + { + utils::download.file(u, dest, mode = "wb", quiet = TRUE) + file.exists(dest) && file.size(dest) > 0L + }, + error = function(e) FALSE + ) + if (isTRUE(ok)) { + return(dest) + } + } + log_warn(sprintf( + "Could not download CRAN source for {.pkg %s} %s.", + package, version + )) + NULL +} + +#' Apply a unified diff to an unpacked source tree +#' +#' Uses `patch -p1 --forward` so an already-applied or non-applying patch fails +#' cleanly (returns FALSE) instead of corrupting the tree. +#' @keywords internal +apply_source_patch <- function(patch_path, pkg_src) { + status <- system2( + "patch", + args = c( + "-p1", "--forward", "--batch", + "-d", shQuote(pkg_src), + "-i", shQuote(patch_path) + ), + stdout = FALSE, stderr = FALSE + ) + identical(status, 0L) +} + +#' Format configure args for `pkgbuild::build(args = ...)` +#' @keywords internal +configure_args_to_build_args <- function(configure_args) { + if (length(configure_args) == 0L) { + return(character(0L)) + } + sprintf("--configure-args=%s", paste(configure_args, collapse = " ")) +} + +#' Build a patched binary for one registry entry, in isolation +#' +#' Downloads CRAN source for `version`, applies the source patch (if any), and +#' builds a binary with the entry's env / configure / Makevars overrides scoped +#' to this build only. Returns the built tarball path, or `NULL` on any failure. +#' @keywords internal +build_patched_binary <- function(entry, version, dest_dir) { + workdir <- tempfile("patch_build_") + dir.create(workdir, recursive = TRUE, showWarnings = FALSE) + on.exit(unlink(workdir, recursive = TRUE, force = TRUE), add = TRUE) + + src_tarball <- download_cran_source(entry$package, version, workdir) + if (is.null(src_tarball)) { + return(NULL) + } + + utils::untar(src_tarball, exdir = workdir) + pkg_src <- file.path(workdir, entry$package) + + if (!is.null(entry$patch_path)) { + if (!apply_source_patch(entry$patch_path, pkg_src)) { + log_warn(sprintf( + "Patch for {.pkg %s} %s did not apply cleanly; skipping patched build.", + entry$package, version + )) + return(NULL) + } + } + + build_env <- entry$env + if (length(entry$makevars) > 0L) { + mk <- tempfile(fileext = ".mk") + writeLines( + vapply( + names(entry$makevars), + function(k) sprintf("%s=%s", k, entry$makevars[[k]]), + character(1L) + ), + mk + ) + build_env$R_MAKEVARS_USER <- mk + } + + tryCatch( + withr::with_envvar(build_env, { + pkgbuild::build( + path = pkg_src, + binary = TRUE, + vignettes = FALSE, + dest_path = dest_dir, + args = configure_args_to_build_args(entry$configure_args), + quiet = TRUE + ) + }), + error = function(e) { + log_warn(sprintf( + "Isolated patched build of {.pkg %s} %s failed: %s", + entry$package, version, conditionMessage(e) + )) + NULL + } + ) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): build a patched binary in isolation from CRAN source" +``` + +--- + +### Task A5: Orchestrate the local patched repo (cache + index) + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: all helpers above. +- Produces: `prepare_patched_repo(patches_dir, platform, arch, r_minor, cache_dir, repo_dir)` → path to a `src/contrib`-style dir containing patched binaries + a `PACKAGES` index, or `NULL` when nothing matched/built. On a cache hit it copies the cached tarball into `repo_dir`; on a miss it builds, then writes the result into `cache_dir`. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("prepare_patched_repo returns NULL when no entries match", { + dir <- withr::local_tempdir() + jsonlite::write_json( + list(list(package = "A", versions = "*", platforms = list("redhat"), + reason = "r")), + file.path(dir, "registry.json"), auto_unbox = TRUE + ) + expect_null( + prepare_patched_repo(dir, "ubuntu-2604", "amd64", "4.5", + cache_dir = withr::local_tempdir(), + repo_dir = withr::local_tempdir()) + ) +}) + +test_that("prepare_patched_repo serves a cached binary and writes an index", { + dir <- withr::local_tempdir() + jsonlite::write_json( + list(list(package = "glue", versions = "*", platforms = list("*"), + env = list(A = "1"), reason = "r")), + file.path(dir, "registry.json"), auto_unbox = TRUE + ) + cache <- withr::local_tempdir() + repo <- withr::local_tempdir() + + local_mocked_bindings( + resolve_patch_version = function(entry) "1.0.0", + build_patched_binary = function(entry, version, dest_dir) { + f <- file.path(dest_dir, sprintf("%s_%s.tar.gz", entry$package, version)) + writeLines("fake binary", f) + f + } + ) + + out <- prepare_patched_repo(dir, "ubuntu-2604", "amd64", "4.5", + cache_dir = cache, repo_dir = repo) + + expect_identical(out, repo) + expect_true(file.exists(file.path(repo, "glue_1.0.0.tar.gz"))) + expect_true(file.exists(file.path(repo, "PACKAGES"))) + # The build result was cached under the key. + expect_length(list.files(cache, pattern = "^glue_1.0.0_.*\\.tar\\.gz$"), 1L) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function prepare_patched_repo". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Prepare a local repo of patched binaries for the current build +#' +#' For each registry entry matching the current platform, ensures a patched +#' binary is present in `repo_dir` (from `cache_dir` if available, else built +#' and then cached) and writes a `PACKAGES` index over them. +#' +#' @param patches_dir Directory with `registry.json`, or `NULL`. +#' @param platform Build platform, e.g. `"ubuntu-2604"`. +#' @param arch Build arch, e.g. `"amd64"`. +#' @param r_minor R `"major.minor"` string, e.g. `"4.5"`. +#' @param cache_dir Persistent cache for patched binaries. +#' @param repo_dir Directory to assemble the local repo in. +#' @return `repo_dir` if at least one patched binary was produced, else `NULL`. +#' @keywords internal +prepare_patched_repo <- function( + patches_dir, + platform, + arch, + r_minor, + cache_dir = file.path("/mnt", "cache", "patched-binaries"), + repo_dir = tempfile("patched_repo_") +) { + entries <- match_patch_entries( + load_patch_registry(patches_dir), platform, arch + ) + if (length(entries) == 0L) { + return(NULL) + } + + dir.create(repo_dir, recursive = TRUE, showWarnings = FALSE) + dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE) + + produced <- 0L + for (entry in entries) { + version <- resolve_patch_version(entry) + if (is.na(version)) { + log_warn(sprintf( + "No CRAN version of {.pkg %s} satisfies '%s'; patch skipped.", + entry$package, entry$versions + )) + next + } + + key <- patch_cache_key(entry, version, platform, arch, r_minor) + cached <- file.path(cache_dir, sprintf("%s.tar.gz", key)) + target <- file.path( + repo_dir, sprintf("%s_%s.tar.gz", entry$package, version) + ) + + if (file.exists(cached)) { + log_info(sprintf( + "Using cached patched binary for {.pkg %s} %s.", + entry$package, version + )) + file.copy(cached, target, overwrite = TRUE) + } else { + log_info(sprintf( + "Applying patch to {.pkg %s} %s [%s]: %s", + entry$package, version, describe_patch(entry), entry$reason + )) + built <- build_patched_binary(entry, version, repo_dir) + if (is.null(built)) { + next + } + if (!identical(normalizePath(built), normalizePath(target))) { + file.copy(built, target, overwrite = TRUE) + } + file.copy(target, cached, overwrite = TRUE) + } + produced <- produced + 1L + } + + if (produced == 0L) { + return(NULL) + } + cranlike::add_PACKAGES( + list.files(repo_dir, pattern = "\\.tar\\.gz$"), + repo_dir + ) + repo_dir +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 5: Update imports and document** + +Add to `DESCRIPTION` `Imports:` (alphabetical) `jsonlite` if not present, and ensure `pkgsearch`, `pkgbuild`, `cranlike`, `withr` are listed (they are). Then: + +```bash +Rscript -e 'devtools::document()' +git add R/patches.R tests/testthat/test-patches.R DESCRIPTION NAMESPACE +git commit -m "feat(patches): orchestrate local patched-binary repo with caching" +``` + +--- + +### Task A6: Wire patched repo into the pak install path + +**Files:** +- Modify: `R/install_helpers.R:333-389` (`run_pak_install_with_mutex`) +- Modify: `R/install-deps.R:23-75` (`install_pkg_sys_deps`) +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: `prepare_patched_repo()`. +- Produces: `run_pak_install_with_mutex(local_clone_dir_single, env_vars, patched_repo = NULL)` — prepends `file://` to `options("repos")` for the install; `install_pkg_sys_deps(package_name, tag, local_clone_dir, platform, aggressive_cleanup = FALSE, patches = NULL, arch = NULL)` — builds the patched repo before installing. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("run_pak_install_with_mutex prepends the patched repo to repos", { + seen <- NULL + local_mocked_bindings( + acquire_pak_mutex = function(...) tempfile(), + release_pak_mutex = function(...) invisible(NULL), + retry_with_backoff = function(func, ...) func() + ) + local_mocked_bindings( + local_install_deps = function(...) { + seen <<- getOption("repos") + invisible(TRUE) + }, + .package = "pak" + ) + + run_pak_install_with_mutex( + tempfile(), list(), patched_repo = "/tmp/patched" + ) + + expect_true(any(grepl("file:///tmp/patched", seen))) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL — `patched_repo` argument not yet accepted / repos not prepended. + +- [ ] **Step 3: Edit `run_pak_install_with_mutex` in `R/install_helpers.R`** + +Change the signature line: + +```r +run_pak_install_with_mutex <- function(local_clone_dir_single, env_vars) { +``` + +to: + +```r +run_pak_install_with_mutex <- function( + local_clone_dir_single, + env_vars, + patched_repo = NULL +) { +``` + +Replace the inner `retry_with_backoff(...)` block (the one wrapping `pak::local_install_deps`) with: + +```r + retry_with_backoff(function() { + withr::with_envvar(env_vars, { + repos <- getOption("repos") + if (!is.null(patched_repo)) { + repos <- c( + patched = sprintf("file://%s", patched_repo), + repos + ) + } + withr::with_options(list(repos = repos), { + # Default to non-verbose (suppressed messages) + suppressMessages(pak::local_install_deps(sprintf( + "%s", + local_clone_dir_single + ))) + }) + }) + }) +``` + +- [ ] **Step 4: Edit `install_pkg_sys_deps` in `R/install-deps.R`** + +Change the signature to add `patches` and `arch`: + +```r +install_pkg_sys_deps <- function( + package_name, + tag, + local_clone_dir, + platform = platform, + aggressive_cleanup = FALSE, + patches = NULL, + arch = NULL +) { +``` + +Immediately before the `run_pak_install_with_mutex(...)` call, insert: + +```r + # Build a local repo of patched binaries (if any apply) and serve it to pak. + r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." + ) + patched_repo <- tryCatch( + prepare_patched_repo(patches, platform, arch, r_minor), + error = function(e) { + log_warn(sprintf("Patch preparation failed: %s", conditionMessage(e))) + NULL + } + ) +``` + +and change the call from: + +```r + run_pak_install_with_mutex( + local_clone_dir_single, + env_vars + ) +``` + +to: + +```r + run_pak_install_with_mutex( + local_clone_dir_single, + env_vars, + patched_repo = patched_repo + ) +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add R/install_helpers.R R/install-deps.R tests/testthat/test-patches.R +git commit -m "feat(patches): serve patched binaries to pak during dep install" +``` + +--- + +### Task A7: Thread `patches` through the public build API + +**Files:** +- Modify: `R/build_binaries.R` (`build_binary_package`, `execute_package_builds`, `build_single_tag`, `handle_system_dependencies`) +- Create: `man-roxygen/param-patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Produces: `build_binary_package(..., patches = NULL)` and the internal chain each carry `patches` down to `install_pkg_sys_deps()`. `handle_system_dependencies(..., patches = NULL)` passes `patches` and `arch` through. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("handle_system_dependencies forwards patches and arch", { + captured <- list() + local_mocked_bindings( + install_pkg_sys_deps = function(package_name, tag, local_clone_dir_single, + platform, patches = NULL, arch = NULL) { + captured <<- list(patches = patches, arch = arch) + invisible(TRUE) + } + ) + handle_system_dependencies( + "RcppParallel", "5.1.11-2", "ubuntu-2604", tempfile(), "amd64", + NULL, NULL, NULL, NULL, NULL, NULL, NULL, + patches = "local/patches" + ) + expect_identical(captured$patches, "local/patches") + expect_identical(captured$arch, "amd64") +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL — `handle_system_dependencies` has no `patches` argument. + +- [ ] **Step 3: Create the roxygen template** + +```r +# man-roxygen/param-patches.R +#' @param patches Optional path to a patch registry directory containing a +#' `registry.json` (and any referenced diff files). When set, matching +#' packages are pre-built as patched binaries and served to `pak` during +#' dependency installation. Defaults to `NULL` (no patching). +``` + +- [ ] **Step 4: Edit the four functions in `R/build_binaries.R`** + +In `build_single_tag()`'s call to `handle_system_dependencies(...)`, add `patches = patches` as the final argument, and add `patches = NULL` to `build_single_tag`'s own signature plus `#' @template param-patches` to its roxygen block. + +Change `handle_system_dependencies` signature to end with `metadata_db_sslmode,` then add `patches = NULL`, and change its inner `install_pkg_sys_deps(...)` call from: + +```r + install_pkg_sys_deps( + package_name, + tag, + local_clone_dir_single, + platform + ) +``` + +to: + +```r + install_pkg_sys_deps( + package_name, + tag, + local_clone_dir_single, + platform, + patches = patches, + arch = arch + ) +``` + +In `execute_package_builds()`, add `patches = NULL` to the signature and pass `patches = patches` into its `build_single_tag(...)` call inside `worker_function`. + +In `build_binary_package()`, add `patches = NULL` to the signature (after `s3_package_cache`), add `#' @template param-patches` to its roxygen, and pass `patches = patches` into the `execute_package_builds(...)` call. + +- [ ] **Step 5: Document, test, and run package check** + +Run: + +```bash +Rscript -e 'devtools::document()' +Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")' +``` + +Expected: PASS for the new test; `man/build_binary_package.Rd` etc. regenerated. + +- [ ] **Step 6: Commit** + +```bash +git add R/build_binaries.R man-roxygen/param-patches.R man NAMESPACE tests/testthat/test-patches.R +git commit -m "feat(patches): thread patches argument through build_binary_package" +``` + +--- + +### Task A8: End-to-end patch test (guarded) and version bump + +**Files:** +- Modify: `tests/testthat/test-patches.R` +- Modify: `DESCRIPTION` (version), `NEWS.md` + +**Interfaces:** +- Produces: a guarded e2e test proving a dependent package builds when its failing dependency is patched. + +- [ ] **Step 1: Add the guarded e2e test** + +```r +test_that("a patched dependency unblocks a dependent build (e2e)", { + skip_if_not(nzchar(Sys.getenv("BINCRAFT_PATCH_E2E"))) + skip_if_offline() + + patches_dir <- withr::local_tempdir() + jsonlite::write_json( + list(list( + package = "RcppParallel", versions = "*", platforms = list("*"), + env = list(RCPP_PARALLEL_USE_TBB = "0"), + reason = "bundled TBB fails on this toolchain" + )), + file.path(patches_dir, "registry.json"), auto_unbox = TRUE + ) + + out <- withr::local_tempdir() + result <- build_binary_package( + "rts2", tag = "latest", local_output_dir_root = out, + upload = FALSE, archive = FALSE, patches = patches_dir + ) + expect_true(isTRUE(result) || identical(result, "skipped")) +}) +``` + +- [ ] **Step 2: Run the e2e test in a container** + +Run: + +```bash +docker run --rm -e BINCRAFT_PATCH_E2E=1 -v "$PWD":/work -w /work \ + reg.devxy.io/rpkgs/build-env-ubuntu:2604 \ + Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")' +``` + +Expected: the e2e test runs (not skipped) and PASSES; build log shows `Applying patch to RcppParallel`. + +- [ ] **Step 3: Bump version and changelog** + +In `DESCRIPTION`, bump `Version:` to `4.3.0.9999`. Prepend to `NEWS.md`: + +```markdown +# bincraft 4.3.0 + +* `build_binary_package()` gains a `patches` argument: a registry of + per-package env / configure / Makevars overrides and source diffs that are + pre-built into patched binaries and served to pak, fixing compiler- and + OS-specific failures (e.g. RcppParallel) including for transitive deps. +``` + +- [ ] **Step 4: Commit** + +```bash +git add tests/testthat/test-patches.R DESCRIPTION NEWS.md +git commit -m "test(patches): guarded end-to-end test; bump to 4.3.0.9999" +``` + +--- + +## Phase B — build-cran-binaries registry and wiring + +All Phase B paths are relative to this repo (`build-cran-binaries`). Phase B depends on a bincraft build that includes Phase A (install the updated bincraft in the build containers, or bump the pinned version in `.crow/build-all-versions-install-deps.yaml`). + +### Task B1: Create the patch registry with the RcppParallel entry + +**Files:** +- Create: `local/patches/registry.json` +- Create: `local/patches/README.md` + +**Interfaces:** +- Produces: the curated registry consumed by bincraft's `patches` argument. + +- [ ] **Step 1: Write the registry** + +```json +[ + { + "package": "RcppParallel", + "versions": "*", + "platforms": ["alpine", "ubuntu-2604"], + "env": { "RCPP_PARALLEL_USE_TBB": "0" }, + "configure_args": [], + "makevars": {}, + "patch": null, + "reason": "bundled Intel TBB fails to build on musl and on newer toolchains (e.g. g++ 15 on ubuntu-2604); disabling TBB falls back to TinyThread" + } +] +``` + +- [ ] **Step 2: Write the README** + +`local/patches/README.md` documents the schema (copy the field table from `specs/2026-06-30-package-patching-design.md`), how to add an entry, and that source diffs go in `local/patches//.patch` referenced by the `patch` field. + +- [ ] **Step 3: Verify it parses** + +Run: `Rscript -e 'x <- jsonlite::fromJSON("local/patches/registry.json", simplifyVector = FALSE); stopifnot(length(x) == 1L, x[[1]]$package == "RcppParallel"); cat("ok\n")'` +Expected: `ok`. + +- [ ] **Step 4: Commit** + +```bash +git add local/patches/registry.json local/patches/README.md +git commit -m "feat(patches): add patch registry with RcppParallel TBB workaround" +``` + +--- + +### Task B2: Registry validator script + +**Files:** +- Create: `local/validate-patches.R` + +**Interfaces:** +- Consumes: `local/patches/registry.json`. +- Produces: a script that exits non-zero on schema violations, missing patch files, or ambiguous overlapping entries. + +- [ ] **Step 1: Write the validator** + +```r +#!/usr/bin/env Rscript +# Validate local/patches/registry.json: schema, referenced patch files, and +# ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. + +dir <- "local/patches" +registry_file <- file.path(dir, "registry.json") +if (!file.exists(registry_file)) { + cat("No registry.json found; nothing to validate.\n") + quit(status = 0L) +} + +reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +required <- c("package", "versions", "platforms", "reason") +errs <- character(0L) + +for (i in seq_along(reg)) { + e <- reg[[i]] + missing <- setdiff(required, names(e)) + if (length(missing) > 0L) { + errs <- c(errs, sprintf( + "entry %d (%s): missing %s", i, + if (is.null(e$package)) "?" else e$package, toString(missing) + )) + } + if (!is.null(e$patch)) { + p <- file.path(dir, e$patch) + if (!file.exists(p)) { + errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing", + i, e$package, p)) + } + } +} + +# Ambiguous overlap: two entries for the same package with identical platforms +# and versions. +keys <- vapply(reg, function(e) { + sprintf("%s|%s|%s", e$package, + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + e$versions) +}, character(1L)) +dups <- keys[duplicated(keys)] +if (length(dups) > 0L) { + errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups)))) +} + +if (length(errs) > 0L) { + cat("Patch registry validation FAILED:\n") + cat(paste0(" - ", errs, "\n")) + quit(status = 1L) +} +cat(sprintf("Patch registry OK (%d entrie(s)).\n", length(reg))) +``` + +- [ ] **Step 2: Run it (expect success on the B1 registry)** + +Run: `Rscript local/validate-patches.R` +Expected: `Patch registry OK (1 entrie(s)).` and exit 0. + +- [ ] **Step 3: Run it against a broken registry (expect failure)** + +Run: + +```bash +cp local/patches/registry.json /tmp/reg.bak +Rscript -e 'writeLines("[{\"package\":\"X\"}]", "local/patches/registry.json")' +Rscript local/validate-patches.R; echo "exit=$?" +cp /tmp/reg.bak local/patches/registry.json +``` + +Expected: prints `validation FAILED` with a missing-field message and `exit=1`. + +- [ ] **Step 4: Commit** + +```bash +git add local/validate-patches.R +git commit -m "feat(patches): add registry validator script" +``` + +--- + +### Task B3: Hook the validator into pre-commit + +**Files:** +- Modify: `.pre-commit-config.yaml` + +**Interfaces:** +- Produces: a local hook that runs `local/validate-patches.R` when the registry or patch files change. + +- [ ] **Step 1: Add the hook** + +Add a `repo: local` hook entry to `.pre-commit-config.yaml`: + +```yaml + - repo: local + hooks: + - id: validate-patches + name: validate patch registry + entry: Rscript local/validate-patches.R + language: system + files: ^local/patches/ + pass_filenames: false +``` + +- [ ] **Step 2: Verify the hook runs** + +Run: `pre-commit run validate-patches --all-files` +Expected: hook passes (`Patch registry OK`). + +- [ ] **Step 3: Commit** + +```bash +git add .pre-commit-config.yaml +git commit -m "ci(patches): validate patch registry in pre-commit" +``` + +--- + +### Task B4: Pass `patches` through the build entry points + +**Files:** +- Modify: `local/build-one.R:99-119` (the `build_binary_package` call) +- Modify: `local/build-all.R:125-145` (the `build_binary_package` call) + +**Interfaces:** +- Consumes: the new bincraft `patches` argument (Phase A) and `local/patches/`. +- Produces: both entry points pass `patches = "local/patches"`. + +- [ ] **Step 1: Edit `local/build-one.R`** + +In the `bincraft::build_binary_package(` call, add as a new argument (e.g. after `archive = TRUE,`): + +```r + patches = "local/patches", +``` + +- [ ] **Step 2: Edit `local/build-all.R`** + +In the `bincraft::build_binary_package(` call, add: + +```r + patches = "local/patches", +``` + +- [ ] **Step 3: Verify the scripts still parse** + +Run: `Rscript -e 'invisible(parse("local/build-one.R")); invisible(parse("local/build-all.R")); cat("parse ok\n")'` +Expected: `parse ok`. + +- [ ] **Step 4: Commit** + +```bash +git add local/build-one.R local/build-all.R +git commit -m "feat(patches): pass patch registry to bincraft build calls" +``` + +--- + +### Task B5: Document the feature in the README + +**Files:** +- Modify: `README.md` + +**Interfaces:** +- Produces: a short "Patching packages" section explaining the registry and linking the design spec. + +- [ ] **Step 1: Add a README section** + +Add a `## Patching packages` section after the "Build Process" section describing: why patching exists (compiler/OS-specific failures cascading via shared deps like RcppParallel), where the registry lives (`local/patches/registry.json`), the two tiers (env/configure/Makevars overrides vs source diffs), and that bincraft pre-builds patched binaries served to pak. Link `specs/2026-06-30-package-patching-design.md`. + +- [ ] **Step 2: Commit** + +```bash +git add README.md +git commit -m "docs(patches): document the package patching workflow" +``` + +--- + +## Self-Review + +**Spec coverage:** +- Registry in this repo, passed to bincraft → Task A7 (`patches` arg), B1 (registry), B4 (wiring). ✓ +- Mechanism in bincraft (resolve → pre-build → prepend repo → install) → A4–A6. ✓ +- Both tiers, env-first → A4 (`env`, `configure_args`, `makevars`, `patch` all applied; env is the cheap default and the RcppParallel entry uses only env). ✓ +- Pre-built patched binary served from a prepended local repo → A0 proof + A5/A6. ✓ +- Cache keyed by pkg/version/platform/arch/rminor/patchhash → A3 (`patch_cache_key`) + A5 (cache use). ✓ +- S3 upload of patched binaries → **partial**: A5 caches locally only. Cross-machine S3 reuse is deferred (see note below) to keep the first cut shippable; local `/mnt/cache` reuse already removes per-dependent rebuilds within a container. Recorded as out-of-first-cut, not dropped. +- Error handling (diff fails → skip+warn; build fails → skip+warn; no match → skip; ambiguous overlap → validation error) → A4 (`apply_source_patch` FALSE path, build tryCatch), A5 (`resolve_patch_version` NA path), B2 (overlap validation). ✓ +- Observability (log line per applied patch) → A5 (`log_info` with `describe_patch`). DB metadata recording of applied patches is deferred with S3 (same note). +- Testing (registry parse/match, cache key, integration, failure path) → A1–A5 unit tests, A8 e2e, A4 patch-fail test. ✓ + +**Deferred from spec (call out to user):** S3 upload/reuse of patched binaries and recording applied patches in the Postgres build-metadata row. The local `/mnt/cache/patched-binaries` cache already prevents repeated rebuilds within a container; S3 reuse across CI jobs is a follow-up. If you want it in the first cut, add a Task A5b (upload `cached` tarball to a `…/patched/` S3 slot and check there before building) and a metadata column — say so and I'll insert them. + +**Placeholder scan:** No TBD/TODO; every code step shows complete code. README/registry-README prose steps describe exact content to write (acceptable for docs). + +**Type consistency:** `prepare_patched_repo` signature is identical across A5 (definition) and A6 (call site, via defaults). `patch_cache_key(entry, version, platform, arch, r_minor)` argument order matches between A3 and A5. `run_pak_install_with_mutex(..., patched_repo)` matches between A6 definition and the A6 test. `handle_system_dependencies(..., patches)` matches between A7 edit and A7 test. `build_patched_binary(entry, version, dest_dir)` matches A4 and A5. + +**Open risk:** Task A0 gates the whole approach; if it fails, the documented source-serving contingency keeps every later task valid with a localized change in A4. diff --git a/provider.tf b/provider.tf index 0a9ba15..b267fcc 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.14" + version = "~> 0.15" } } } diff --git a/specs/2026-06-30-package-patching-design.md b/specs/2026-06-30-package-patching-design.md new file mode 100644 index 0000000..1344173 --- /dev/null +++ b/specs/2026-06-30-package-patching-design.md @@ -0,0 +1,157 @@ +# Design: Dynamic per-package patching during binary builds + +Date: 2026-06-30 +Status: Approved (pending spec review) + +## Problem + +Some CRAN packages fail to compile on specific build platforms due to compiler- or OS-specific issues that have nothing to do with the package being built. +The canonical example is `RcppParallel`: its bundled Intel TBB sources fail to build on musl (Alpine) and on newer OS/compiler combinations. +Observed failure on `ubuntu-2604` ("resolute") with `g++ 15.2.0`: + +``` +../build/common.inc:74: *** "" is not supported. Add build/.inc file with os-specific settings . Stop. +make: *** [Makevars:163: tbb] Error 2 +ERROR: compilation failed for package 'RcppParallel' +``` + +Because `RcppParallel` is a dependency of many packages, a single such failure cascades: every dependent package (e.g. `rts2`) also fails, even though nothing is wrong with the dependent itself. + +Today there is no way to intervene. +A package can only be **excluded** (`local/excluded-packages.json`), which is all-or-nothing and does not help dependents. + +## Goal + +Allow a curated set of packages to be "patched" — via lightweight build-time overrides or, when necessary, real source diffs — **before** they are installed, whether the package is a direct build target or a transitive dependency pulled in by `pak`. + +## Key constraint that drives the design + +When `RcppParallel` fails here, it is being installed as a **transitive dependency** by `pak`, inside `bincraft::build_binary_package()`. +`pak` downloads, configures, and compiles it in one subprocess; this repo never touches that source. +For a fix to reach a dependency-of-a-dependency, the fixed package must be visible to `pak` itself, where `pak`'s repositories/sources are configured — which is inside `bincraft`. + +Decisions taken during brainstorming: + +- **Mechanism lives in `bincraft`** (the engine), because only there can transitive deps be influenced. +- **Patch tiers: both, env-overrides first.** Support cheap per-package env vars / configure args / Makevars (version-independent) *and* true source diffs (version-pinned), preferring the lightweight override. +- **Registry data lives in this repo** (`build-cran-binaries`) and is passed into `bincraft`, keeping `bincraft` as pure mechanism and the frequently-changing policy data with operational config. + +## Approaches considered + +| Approach | How pak sees the fix | Verdict | +|---|---|---| +| A. Patched **source** repo — drop patched `.tar.gz` source into a local repo, prepend it | pak recompiles from your source | Simple, but env-tier overrides leak globally (one subprocess builds everything) and the dep recompiles on every dependent build | +| **B. Pre-built patched binary repo (chosen)** | pak installs a ready binary by repo priority | Per-package scoping is free; no recompile; the binary is a cacheable/uploadable artifact that fits the existing system | +| C. pkgdepends per-build hook | intercept each build | No clean per-package pre-compile hook exists; fragile | + +Chosen: **B**. + +## Architecture + +### Registry (this repo) + +``` +local/patches/ + registry.json # the manifest + RcppParallel/ + fix.patch # optional source diff, referenced by an entry +``` + +`registry.json` is an array of entries: + +```json +[ + { + "package": "RcppParallel", + "versions": "*", + "platforms": ["alpine", "ubuntu-2604"], + "env": { "RCPP_PARALLEL_USE_TBB": "0" }, + "configure_args": [], + "makevars": {}, + "patch": null, + "reason": "bundled TBB fails to build on musl / newer compilers" + } +] +``` + +Field semantics: + +- `package` (string, required): CRAN package name. +- `versions` (string, required): `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. + Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. +- `platforms` (array of strings, required): matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). + An entry matches if any listed token matches any build token. + `["*"]` matches all platforms. +- `env` (object, optional): environment variables exported only for this package's isolated build. +- `configure_args` (array, optional): passed as `--configure-args` to the isolated build. +- `makevars` (object, optional): key/value pairs written into a package-local Makevars for the isolated build. +- `patch` (string or null, optional): path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. +- `reason` (string, required): human explanation, surfaced in logs and metadata. + +A fix is any combination of `env`, `configure_args`, `makevars`, and `patch`. +"Env-first" is an authoring guideline (prefer the lightweight override) and an ordering of effort, not a runtime branch — all present fields are applied together for the isolated build. + +### Flow (inside bincraft, around existing pak resolution) + +1. **Resolve** the dependency set (dry-run) to learn the concrete versions `pak` will install. + Reuse bincraft's existing resolution where possible (e.g. a `pkgdepends` proposal: `$resolve()` → inspect resolution → ... → `$solve()` / `$install()` after the local repo is prepended). +2. For each resolved package that matches a registry entry (name + `versions` + `platforms`): obtain a **patched binary** for the exact `version × platform × arch × R-minor`: + - **Cache hit** (local `/mnt/cache/patched-binaries/` or S3): fetch it into the local repo. + - **Cache miss**: download the CRAN **source** for that version, apply the source `patch` (if any) to the unpacked tree, build the binary in isolation with `env` / `configure_args` / `makevars` applied, then place the binary in the local repo and write it to the cache (and S3 if uploading is enabled). +3. **Prepend** the local binary repo (`file://…`) to `pak`'s repo list, and regenerate its `PACKAGES` index. +4. Run the **normal install**. + `pak` resolves the patched binary for the matched package — direct or transitive — because it wins on repo priority for an equal version, and installs it without recompiling. + +### Caching (essential) + +`RcppParallel` is a dependency of dozens of packages; without caching the fix would be rebuilt on every dependent build. +Patched binaries are keyed by: + +``` +_____ +``` + +`patchhash` is a hash of the normalized registry entry plus the referenced diff file contents. +Editing a patch therefore changes the hash and auto-invalidates stale cached binaries. + +- Local cache: `/mnt/cache/patched-binaries/`. +- Optional S3 cache for cross-build reuse: a dedicated `…/patched/` slot under the existing arch/codename structure, mirroring how normal binaries are stored. + +### S3 upload + +Patched binaries **are** uploaded to S3 (in addition to the local cache) so they are reused across CI jobs and machines, not just within one container. +They live in a separate `patched/` slot and are not published into the user-facing `src/contrib` index — they are an internal build accelerator, not a distributed artifact. + +## Error handling + +- **Source diff fails to apply** (CRAN moved past the pinned version): log a clear warning, skip that entry, and proceed. + The package builds unpatched (status quo) and may fail. + The skipped/failed-to-apply patch is surfaced in build metadata. +- **Pre-build of the patched binary fails**: log a warning, skip, proceed. +- **No version or platform match**: skip silently (the entry simply does not apply to this build). +- **Overlapping entries for one package**: the most specific entry wins (a concrete `platforms`/`versions` beats `"*"`). + Genuine ambiguity (two equally specific, conflicting entries) is a validation error reported before the build. + +## Observability + +- One log line per applied patch, e.g.: `Applying patch to RcppParallel 5.1.11-2 [env: RCPP_PARALLEL_USE_TBB=0]: bundled TBB fails on musl / newer compilers` +- The set of applied patches (package, version, `patchhash`) is recorded in the Postgres build-metadata row for the build, so it is queryable later. + +## Testing + +- **Unit (registry):** parsing and matching — version constraints, platform token matching, precedence/specificity, and detection of ambiguous overlaps. +- **Unit (cache key):** `patchhash` changes when the entry or diff changes; is stable otherwise. +- **Integration:** `RcppParallel` on `resolute` (and/or Alpine) fails to build without a registry entry and succeeds with one; a dependent package such as `rts2` succeeds once the dependency is patched. +- **Failure path:** an entry pinned to an old version against a newer CRAN release → graceful skip with a warning, build continues. + +## Out of scope + +- Shipping a default registry inside `bincraft` (registry is repo-local for now; a baseline-in-engine + repo-override model can come later if needed). +- Publishing patched binaries into the public `src/contrib` index. +- Automatic detection of which packages need patches — entries are curated by hand. + +## Split of work + +- **bincraft:** the mechanism — registry ingestion, resolution hook, isolated patched-binary build, caching/upload, local-repo prepend, logging, metadata recording. + A new `patches` argument on `build_binary_package()`. +- **build-cran-binaries (this repo):** the `local/patches/` registry and diffs, passing `patches = "local/patches"` through `build-one.R` / `build-all.R`, and documentation.