diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 71f808b..f1fe920 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,9 +62,9 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' + - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), workers = 2L)' backend_options: kubernetes: resources: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index e0cca86..389b57b 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -10,22 +10,11 @@ variables: - arm64 default: amd64 OS: - description: "Base OS image name." - options: - - alpine - - redhat - - ubuntu + description: 'Base OS image name (e.g. alpine, redhat, ubuntu).' default: alpine OS_VERSION: - description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." - options: - - "3.24" - - "8" - - "9" - - "10" - - "jammy" - - "noble" - default: "3.24" + description: 'OS version / image tag (e.g. 3.24, 9, jammy, noble).' + default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -53,9 +42,6 @@ steps: image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN GITHUB_PAT: @@ -80,9 +66,8 @@ steps: - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - # Pin the same bincraft version the build steps use, so the precomputed - # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 3932811..1d4feee 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -11,25 +11,11 @@ variables: - arm64 default: amd64 OS: - description: "Base OS image name." - options: - - alpine - - redhat - - ubuntu + description: 'Base OS image name (e.g. alpine, redhat, ubuntu).' default: alpine OS_VERSION: - description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." - options: - - "3.22" - - "3.23" - - "3.24" - - "8" - - "9" - - "10" - - "jammy" - - "noble" - - "resolute" - default: "3.24" + description: 'OS version / image tag (e.g. 3.24, 9, jammy, noble).' + default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -90,9 +76,6 @@ steps: image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW B2_S3_ACCESS_KEY: @@ -122,14 +105,6 @@ steps: # growth is bounded separately by trim_pkgcache_metadata() in build-all.R. - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # The primary pass must not rely on build-all-versions-install-deps having - # run on *this* agent: depends_on only orders the steps, but the cache - # volume is per-agent, so a job landing on an agent where install-deps did - # not run would otherwise use a stale bincraft (which resolves `platform` - # to a zero-length value and breaks every metadata query and the sysdeps - # install). Pin bincraft here, exactly like the R-minor pass below. - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -141,12 +116,11 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done - # archive missed packages; first arg is the codename (e.g. "alpine324"), - # derived via bincraft like the upload step, not paste(OS, OS_VERSION). - - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(bincraft::set_codename(NULL), Sys.getenv('ARCH'), s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), workers = $NCPUS)" + # archive missed packages + - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" backend_options: docker: resources: @@ -160,9 +134,6 @@ steps: image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none B2_S3_ACCESS_KEY: from_secret: B2_S3_ACCESS_KEY B2_S3_SECRET_KEY: diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 3da41fd..9d40c67 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -15,8 +15,6 @@ variables: - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -27,8 +25,6 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 default: all when: @@ -68,18 +64,6 @@ matrix: IMG: alpine:3.24 OS_ID: alpine323 PROCESS_NEW: "FALSE" - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - OS_ID: alpine324 - PROCESS_NEW: "FALSE" - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - OS_ID: alpine324 - PROCESS_NEW: "FALSE" - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 @@ -140,27 +124,12 @@ matrix: IMG: ubuntu:noble OS_ID: noble PROCESS_NEW: "TRUE" - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: ubuntu:resolute - OS_ID: resolute - PROCESS_NEW: "TRUE" - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: ubuntu:resolute - OS_ID: resolute - PROCESS_NEW: "TRUE" steps: - name: 'Processing Updates' image: reg.devxy.io/rpkgs/build-env-${IMG} pull: true environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW B2_S3_ACCESS_KEY: @@ -195,13 +164,13 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) for RBIN in /opt/R/[0-9]*/bin/R; do @@ -211,8 +180,8 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true - R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | @@ -233,9 +202,6 @@ steps: - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY SUBDOMAIN1: 'cran.devxy.io' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 934c518..843429e 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -103,9 +103,6 @@ steps: image: reg.devxy.io/rpkgs/build-env-${IMG} pull: true environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW B2_S3_ACCESS_KEY: @@ -132,12 +129,12 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: docker: resources: diff --git a/.gitignore b/.gitignore index f4124a2..94e2c2f 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,3 @@ docs/ local/test.R .DS_Store docs/ - -# Superpowers SDD scratch (briefs, reports, ledger) -.superpowers/ diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml index 032c0ea..66e7cc8 100644 --- a/.markdownlint-cli2.yaml +++ b/.markdownlint-cli2.yaml @@ -2,6 +2,3 @@ ignores: - LICENSE.md - docs/superpowers/** - # Internal design docs (specs/plans) are not user-facing reference material. - - specs/** - - plans/** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0f4422..6784ad2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,19 +16,19 @@ repos: args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.23.0 + rev: v0.22.1 hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.4 + rev: v3.8.4 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit - rev: 0.10.0 + rev: 0.9.0 hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.8.0 + rev: v3.7.0 hooks: - id: editorconfig-checker - repo: https://github.com/adrienverge/yamllint.git @@ -43,9 +43,3 @@ repos: entry: YAML filenames must have .yaml extension. language: fail files: .yml$ - - id: validate-patches - name: validate patch registry - entry: Rscript local/validate-patches.R - language: system - files: ^local/(patches/|validate-patches\.R$) - pass_filenames: false diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 6ef10fa..2ae039e 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.15.1" - constraints = "~> 0.15" + version = "0.14.3" + constraints = "~> 0.14" hashes = [ - "h1:/2NUpbtjkc+w6n5V3kGP0rSzGjN0K2Wdfe2K+CZdmhU=", - "h1:1TOrpmCR0aT5xX1sjt70zqlYkMJnVe6r0nx2B0DS/mE=", - "h1:4uC8r8ILr+vZJ230uGqIUaWQ7uORCrIzoEuYrAq2JuE=", - "h1:6b+2osJUfwcaYZ6mathLPf/58sr/4XkLXQrcSufnkMk=", - "h1:CPldfjf79QS8mIP+GWoS0FVhrFlyjvN2+39zfyP76Ik=", - "h1:EvXICHyGIKpoYlDeKHOPzfmpvdRdhgsOMcR2nb6+tKY=", - "h1:GZLq+nDxS1CyBH0ELGTSQj9X7ozemJ1jpPA4KwbtR+c=", - "h1:ISNFqL745IQgZ6yMLy8ofV8ixbYqZYa9JKdi2W3pmNk=", - "h1:IrNrEuvFd0nYDGQefwmT8d1CSJb9e8LN5w9vw1ODp7E=", - "h1:Ms79slY9bZ94+n4cwIHvI9+/cvbucwo6S1+z5KAiznw=", - "h1:NWA9XSEBcpSkgwwIvl6tHrxGQY3uYhqNS4Vnb9RLyLQ=", - "h1:UjvxxxggicLtiE3yTe1Gx0oLUTeZpWmgIfXuHzWHn1c=", - "h1:VgJjo14DGkU4Jwo4D3GT4/5sq1tdjiZscKS5l3cb890=", - "h1:dBu3AW5YNLIvbBIMNk3wUHKw4TW+BDbj34a+mCqYhWE=", - "h1:i5oGD06nQ3JRsBIa2u4wCej+ETgp970CFl75dOKkHno=", - "h1:mTqR+vD1AWPx+mu7S0/pzBy71z7WOrjH7arOP77PXh4=", - "h1:nmTM61G8vYjpofeEqspMORpsNvTGCNZySGfjdXardL0=", - "zh:0f9bf5aaa47164a4d6ae4433d5e285a9456a5053401b2bad4ed68622f574ddee", - "zh:3039bee421fb8855a919f449fc731d145254371f5f0c39cc4660f3aeed6a8b10", - "zh:36664b08186e0c194747b18dee24ed97327c6e704133d4cf0df27abef1652f86", - "zh:3c7eae99d8c5ff65dfb99c8b9c1980147282d68971e1ef1ff0f126a6bff59d8f", - "zh:5293cc21abf54f4e5745437ca2d40d206aae323b2e1d41cf45dcc63a8868cbb3", - "zh:5994e5145e616e7e881010717e4c7def2945eb6d933f62db4ec3167732ccac84", - "zh:7994db9ed3fdb6cbf21f2154ea962cf82f04e425988bedc6657de2497d6cb6c3", + "h1:2VmbbvV/3fVb/dHZ5m3CxSPoptpn/PuhvSgV/RA9Ae8=", + "h1:4MANuttWPyJGZlfbx5P4i2Jpbmvda1tfWgd3olK2nhY=", + "h1:AsSfDvm6flYQgjFnFVaRPv7v1/P5pj36dQ4N103Nw+A=", + "h1:D2AK5psWQmZqrOHsbeV70OP/zcG92i++AmGKBz/HVMA=", + "h1:DpJWi/bhHoZa/ccy2YB/c7HJjTlR6CuZBxGFCxU2uJk=", + "h1:Iz47FCzHBOZC1RrTxKwJwKsH6bdEvdYtO0+RBpbmz8I=", + "h1:RJQm4qEfljAmcJaFf45uXwUhtbRTwfDuiUWZ6Q7X+Cw=", + "h1:YQBi0MRW+dOfkxQYbjKkzDm7UIcc2FeFm8Cck+W3kgs=", + "h1:bWq/zJ2Chr5TNuz9y3eaGmzrmXsX30pl5HkGpxE1kVM=", + "h1:hwzEskDnFHaJFvKNAGv/wIxRT2SHcXLfwmBb8XNspq4=", + "h1:jGggry0lZmG0S0inM7Odae3275na9Gp3O97ZlvBiA2s=", + "h1:mJdpVc9qOpqXcPpptWfAET8qloo6qFMb+NzHesazybk=", + "h1:nbpkaopSGlmxkKLtNe9sKx9IqojoicD1+hNRdaYd5ok=", + "h1:pwuoUqikuVOut3RtPTnu6pNoDnyeXlEX7Rfb2Hr1Ch0=", + "h1:u0naFiWGrBVhI3AsUb/ZfS5N/GeiRk0ZkxjHtA10lF0=", + "h1:uozoSPzZ7MJIHj5O62W8n86NIfH4JtF7GvMNI6rKlrM=", + "h1:vDS308OeXVVgjojMCAAEQ1VshmOLjwra7obliKB5fv8=", + "zh:23200ad919df4c44a720c667c536d3a0ae0c235e1f7b53b9e56f88005bf01891", + "zh:2fe5445f578c6a244995538fde6dd7ef0dd5f3969686cba3d91fcf28ea31278d", + "zh:3a5a8139ed177ccbee74feb4d19c881e18723e4bf3a64b6d35931420402a2a1a", + "zh:5189bd6c03ec3c428185992f519f3bf520d4e9eba8771c240814adbecdf60140", + "zh:532ac8eb94a84ccf046d4cb4eded825269effdbb7597f392f848fbfad5dcae1d", + "zh:5f100279d73ce998c7bf4ffa2bbcd617ee307bf51c4a94146f210aeec268c4d0", + "zh:80cb58abd2e431386ab1b6ad186a6e6e5347448993bacc831c0251b9016445b9", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:93aa863e536ba9376ccf9e614e9edc9b214a2ce8c4316d416a8e249b436f52d2", - "zh:ab5cb4baeda57559686a0ccf0e09158aa64624ee6ba0ef32b769f13b11a43068", - "zh:ae9388b62eede8fd9272407bf75f8241a965bd489d45ec9dd3f9fac696d500e1", - "zh:caa5befd16960e2f69c7ec483e228e5ff43ab0979c17f1b874c9ffaa1c7c0e43", - "zh:cddd3e1067defa06a4e4ad5cb3940c7943e29c42417de57236aa7d3e2aeaae13", - "zh:cdfa44d591d0805116159556947904d70f534c6816188c45cf3a7544d2722ac9", - "zh:d607e9f1f3e09f13404f219e1893e3b3c77aece4afb54e999f934c021f41f576", - "zh:d8a397aca95125c6a0c0c78d2ded5843b9204effa9f5cf7419f017b500ad9228", - "zh:f5499eaff0d221725ad209d27d87c5b46d5c554caad7dc42947c760377abe3b0", - "zh:fc5f5cf433abc83e5169fa222992ec521c0c802075970251e3dd2c1d50c4f5c1", + "zh:9ac734c6aae19ea0832269f1f1dc3c3e100855c09b6168eecd8693958515b228", + "zh:a0e3b75176c71e5f41a6552cef57ad67d11778cb59fd2eb792e69f662b67ae01", + "zh:b4d0e1dd9a80d1ac9a3f1beaefb58d4b944cb0a30bdc672b9037a69b4dc20ec3", + "zh:b8309337e966528991cc2dd61ff0247864e4c3c7bef7dcf225f066edb6171050", + "zh:c34ce5fcdd481214697505bceb616a7dfcda7702f1afefa7dc5459965b8e01ca", + "zh:cb3f662ad06bb86aa763a5644ba318bc3db0ed30b04eb810c478e81f4b012d55", + "zh:cf0f0a459e2b6b1e014af9f53f32b5f555a249b61a0dcaccd3e4fe89178d8aa5", + "zh:d4dfb653a0295bfbbebcb4bb022b490f2c24ca31287ebf77fe9dcaee2e5ea658", + "zh:e49ba31aa092a30ea8a1091f7b26dbe41c3998f924f5b0d82810f27fc2f63501", + "zh:ea4d7d5688caf2f9cda3a1721779b61a051db6b43ca7267ae2712bd1262cfcfd", ] } diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index faae5f1..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,14 +0,0 @@ -# CLAUDE.md - -Crow CI pipelines (`.crow/`) and local tooling (`local/`) that build CRAN binary packages (incl. Alpine/musl) and upload them to Backblaze B2. - -## Conventions - -- **PRs:** the remote is Forgejo on `codefloe.com`; use `fj -H codefloe.com` (not `gh`). -- **Storage:** Backblaze B2 bucket `devxy-r-builds` (endpoints configured in the `.crow/` pipelines). - -## Gotchas - -- **B2 requires authentication for the list-bucket API.** Anonymous GET only works for individual public-read objects — an empty listing means missing credentials, not an empty bucket. -- Weekly-rebuild pipelines run for hours; watch them as background tasks and fetch Crow logs yourself instead of having the user paste progress. -- musl builds of packages with bundled native deps (e.g. RcppParallel/TBB) recur as failures; check for an existing patch before re-deriving a fix. diff --git a/README.md b/README.md index 28541a1..278a475 100644 --- a/README.md +++ b/README.md @@ -53,28 +53,6 @@ For every package+tag combination: 1. Archive old package versions and keep the latest one in the root 1. Delete local binaries after successful upload -## Patching packages - -Some CRAN packages fail to compile on specific platforms due to compiler- or OS-specific issues unrelated to the package itself. -The canonical example is `RcppParallel`, whose bundled TBB sources fail on musl (Alpine) and newer compiler/OS combinations. -Because such packages are often transitive dependencies of many others, a single failure cascades: all dependents fail even though nothing is wrong with the dependent itself. - -To address this, frequently-failing packages can be "patched" before they are installed — whether as a direct build target or a transitive dependency pulled in by `pak`. - -The patch registry lives in `local/patches/registry.json`. -Each entry specifies a package and the platforms/versions it applies to, along with either lightweight build-time overrides (environment variables, configure arguments, Makevars) or a source diff (for deeper fixes). -See `local/patches/README.md` for the complete schema. - -Patching uses a two-tier approach: - -1. **Lightweight overrides:** environment variables, configure arguments, or Makevars settings applied during build — typically version-independent and fast. -2. **Source diffs:** unified diff patches applied to the unpacked source before building — more powerful but version-pinned. - -The system is implemented in `bincraft`: when a package needs patching, `bincraft` pre-builds it with the patch and serves the patched binary to `pak`, ensuring transitive dependents receive the fixed package. -This way, the fix cascades to all packages that depend on it. - -For the design rationale and architecture, see `specs/2026-06-30-package-patching-design.md`. - ## Build Environment Binaries are built on a mixed-architecture Kubernetes cluster using CI. @@ -221,7 +199,7 @@ Tag does not have a NAMESPACE file and hence cannot be built. Dependency not available: Either because the dependency was not declared or errored itself during installation. ```text - In function '\033[01m\033[KRcpp::List solveRRBLUP(const mat&, const mat&, const mat&)\033[m\033[K':\n\033[01m\033[KMME.cpp:162:61:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope\n 162 | double ll = -0.5*(double(optRes[\"objective\"])+df+df*log(2*\033[01;31m\033[KPI\033[m\033[K/df));\n | \033[01;31m\033[K^~\033[m\033[K\n\033[01m\033[KMME.cpp:\033[m\033[K In function '\033[01m\033[KRcpp::List solveRRBLUPMV(const mat&, const mat&, const mat&, int, double)\033[m\033[K':\n\033[01m\033[KMME.cpp:277:31:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope; did you mean '\033[01m\033[KHI\033[m\033[K'?\n 277 | ll -= double(n*m)/2.0*log(2*\033[01;31m\033[KPI\033[m\033[K);\n | \033[01;31m\033[K^~\033[m\033[K\n | \033[32m\033[KHI\033[m\033[K\nmake: *** [/opt/R/4.4.2/lib/R/etc/Makeconf:204: MME.o] Error 1\nERROR: compilation failed for package 'AlphaSimR'\n* removing '/tmp/RtmpclI5CE/temp_libpath11135d215d5/AlphaSimR'\n + In function '\033[01m\033[KRcpp::List solveRRBLUP(const mat&, const mat&, const mat&)\033[m\033[K':\n\033[01m\033[KMME.cpp:162:61:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope\n 162 | double ll = -0.5*(double(optRes[\"objective\"])+df+df*log(2*\033[01;31m\033[KPI\033[m\033[K/df));\n | \033[01;31m\033[K^~\033[m\033[K\n\033[01m\033[KMME.cpp:\033[m\033[K In function '\033[01m\033[KRcpp::List solveRRBLUPMV(const mat&, const mat&, const mat&, int, double)\033[m\033[K':\n\033[01m\033[KMME.cpp:277:31:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope; did you mean '\033[01m\033[KHI\033[m\033[K'?\n 277 | ll -= double(n*m)/2.0*log(2*\033[01;31m\033[KPI\033[m\033[K);\n | \033[01;31m\033[K^~\033[m\033[K\n | \033[32m\033[KHI\033[m\033[K\nmake: *** [/opt/R/4.4.1/lib/R/etc/Makeconf:204: MME.o] Error 1\nERROR: compilation failed for package 'AlphaSimR'\n* removing '/tmp/RtmpclI5CE/temp_libpath11135d215d5/AlphaSimR'\n ``` Compiler error: Possible reasons: too old CXX code which cannot be compiled anymore with CXX14 or CXX17. diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 5bc107a..2cd4d51 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -16,11 +16,6 @@ ARG CACHEBUST WORKDIR /work COPY build-one.R /work/build-one.R -# Resolve and install the latest bincraft release dynamically (no hardcoded pin). -COPY install-bincraft.R /work/install-bincraft.R -# Ship the patch registry so build-one.R's `patches = "local/patches"` resolves -# (build context is `local/`, CWD is /work). -COPY patches /work/local/patches RUN --mount=type=secret,id=b2_access,required=true \ --mount=type=secret,id=b2_secret,required=true \ @@ -32,9 +27,6 @@ RUN --mount=type=secret,id=b2_access,required=true \ export GITHUB_PAT="$(cat /run/secrets/github_pat 2>/dev/null || true)" && \ export GIT_TERMINAL_PROMPT=0 && \ export OTEL_SDK_DISABLED=true && \ - export OTEL_R_TRACES_EXPORTER=none && \ - export OTEL_R_LOGS_EXPORTER=none && \ - export OTEL_R_METRICS_EXPORTER=none && \ XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run || true); \ XVFB_ARGS=""; \ if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi; \ @@ -45,7 +37,7 @@ RUN --mount=type=secret,id=b2_access,required=true \ echo "No working virtual display; building without xvfb" >&2; \ fi; \ run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \ - ensure_bincraft() { "$1" -q -e 'source("/work/install-bincraft.R")'; }; \ + ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")'; }; \ PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ seen=" $PRIMARY_MINOR "; \ prc=0; failed=""; \ diff --git a/docker/buildkitd.toml b/docker/buildkitd.toml deleted file mode 100644 index ebbeb38..0000000 --- a/docker/buildkitd.toml +++ /dev/null @@ -1,41 +0,0 @@ -# BuildKit GC config for the remote buildx builders (artemis/amd64, gaia/arm64). -# -# Apply when creating the docker-container builders: -# docker buildx create --name artemis --driver docker-container \ -# --config docker/buildkitd.toml ssh:// -# docker buildx create --name gaia --driver docker-container \ -# --config docker/buildkitd.toml ssh:// -# -# Why: BuildKit's default GC caps the ephemeral cache tier — RUN -# --mount=type=cache mounts, local build context, git checkouts — at a -# hardcoded 512 MB (shown as "488.3 MiB" in `buildx inspect`). Across our -# 7-distro build matrix that fills instantly and forces re-downloads of -# system + R packages every rebuild. The first rule below raises that tier. -# -# Limits are absolute (not %) on purpose: artemis and gaia have very -# different free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB), so a -# percentage would mean wildly different real budgets. minFreeSpace = 20 GB -# keeps the tight Hetzner host safe while staying modest on the Mac mini. - -[worker.oci] - gc = true - - # Tier 1 — ephemeral caches (cache mounts, local context, git checkouts). - # Raised from the 512 MB default to 8 GB, retained for 7 days so weekly - # rebuilds reuse downloaded packages instead of re-fetching them. - [[worker.oci.gcpolicy]] - filters = [ - "type==source.local", - "type==exec.cachemount", - "type==source.git.checkout", - ] - keepDuration = "168h" - maxUsedSpace = "8GB" - - # Tier 2 — everything else (image layers, RUN exec results). Bounds the - # whole buildkit cache and always leaves 20 GB free on the host disk. - [[worker.oci.gcpolicy]] - all = true - reservedSpace = "2GB" - maxUsedSpace = "40GB" - minFreeSpace = "20GB" diff --git a/justfile b/justfile index a9a705e..2d9388d 100644 --- a/justfile +++ b/justfile @@ -10,10 +10,8 @@ # - buildx builders named `artemis` (amd64) and `gaia` (arm64), created with the # docker-container driver (runs BuildKit on the remote host's docker daemon over # SSH). The default `remote` driver does NOT work with an ssh:// docker host. -# Pass --config docker/buildkitd.toml so BuildKit's GC keeps a usable cache -# (the default caps the cache-mount tier at 512 MB, forcing re-downloads). -# docker buildx create --name artemis --driver docker-container --config docker/buildkitd.toml ssh:// -# docker buildx create --name gaia --driver docker-container --config docker/buildkitd.toml ssh:// +# docker buildx create --name artemis --driver docker-container ssh:// +# docker buildx create --name gaia --driver docker-container ssh:// # - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional) # # Overridable (env or `just VAR=… rebuild …`): diff --git a/local/build-all.R b/local/build-all.R index 1df1f83..56f1fbc 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -64,38 +64,12 @@ sprintf("# of package versions for this job: %s", nrow(chunk)) exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] chunk <- chunk[!chunk$Package %in% exclude, ] -# Skip package versions already attempted in a previous run (built or errored). +# Skip package versions already built in a previous run. # pkgs_to_build.rds is a static snapshot from the install-deps step, so on a # restart it still lists everything an interrupted run already produced. The # metadata DB reflects that progress, so we re-derive the remaining set here. -# We exclude *all* attempted versions, not just successful ones: a previously -# errored version is skipped by build_binary_package() anyway, so leaving it in -# the chunk only makes the job cycle through it one-by-one for no benefit. -# Derive platform + arch from the running container, mirroring the codename -> -# platform mapping bincraft uses internally. The OS/OS_VERSION selectors are -# workflow-level CI variables that are not injected into the container -# environment, so Sys.getenv() would return "" and this pre-filter would query -# platform "-" and skip nothing. -codename <- bincraft::set_codename(NULL) -platform <- switch( - codename, - jammy = "ubuntu-2204", - noble = "ubuntu-2404", - resolute = "ubuntu-2604", - rhel10 = "redhat-10", - rhel9 = "redhat-9", - rhel8 = "redhat-8", - alpine320 = "alpine-320", - alpine321 = "alpine-321", - alpine322 = "alpine-322", - alpine323 = "alpine-323", - alpine324 = "alpine-324", - alpine325 = "alpine-325", - alpine326 = "alpine-326", - NA_character_ -) -local_machine <- Sys.info()[["machine"]] -arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" +platform <- paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-") +arch <- Sys.getenv("ARCH") con <- DBI::dbConnect( RPostgres::Postgres(), dbname = "build_metadata", @@ -107,13 +81,13 @@ con <- DBI::dbConnect( ) built <- DBI::dbGetQuery( con, - "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2", + "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2 AND error_occurred = FALSE", params = list(platform, arch) ) DBI::dbDisconnect(con) before <- nrow(chunk) chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] -sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) +sprintf("Skipped %d already-built package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, @@ -147,7 +121,6 @@ mapply( metadata_db_sslmode = "require", metadata_db_port = 15432, archive = TRUE, - patches = "local/patches", upload = TRUE, store_build_metadata = TRUE ) diff --git a/local/build-one.R b/local/build-one.R index 07c0b6a..f2d6ebd 100644 --- a/local/build-one.R +++ b/local/build-one.R @@ -103,7 +103,6 @@ for (ver in versions) { force = TRUE, upload = TRUE, archive = TRUE, - patches = "local/patches", store_build_metadata = TRUE, s3_endpoint = s3$s3_endpoint, s3_region = s3$s3_region, diff --git a/local/install-bincraft.R b/local/install-bincraft.R deleted file mode 100644 index 4665b3b..0000000 --- a/local/install-bincraft.R +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env Rscript - -# Install the latest tagged bincraft release, resolved dynamically, so the CI -# workflows and the build-one image never pin a hardcoded version (no more -# editing `@vX.Y.Z` in many places on every release). -# -# Run with the R whose library should receive bincraft: -# Rscript local/install-bincraft.R -# or, to target a specific R from a shell loop: -# "$RBIN" -q -e 'source("local/install-bincraft.R")' -# -# How it works: list the remote tags with `git ls-remote` (no token needed for -# the public repo), keep the `vX.Y.Z` release tags, pick the highest version, -# and install it with pak. pak is idempotent on the git ref, so re-running keeps -# the package when it is already current and only updates when a newer tag ships. -# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is -# identical across git versions and `system2()` argument handling. - -repo_url <- Sys.getenv( - "BINCRAFT_GIT_URL", - unset = "https://codefloe.com/rpkgs/bincraft.git" -) - -# GIT_TERMINAL_PROMPT=0 keeps a non-interactive run from hanging on auth. -refs <- system2( - "git", - c("ls-remote", "--tags", repo_url), - stdout = TRUE, - stderr = FALSE, - env = "GIT_TERMINAL_PROMPT=0" -) -tags <- sub(".*refs/tags/", "", refs) -tags <- tags[!grepl("\\^\\{\\}$", tags)] # drop dereferenced "...^{}" lines -tags <- grep("^v[0-9]", tags, value = TRUE) # only vX.Y.Z release tags -if (length(tags) == 0L) { - stop( - "Could not resolve any bincraft release tag from ", - repo_url, - call. = FALSE - ) -} -latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][ - 1L -] - -message(sprintf("Installing latest bincraft release: %s", latest)) -pak::pak(sprintf("git::%s@%s", repo_url, latest)) -message(sprintf( - "bincraft %s installed (%s)", - as.character(utils::packageVersion("bincraft")), - latest -)) diff --git a/local/patches/README.md b/local/patches/README.md deleted file mode 100644 index af24b2b..0000000 --- a/local/patches/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Patch Registry - -This directory contains the curated registry of per-package build-time patches consumed by bincraft's `patches` argument. - -## Schema - -The registry is defined in `registry.json` as an array of patch entries. Each entry specifies lightweight build-time overrides (environment variables, configure arguments, Makevars) and optionally a source diff to apply before building. - -### Field semantics - -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| `package` | string | yes | CRAN package name. | -| `versions` | string | yes | `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. | -| `platforms` | array of strings | yes | Matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). An entry matches if any listed token matches any build token. `["*"]` matches all platforms. | -| `env` | object | no | Environment variables exported only for this package's isolated build. | -| `configure_args` | array | no | Arguments passed as `--configure-args` to the isolated build. | -| `makevars` | object | no | Key/value pairs written into a package-local Makevars for the isolated build. | -| `patch` | string or null | no | Path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. | -| `reason` | string | yes | Human explanation, surfaced in logs and metadata. | - -## Adding an entry - -To add a new patch entry: - -1. Add an object to the array in `registry.json` with the fields documented above. - Start with lightweight overrides (environment variables, configure arguments, Makevars) before resorting to source diffs. - -2. If a source diff is needed, place it in `local/patches//.patch` and reference its path in the `patch` field. - For example, a diff for `RcppParallel` would go in `local/patches/RcppParallel/fix.patch` and be referenced as `"patch": "RcppParallel/fix.patch"`. - -3. The `reason` field should clearly explain why the patch is needed and what problem it solves. - -## Validation - -The registry is validated and applied by bincraft during the build process. -For manual validation, run the validator from the repo root: - -```bash -Rscript local/validate-patches.R -``` - -This validates the schema, referenced patch-file existence, and checks for duplicate entries across platforms and versions. diff --git a/local/patches/RcppParallel/disable-tbb.patch b/local/patches/RcppParallel/disable-tbb.patch deleted file mode 100644 index 0fe8ad9..0000000 --- a/local/patches/RcppParallel/disable-tbb.patch +++ /dev/null @@ -1,16 +0,0 @@ -diff --git a/src/Makevars.in b/src/Makevars.in -index be8445f..faee771 100644 ---- a/src/Makevars.in -+++ b/src/Makevars.in -@@ -60,7 +60,10 @@ else - endif - - ifeq ($(UNAME), Linux) -- USE_TBB=Linux -+ # bincraft patch: the bundled Intel TBB build hangs/fails on musl (Alpine) -+ # and newer toolchains (g++ 15). Skip it (leave USE_TBB unset) and force the -+ # TinyThread backend so RcppParallel still builds. -+ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=0 - endif - - ifeq ($(UNAME), SunOS) diff --git a/local/patches/fs/force-vendored-libuv.patch b/local/patches/fs/force-vendored-libuv.patch deleted file mode 100644 index c69e8b9..0000000 --- a/local/patches/fs/force-vendored-libuv.patch +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/configure b/configure ---- a/configure -+++ b/configure -@@ -11,6 +11,15 @@ - PKG_TEST_HEADER="" - PKG_LIBS="-luv" - -+# bincraft patch: force the vendored static libuv so the resulting binary -+# is self-contained. fs configure otherwise links system libuv whenever -+# pkg-config finds libuv-devel (installed as a build-time sysreq), yielding -+# an fs.so with NEEDED libuv.so.1 that fails to dyn.load on machines lacking -+# runtime libuv (install.packages/renv do not install SystemRequirements). -+echo "Building static libuv (bincraft: forced vendored)" 1>&2 -+cp -f src/Makevars.vendor src/Makevars -+exit 0 -+ - # Use pkg-config if available - if [ `command -v pkg-config` ]; then - PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` diff --git a/local/patches/registry.json b/local/patches/registry.json deleted file mode 100644 index 7a197d7..0000000 --- a/local/patches/registry.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "package": "RcppParallel", - "versions": "*", - "platforms": ["alpine", "ubuntu-2604"], - "env": {}, - "configure_args": [], - "makevars": {}, - "patch": "RcppParallel/disable-tbb.patch", - "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); patch unsets USE_TBB and forces -DRCPP_PARALLEL_USE_TBB=0 so RcppParallel skips the bundled build and uses the TinyThread backend" - }, - { - "package": "fs", - "versions": "*", - "platforms": ["*"], - "env": {}, - "configure_args": [], - "makevars": {}, - "patch": "fs/force-vendored-libuv.patch", - "reason": "fs 2.x configure links system libuv whenever pkg-config finds libuv-devel (installed as a build-time sysreq), producing an fs.so with NEEDED libuv.so.1. That binary fails to dyn.load on consumer machines lacking runtime libuv, because install.packages/renv do not install SystemRequirements (only pak does, and only in the build container). The patch short-circuits configure to copy src/Makevars.vendor and build the bundled static libuv (needs cmake) so the binary is self-contained on every platform. An env/pkg-config override was tried first but the rebuilt binary still linked libuv.so.1, so a source patch is used instead." - } -] diff --git a/local/validate-patches.R b/local/validate-patches.R deleted file mode 100644 index 5f5aace..0000000 --- a/local/validate-patches.R +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env Rscript -# Validate local/patches/registry.json: schema, referenced patch files, and -# ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. - -dir <- "local/patches" -registry_file <- file.path(dir, "registry.json") -if (!file.exists(registry_file)) { - cat("No registry.json found; nothing to validate.\n") - quit(status = 0L) -} - -or_q <- function(x) if (is.null(x)) "?" else x - -reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) -required <- c("package", "versions", "platforms", "reason") -errs <- character(0L) - -for (i in seq_along(reg)) { - e <- reg[[i]] - missing <- setdiff(required, names(e)) - if (length(missing) > 0L) { - errs <- c(errs, sprintf( - "entry %d (%s): missing %s", i, - if (is.null(e$package)) "?" else e$package, toString(missing) - )) - } - if (!is.null(e$patch)) { - p <- file.path(dir, e$patch) - if (!file.exists(p)) { - errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing", - i, e$package, p)) - } - } -} - -# Ambiguous overlap: two entries for the same package with identical platforms -# and versions. -keys <- vapply(reg, function(e) { - sprintf( - "%s|%s|%s", - or_q(e$package), - paste(sort(as.character(unlist(e$platforms))), collapse = ","), - or_q(e$versions) - ) -}, character(1L)) -dups <- keys[duplicated(keys)] -if (length(dups) > 0L) { - errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups)))) -} - -if (length(errs) > 0L) { - cat("Patch registry validation FAILED:\n") - cat(paste0(" - ", errs, "\n")) - quit(status = 1L) -} -cat(sprintf("Patch registry OK (%d %s).\n", length(reg), if (length(reg) == 1L) "entry" else "entries")) diff --git a/plans/2026-06-30-package-patching-implementation.md b/plans/2026-06-30-package-patching-implementation.md deleted file mode 100644 index 3a743d7..0000000 --- a/plans/2026-06-30-package-patching-implementation.md +++ /dev/null @@ -1,1477 +0,0 @@ -# Package Patching Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Let a curated registry of packages be "patched" (env/configure/Makevars overrides and/or source diffs) before `pak` installs them, including when they are transitive dependencies, so compiler-/OS-specific failures like `RcppParallel` stop cascading. - -**Architecture:** The mechanism lives in `bincraft`. Before `pak::local_install_deps()` runs, bincraft pre-builds each registry-matched package as a binary (reusing `pkgbuild::build(binary=TRUE)` + `cranlike::add_PACKAGES`), caches it, and serves it from a local `file://` repo prepended to `options("repos")` so `pak` installs the patched binary — direct or transitive — without recompiling. The curated registry (`registry.json` + diff files) lives in this repo and is passed in via a new `patches` argument. - -**Tech Stack:** R, `pak`, `pkgbuild`, `cranlike` (fork `pat-s/cranlike@s3`), `pkgsearch`, `withr`, `jsonlite`, `testthat` (3e), `mockery`. Two repos: `bincraft` at `/Users/pjs/git/codefloe.com/rpkgs/bincraftr`, and this repo `build-cran-binaries`. - -## Global Constraints - -- bincraft package name is `bincraft`; repo dir is `…/rpkgs/bincraftr`. Current version `4.2.2.9999`. -- Logging uses bincraft's wrappers only: `log_info`, `log_warn`, `log_success`, `log_debug`, `log_error`, `log_header` (never bare `message`/`cat` in package code). -- `cli`-style inline markup is allowed in log messages (e.g. `{.pkg %s}`, `{.path %s}`); curly braces in dynamic/error text must be escaped as already done in `install_helpers.R`. -- Tests: `testthat` 3rd edition, files at `tests/testthat/test-.R`, network/build tests guarded with `skip_on_cran()` / `skip_if_offline()`; end-to-end build tests guarded behind `skip_if_not(nzchar(Sys.getenv("BINCRAFT_PATCH_E2E")))`. -- New exported functions need roxygen with `@keywords internal` for non-user helpers; run `devtools::document()` after adding roxygen. -- Patched-binary cache dir default: `/mnt/cache/patched-binaries`. -- Platform tokens for matching = `c(, , )`, e.g. `ubuntu-2604` → `c("ubuntu-2604","ubuntu","amd64")`. -- Registry entry required fields: `package`, `versions`, `platforms`, `reason`. Optional: `env`, `configure_args`, `makevars`, `patch`. -- One sentence per line in prose/commit messages; do not hard-wrap at 80 columns. -- Use `fj -H codefloe.com` for any PR operations (Forgejo), not `gh`. - ---- - -## Phase A — bincraft mechanism - -All Phase A paths are relative to `/Users/pjs/git/codefloe.com/rpkgs/bincraftr`. - -### Task A0: Proof of mechanism — pak installs a patched binary from a prepended `file://` repo - -This de-risks the core assumption before building anything on top: that `pak` installs a binary from a local `file://` repo in preference to CRAN for an equal version, and does so without recompiling. If this fails, the contingency (documented in Step 4) is to serve patched *source* and rely on `pkgcache` build-caching — the rest of the plan changes only inside `build_patched_binary()`. - -**Files:** -- Create: `tools/verify-patch-mechanism.R` - -**Interfaces:** -- Produces: a runnable script proving `pak::pkg_install()` resolves a local patched binary over CRAN. No package API. - -- [ ] **Step 1: Write the verification script** - -```r -# tools/verify-patch-mechanism.R -# Proves pak installs a patched binary from a prepended file:// repo instead of -# CRAN's, without recompiling. Run inside a Linux build-env container: -# Rscript tools/verify-patch-mechanism.R -# Exits 0 on success, 1 on failure. - -pkg <- "glue" # small, pure-R CRAN package -sentinel <- "PatchMechanismProof" - -work <- tempfile("verify_") -repo <- file.path(work, "repo", "src", "contrib") -lib <- file.path(work, "lib") -dir.create(repo, recursive = TRUE) -dir.create(lib, recursive = TRUE) - -# 1. Download CRAN source for the current version. -ap <- available.packages(repos = "https://cloud.r-project.org") -ver <- ap[pkg, "Version"] -src <- file.path(work, sprintf("%s_%s.tar.gz", pkg, ver)) -download.file( - sprintf("https://cloud.r-project.org/src/contrib/%s_%s.tar.gz", pkg, ver), - src, mode = "wb" -) - -# 2. Unpack, inject a sentinel field into DESCRIPTION, build a binary. -untar(src, exdir = work) -desc <- file.path(work, pkg, "DESCRIPTION") -writeLines(c(readLines(desc), sprintf("%s: yes", sentinel)), desc) -pkgbuild::build( - file.path(work, pkg), binary = TRUE, vignettes = FALSE, - dest_path = repo, quiet = TRUE -) -built <- list.files(repo, pattern = sprintf("^%s_.*\\.tar\\.gz$", pkg), full.names = TRUE) -file.rename(built[1L], file.path(repo, sprintf("%s_%s.tar.gz", pkg, ver))) -cranlike::add_PACKAGES(sprintf("%s_%s.tar.gz", pkg, ver), repo) - -# 3. Install with the local repo prepended; assert our patched build won. -withr::with_options( - list(repos = c(patched = sprintf("file://%s", dirname(dirname(repo))), - CRAN = "https://cloud.r-project.org")), - pak::pkg_install(pkg, lib = lib, ask = FALSE, upgrade = FALSE) -) - -installed_desc <- file.path(lib, pkg, "DESCRIPTION") -ok <- file.exists(installed_desc) && - any(grepl(sentinel, readLines(installed_desc))) - -if (ok) { - cat("PROOF PASSED: pak installed the patched local binary.\n") - quit(status = 0L) -} else { - cat("PROOF FAILED: pak did not install the patched local binary.\n") - quit(status = 1L) -} -``` - -- [ ] **Step 2: Run the proof in a build-env container** - -Run (amd64 example; use any supported build-env image): - -```bash -just build-single ubuntu 2604 amd64 4.5.0 glue 1.0.0 1 || true # warms the env -docker run --rm -v "$PWD":/work -w /work reg.devxy.io/rpkgs/build-env-ubuntu:2604 \ - Rscript tools/verify-patch-mechanism.R -``` - -Expected: final line `PROOF PASSED: pak installed the patched local binary.` and exit status 0. - -- [ ] **Step 3: Commit** - -```bash -git add tools/verify-patch-mechanism.R -git commit -m "test(patches): prove pak installs a patched binary from a local file:// repo" -``` - -- [ ] **Step 4: Record the outcome / contingency** - -If the proof PASSED, proceed to Task A1 unchanged. -If it FAILED (pak recompiled or picked CRAN's), the mechanism switches to serving patched *source*: in Task A4 `build_patched_binary()` skips `pkgbuild::build()` and instead repackages the patched source tree with `pkgbuild::build(binary = FALSE)`; everything else (registry, matching, cache, repo prepend) is unchanged because `pak` build-caches the compiled result via `pkgcache`. Note the chosen path in the commit message and continue. - ---- - -### Task A1: Registry loading and normalization - -**Files:** -- Create: `R/patches.R` -- Test: `tests/testthat/test-patches.R` - -**Interfaces:** -- Produces: `load_patch_registry(patches_dir)` → `list()` of normalized entries; each entry is a named list with `package`, `versions`, `platforms` (character vector), `env` (named list), `configure_args` (character), `makevars` (named list), `reason`, and `patch_path` (absolute path or `NULL`). `normalize_patch_entry(entry, patches_dir)` → one normalized entry; errors on missing required field or missing patch file. - -- [ ] **Step 1: Write the failing test** - -```r -# tests/testthat/test-patches.R -test_that("load_patch_registry parses and normalizes entries", { - dir <- withr::local_tempdir() - writeLines("--- a patch ---", file.path(dir, "fix.patch")) - jsonlite::write_json( - list(list( - package = "RcppParallel", versions = "*", - platforms = list("alpine", "ubuntu-2604"), - env = list(RCPP_PARALLEL_USE_TBB = "0"), - patch = "fix.patch", reason = "bundled TBB fails" - )), - file.path(dir, "registry.json"), auto_unbox = TRUE - ) - - reg <- load_patch_registry(dir) - - expect_length(reg, 1L) - expect_identical(reg[[1L]]$package, "RcppParallel") - expect_identical(reg[[1L]]$platforms, c("alpine", "ubuntu-2604")) - expect_identical(reg[[1L]]$env$RCPP_PARALLEL_USE_TBB, "0") - expect_identical(reg[[1L]]$configure_args, character(0L)) - expect_true(file.exists(reg[[1L]]$patch_path)) -}) - -test_that("load_patch_registry returns empty list when no registry", { - expect_identical(load_patch_registry(NULL), list()) - expect_identical(load_patch_registry(withr::local_tempdir()), list()) -}) - -test_that("normalize_patch_entry errors on missing required field", { - expect_error( - normalize_patch_entry(list(package = "x"), tempdir()), - "missing required field" - ) -}) - -test_that("normalize_patch_entry errors on missing patch file", { - expect_error( - normalize_patch_entry( - list(package = "x", versions = "*", platforms = "alpine", - reason = "r", patch = "nope.patch"), - tempdir() - ), - "does not exist" - ) -}) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: FAIL with "could not find function load_patch_registry". - -- [ ] **Step 3: Write minimal implementation** - -```r -# R/patches.R - -#' Load and validate the patch registry -#' -#' Reads `registry.json` from `patches_dir` and returns normalized entries. -#' -#' @param patches_dir Directory containing `registry.json` and any patch files, -#' or `NULL` to disable patching. -#' @return A list of normalized patch entries (possibly empty). -#' @keywords internal -load_patch_registry <- function(patches_dir) { - if (is.null(patches_dir)) { - return(list()) - } - registry_file <- file.path(patches_dir, "registry.json") - if (!file.exists(registry_file)) { - log_warn(sprintf( - "Patch directory {.path %s} has no registry.json; patching disabled.", - patches_dir - )) - return(list()) - } - raw <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) - lapply(raw, normalize_patch_entry, patches_dir = patches_dir) -} - -#' Normalize and validate a single patch registry entry -#' -#' @param entry A list parsed from `registry.json`. -#' @param patches_dir Directory used to resolve a relative `patch` path. -#' @return The entry with defaults filled and `patch_path` resolved. -#' @keywords internal -normalize_patch_entry <- function(entry, patches_dir) { - required <- c("package", "versions", "platforms", "reason") - missing <- setdiff(required, names(entry)) - if (length(missing) > 0L) { - stop( - sprintf("Patch entry is missing required field(s): %s", toString(missing)), - call. = FALSE - ) - } - entry$platforms <- as.character(unlist(entry$platforms)) - entry$env <- if (is.null(entry$env)) list() else entry$env - entry$configure_args <- if (is.null(entry$configure_args)) { - character(0L) - } else { - as.character(unlist(entry$configure_args)) - } - entry$makevars <- if (is.null(entry$makevars)) list() else entry$makevars - if (!is.null(entry$patch)) { - patch_path <- file.path(patches_dir, entry$patch) - if (!file.exists(patch_path)) { - stop( - sprintf( - "Patch file '%s' for package '%s' does not exist.", - patch_path, entry$package - ), - call. = FALSE - ) - } - entry$patch_path <- patch_path - } else { - entry$patch_path <- NULL - } - entry -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: PASS (4 tests). - -- [ ] **Step 5: Commit** - -```bash -git add R/patches.R tests/testthat/test-patches.R -git commit -m "feat(patches): load and validate the patch registry" -``` - ---- - -### Task A2: Platform matching and version-constraint satisfaction - -**Files:** -- Modify: `R/patches.R` -- Test: `tests/testthat/test-patches.R` - -**Interfaces:** -- Consumes: normalized entries from Task A1. -- Produces: `build_platform_tokens(platform, arch)` → character vector; `entry_matches_platform(entry, tokens)` → logical; `match_patch_entries(registry, platform, arch)` → filtered list; `version_satisfies(version, constraint)` → logical (constraint forms: `"*"` handled by caller, `"x.y.z"` exact, `">=x"`, `"<=x"`, `">x"`, `"=5.1.0")) - expect_false(version_satisfies("5.0.0", ">=5.1.0")) - expect_true(version_satisfies("5.1.11-2", "<=5.1.11-2")) - expect_false(version_satisfies("5.1.12", "<=5.1.11-2")) -}) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: FAIL with "could not find function build_platform_tokens". - -- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** - -```r -#' Build platform tokens for patch matching -#' @keywords internal -build_platform_tokens <- function(platform, arch) { - family <- sub("-.*$", "", platform) - unique(c(platform, family, arch)) -} - -#' Does a patch entry apply to the current platform tokens? -#' @keywords internal -entry_matches_platform <- function(entry, tokens) { - any(entry$platforms == "*") || - length(intersect(entry$platforms, tokens)) > 0L -} - -#' Filter registry entries applicable to the current build -#' @keywords internal -match_patch_entries <- function(registry, platform, arch) { - if (length(registry) == 0L) { - return(list()) - } - tokens <- build_platform_tokens(platform, arch) - Filter(function(e) entry_matches_platform(e, tokens), registry) -} - -#' Test whether a version satisfies a single constraint -#' -#' @param version A version string (CRAN style, may contain `-`). -#' @param constraint One of `"x.y.z"`, `"==x"`, `">=x"`, `"<=x"`, `">x"`, `"=|<=|==|>|<)?\\s*(.+)$", constraint) - )[[1L]] - op <- parts[2L] - target <- parts[3L] - v <- package_version(version) - t <- package_version(target) - if (op == "" || op == "==") { - return(v == t) - } - switch( - op, - ">=" = v >= t, - "<=" = v <= t, - ">" = v > t, - "<" = v < t, - FALSE - ) -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add R/patches.R tests/testthat/test-patches.R -git commit -m "feat(patches): platform matching and version-constraint checks" -``` - ---- - -### Task A3: Cache key and version resolution - -**Files:** -- Modify: `R/patches.R` -- Test: `tests/testthat/test-patches.R` - -**Interfaces:** -- Consumes: normalized entries. -- Produces: `patch_cache_key(entry, version, platform, arch, r_minor)` → string `"_____"`, where `hash12` covers `env`/`configure_args`/`makevars`/patch bytes; `resolve_patch_version(entry)` → latest CRAN version satisfying `entry$versions`, or `NA_character_`; `describe_patch(entry)` → short human label. - -- [ ] **Step 1: Write the failing test** - -```r -test_that("patch_cache_key is stable and sensitive to env/patch changes", { - e1 <- list(package = "P", env = list(A = "1"), - configure_args = character(0L), makevars = list(), - patch_path = NULL) - e2 <- e1; e2$env <- list(A = "2") - - k1 <- patch_cache_key(e1, "1.0", "alpine-324", "amd64", "4.5") - expect_identical(k1, patch_cache_key(e1, "1.0", "alpine-324", "amd64", "4.5")) - expect_false(identical( - k1, patch_cache_key(e2, "1.0", "alpine-324", "amd64", "4.5") - )) - expect_match(k1, "^P_1.0_alpine-324_amd64_4.5_[0-9a-f]{12}$") -}) - -test_that("resolve_patch_version returns latest for wildcard, NA when unmet", { - local_mocked_bindings( - cran_package = function(pkg) list(Version = "5.1.12"), - .package = "pkgsearch" - ) - expect_identical( - resolve_patch_version(list(package = "RcppParallel", versions = "*")), - "5.1.12" - ) - expect_identical( - resolve_patch_version(list(package = "RcppParallel", versions = ">=9.0")), - NA_character_ - ) -}) - -test_that("describe_patch summarizes the active overrides", { - expect_match( - describe_patch(list(env = list(RCPP_PARALLEL_USE_TBB = "0"), - configure_args = character(0L), makevars = list(), - patch_path = NULL)), - "env: RCPP_PARALLEL_USE_TBB=0" - ) - expect_match( - describe_patch(list(env = list(), configure_args = character(0L), - makevars = list(), patch_path = "/x/fix.patch")), - "source patch" - ) -}) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: FAIL with "could not find function patch_cache_key". - -- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** - -```r -#' Compute the cache key for a patched binary -#' @keywords internal -patch_cache_key <- function(entry, version, platform, arch, r_minor) { - payload <- list( - env = entry$env, - configure_args = entry$configure_args, - makevars = entry$makevars, - patch = if (!is.null(entry$patch_path)) { - readBin(entry$patch_path, "raw", file.size(entry$patch_path)) - } else { - raw(0L) - } - ) - tmp <- tempfile() - on.exit(unlink(tmp), add = TRUE) - saveRDS(payload, tmp) - hash <- substr(unname(tools::md5sum(tmp)), 1L, 12L) - sprintf( - "%s_%s_%s_%s_%s_%s", - entry$package, version, platform, arch, r_minor, hash - ) -} - -#' Resolve the CRAN version to build for a patch entry -#' -#' Returns the latest CRAN version satisfying the entry's `versions` constraint, -#' or `NA_character_` when CRAN's latest does not satisfy it or lookup fails. -#' @keywords internal -resolve_patch_version <- function(entry) { - latest <- tryCatch( - pkgsearch::cran_package(entry$package)$Version, - error = function(e) NA_character_ - ) - if (is.na(latest)) { - return(NA_character_) - } - if (identical(entry$versions, "*") || version_satisfies(latest, entry$versions)) { - return(latest) - } - NA_character_ -} - -#' Short human label describing a patch entry's overrides -#' @keywords internal -describe_patch <- function(entry) { - bits <- character(0L) - if (length(entry$env) > 0L) { - bits <- c(bits, sprintf( - "env: %s", - paste( - names(entry$env), - unlist(entry$env), - sep = "=", collapse = "," - ) - )) - } - if (length(entry$configure_args) > 0L) { - bits <- c(bits, sprintf("configure: %s", toString(entry$configure_args))) - } - if (length(entry$makevars) > 0L) { - bits <- c(bits, "makevars") - } - if (!is.null(entry$patch_path)) { - bits <- c(bits, "source patch") - } - if (length(bits) == 0L) "no-op" else paste(bits, collapse = "; ") -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: PASS. (Note: `local_mocked_bindings` requires testthat >= 3.1.7; bincraft uses 3e.) - -- [ ] **Step 5: Commit** - -```bash -git add R/patches.R tests/testthat/test-patches.R -git commit -m "feat(patches): cache key, version resolution, and patch description" -``` - ---- - -### Task A4: Build a patched binary in isolation - -**Files:** -- Modify: `R/patches.R` -- Test: `tests/testthat/test-patches.R` - -**Interfaces:** -- Consumes: a normalized entry, a resolved `version`, a `dest_dir`. -- Produces: `download_cran_source(package, version, dest_dir, cran)` → path or `NULL`; `apply_source_patch(patch_path, pkg_src)` → logical; `configure_args_to_build_args(configure_args)` → character; `build_patched_binary(entry, version, dest_dir)` → path to built binary tarball or `NULL`. - -- [ ] **Step 1: Write the failing test** - -```r -test_that("configure_args_to_build_args formats configure args", { - expect_identical(configure_args_to_build_args(character(0L)), character(0L)) - expect_identical( - configure_args_to_build_args(c("--with-foo", "--no-bar")), - "--configure-args=--with-foo --no-bar" - ) -}) - -test_that("apply_source_patch returns FALSE when patch does not apply", { - src <- withr::local_tempdir() - writeLines("unrelated content", file.path(src, "file.txt")) - bad_patch <- tempfile(fileext = ".patch") - writeLines(c( - "--- a/missing.txt", "+++ b/missing.txt", - "@@ -1 +1 @@", "-nope", "+nope2" - ), bad_patch) - expect_false(apply_source_patch(bad_patch, src)) -}) - -test_that("build_patched_binary returns NULL when download fails", { - local_mocked_bindings(download_cran_source = function(...) NULL) - expect_null( - build_patched_binary( - list(package = "P", env = list(), configure_args = character(0L), - makevars = list(), patch_path = NULL), - "1.0", withr::local_tempdir() - ) - ) -}) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: FAIL with "could not find function configure_args_to_build_args". - -- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** - -```r -#' Download a CRAN source tarball for an exact version -#' @keywords internal -download_cran_source <- function( - package, - version, - dest_dir, - cran = "https://cloud.r-project.org" -) { - fname <- sprintf("%s_%s.tar.gz", package, version) - urls <- c( - sprintf("%s/src/contrib/%s", cran, fname), - sprintf("%s/src/contrib/Archive/%s/%s", cran, package, fname) - ) - dest <- file.path(dest_dir, fname) - for (u in urls) { - ok <- tryCatch( - { - utils::download.file(u, dest, mode = "wb", quiet = TRUE) - file.exists(dest) && file.size(dest) > 0L - }, - error = function(e) FALSE - ) - if (isTRUE(ok)) { - return(dest) - } - } - log_warn(sprintf( - "Could not download CRAN source for {.pkg %s} %s.", - package, version - )) - NULL -} - -#' Apply a unified diff to an unpacked source tree -#' -#' Uses `patch -p1 --forward` so an already-applied or non-applying patch fails -#' cleanly (returns FALSE) instead of corrupting the tree. -#' @keywords internal -apply_source_patch <- function(patch_path, pkg_src) { - status <- system2( - "patch", - args = c( - "-p1", "--forward", "--batch", - "-d", shQuote(pkg_src), - "-i", shQuote(patch_path) - ), - stdout = FALSE, stderr = FALSE - ) - identical(status, 0L) -} - -#' Format configure args for `pkgbuild::build(args = ...)` -#' @keywords internal -configure_args_to_build_args <- function(configure_args) { - if (length(configure_args) == 0L) { - return(character(0L)) - } - sprintf("--configure-args=%s", paste(configure_args, collapse = " ")) -} - -#' Build a patched binary for one registry entry, in isolation -#' -#' Downloads CRAN source for `version`, applies the source patch (if any), and -#' builds a binary with the entry's env / configure / Makevars overrides scoped -#' to this build only. Returns the built tarball path, or `NULL` on any failure. -#' @keywords internal -build_patched_binary <- function(entry, version, dest_dir) { - workdir <- tempfile("patch_build_") - dir.create(workdir, recursive = TRUE, showWarnings = FALSE) - on.exit(unlink(workdir, recursive = TRUE, force = TRUE), add = TRUE) - - src_tarball <- download_cran_source(entry$package, version, workdir) - if (is.null(src_tarball)) { - return(NULL) - } - - utils::untar(src_tarball, exdir = workdir) - pkg_src <- file.path(workdir, entry$package) - - if (!is.null(entry$patch_path)) { - if (!apply_source_patch(entry$patch_path, pkg_src)) { - log_warn(sprintf( - "Patch for {.pkg %s} %s did not apply cleanly; skipping patched build.", - entry$package, version - )) - return(NULL) - } - } - - build_env <- entry$env - if (length(entry$makevars) > 0L) { - mk <- tempfile(fileext = ".mk") - writeLines( - vapply( - names(entry$makevars), - function(k) sprintf("%s=%s", k, entry$makevars[[k]]), - character(1L) - ), - mk - ) - build_env$R_MAKEVARS_USER <- mk - } - - tryCatch( - withr::with_envvar(build_env, { - pkgbuild::build( - path = pkg_src, - binary = TRUE, - vignettes = FALSE, - dest_path = dest_dir, - args = configure_args_to_build_args(entry$configure_args), - quiet = TRUE - ) - }), - error = function(e) { - log_warn(sprintf( - "Isolated patched build of {.pkg %s} %s failed: %s", - entry$package, version, conditionMessage(e) - )) - NULL - } - ) -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add R/patches.R tests/testthat/test-patches.R -git commit -m "feat(patches): build a patched binary in isolation from CRAN source" -``` - ---- - -### Task A5: Orchestrate the local patched repo (cache + index) - -**Files:** -- Modify: `R/patches.R` -- Test: `tests/testthat/test-patches.R` - -**Interfaces:** -- Consumes: all helpers above. -- Produces: `prepare_patched_repo(patches_dir, platform, arch, r_minor, cache_dir, repo_dir)` → path to a `src/contrib`-style dir containing patched binaries + a `PACKAGES` index, or `NULL` when nothing matched/built. On a cache hit it copies the cached tarball into `repo_dir`; on a miss it builds, then writes the result into `cache_dir`. - -- [ ] **Step 1: Write the failing test** - -```r -test_that("prepare_patched_repo returns NULL when no entries match", { - dir <- withr::local_tempdir() - jsonlite::write_json( - list(list(package = "A", versions = "*", platforms = list("redhat"), - reason = "r")), - file.path(dir, "registry.json"), auto_unbox = TRUE - ) - expect_null( - prepare_patched_repo(dir, "ubuntu-2604", "amd64", "4.5", - cache_dir = withr::local_tempdir(), - repo_dir = withr::local_tempdir()) - ) -}) - -test_that("prepare_patched_repo serves a cached binary and writes an index", { - dir <- withr::local_tempdir() - jsonlite::write_json( - list(list(package = "glue", versions = "*", platforms = list("*"), - env = list(A = "1"), reason = "r")), - file.path(dir, "registry.json"), auto_unbox = TRUE - ) - cache <- withr::local_tempdir() - repo <- withr::local_tempdir() - - local_mocked_bindings( - resolve_patch_version = function(entry) "1.0.0", - build_patched_binary = function(entry, version, dest_dir) { - f <- file.path(dest_dir, sprintf("%s_%s.tar.gz", entry$package, version)) - writeLines("fake binary", f) - f - } - ) - - out <- prepare_patched_repo(dir, "ubuntu-2604", "amd64", "4.5", - cache_dir = cache, repo_dir = repo) - - expect_identical(out, repo) - expect_true(file.exists(file.path(repo, "glue_1.0.0.tar.gz"))) - expect_true(file.exists(file.path(repo, "PACKAGES"))) - # The build result was cached under the key. - expect_length(list.files(cache, pattern = "^glue_1.0.0_.*\\.tar\\.gz$"), 1L) -}) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: FAIL with "could not find function prepare_patched_repo". - -- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** - -```r -#' Prepare a local repo of patched binaries for the current build -#' -#' For each registry entry matching the current platform, ensures a patched -#' binary is present in `repo_dir` (from `cache_dir` if available, else built -#' and then cached) and writes a `PACKAGES` index over them. -#' -#' @param patches_dir Directory with `registry.json`, or `NULL`. -#' @param platform Build platform, e.g. `"ubuntu-2604"`. -#' @param arch Build arch, e.g. `"amd64"`. -#' @param r_minor R `"major.minor"` string, e.g. `"4.5"`. -#' @param cache_dir Persistent cache for patched binaries. -#' @param repo_dir Directory to assemble the local repo in. -#' @return `repo_dir` if at least one patched binary was produced, else `NULL`. -#' @keywords internal -prepare_patched_repo <- function( - patches_dir, - platform, - arch, - r_minor, - cache_dir = file.path("/mnt", "cache", "patched-binaries"), - repo_dir = tempfile("patched_repo_") -) { - entries <- match_patch_entries( - load_patch_registry(patches_dir), platform, arch - ) - if (length(entries) == 0L) { - return(NULL) - } - - dir.create(repo_dir, recursive = TRUE, showWarnings = FALSE) - dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE) - - produced <- 0L - for (entry in entries) { - version <- resolve_patch_version(entry) - if (is.na(version)) { - log_warn(sprintf( - "No CRAN version of {.pkg %s} satisfies '%s'; patch skipped.", - entry$package, entry$versions - )) - next - } - - key <- patch_cache_key(entry, version, platform, arch, r_minor) - cached <- file.path(cache_dir, sprintf("%s.tar.gz", key)) - target <- file.path( - repo_dir, sprintf("%s_%s.tar.gz", entry$package, version) - ) - - if (file.exists(cached)) { - log_info(sprintf( - "Using cached patched binary for {.pkg %s} %s.", - entry$package, version - )) - file.copy(cached, target, overwrite = TRUE) - } else { - log_info(sprintf( - "Applying patch to {.pkg %s} %s [%s]: %s", - entry$package, version, describe_patch(entry), entry$reason - )) - built <- build_patched_binary(entry, version, repo_dir) - if (is.null(built)) { - next - } - if (!identical(normalizePath(built), normalizePath(target))) { - file.copy(built, target, overwrite = TRUE) - } - file.copy(target, cached, overwrite = TRUE) - } - produced <- produced + 1L - } - - if (produced == 0L) { - return(NULL) - } - cranlike::add_PACKAGES( - list.files(repo_dir, pattern = "\\.tar\\.gz$"), - repo_dir - ) - repo_dir -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: PASS. - -- [ ] **Step 5: Update imports and document** - -Add to `DESCRIPTION` `Imports:` (alphabetical) `jsonlite` if not present, and ensure `pkgsearch`, `pkgbuild`, `cranlike`, `withr` are listed (they are). Then: - -```bash -Rscript -e 'devtools::document()' -git add R/patches.R tests/testthat/test-patches.R DESCRIPTION NAMESPACE -git commit -m "feat(patches): orchestrate local patched-binary repo with caching" -``` - ---- - -### Task A6: Wire patched repo into the pak install path - -**Files:** -- Modify: `R/install_helpers.R:333-389` (`run_pak_install_with_mutex`) -- Modify: `R/install-deps.R:23-75` (`install_pkg_sys_deps`) -- Test: `tests/testthat/test-patches.R` - -**Interfaces:** -- Consumes: `prepare_patched_repo()`. -- Produces: `run_pak_install_with_mutex(local_clone_dir_single, env_vars, patched_repo = NULL)` — prepends `file://` to `options("repos")` for the install; `install_pkg_sys_deps(package_name, tag, local_clone_dir, platform, aggressive_cleanup = FALSE, patches = NULL, arch = NULL)` — builds the patched repo before installing. - -- [ ] **Step 1: Write the failing test** - -```r -test_that("run_pak_install_with_mutex prepends the patched repo to repos", { - seen <- NULL - local_mocked_bindings( - acquire_pak_mutex = function(...) tempfile(), - release_pak_mutex = function(...) invisible(NULL), - retry_with_backoff = function(func, ...) func() - ) - local_mocked_bindings( - local_install_deps = function(...) { - seen <<- getOption("repos") - invisible(TRUE) - }, - .package = "pak" - ) - - run_pak_install_with_mutex( - tempfile(), list(), patched_repo = "/tmp/patched" - ) - - expect_true(any(grepl("file:///tmp/patched", seen))) -}) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: FAIL — `patched_repo` argument not yet accepted / repos not prepended. - -- [ ] **Step 3: Edit `run_pak_install_with_mutex` in `R/install_helpers.R`** - -Change the signature line: - -```r -run_pak_install_with_mutex <- function(local_clone_dir_single, env_vars) { -``` - -to: - -```r -run_pak_install_with_mutex <- function( - local_clone_dir_single, - env_vars, - patched_repo = NULL -) { -``` - -Replace the inner `retry_with_backoff(...)` block (the one wrapping `pak::local_install_deps`) with: - -```r - retry_with_backoff(function() { - withr::with_envvar(env_vars, { - repos <- getOption("repos") - if (!is.null(patched_repo)) { - repos <- c( - patched = sprintf("file://%s", patched_repo), - repos - ) - } - withr::with_options(list(repos = repos), { - # Default to non-verbose (suppressed messages) - suppressMessages(pak::local_install_deps(sprintf( - "%s", - local_clone_dir_single - ))) - }) - }) - }) -``` - -- [ ] **Step 4: Edit `install_pkg_sys_deps` in `R/install-deps.R`** - -Change the signature to add `patches` and `arch`: - -```r -install_pkg_sys_deps <- function( - package_name, - tag, - local_clone_dir, - platform = platform, - aggressive_cleanup = FALSE, - patches = NULL, - arch = NULL -) { -``` - -Immediately before the `run_pak_install_with_mutex(...)` call, insert: - -```r - # Build a local repo of patched binaries (if any apply) and serve it to pak. - r_minor <- paste( - R.version$major, - strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], - sep = "." - ) - patched_repo <- tryCatch( - prepare_patched_repo(patches, platform, arch, r_minor), - error = function(e) { - log_warn(sprintf("Patch preparation failed: %s", conditionMessage(e))) - NULL - } - ) -``` - -and change the call from: - -```r - run_pak_install_with_mutex( - local_clone_dir_single, - env_vars - ) -``` - -to: - -```r - run_pak_install_with_mutex( - local_clone_dir_single, - env_vars, - patched_repo = patched_repo - ) -``` - -- [ ] **Step 5: Run test to verify it passes** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add R/install_helpers.R R/install-deps.R tests/testthat/test-patches.R -git commit -m "feat(patches): serve patched binaries to pak during dep install" -``` - ---- - -### Task A7: Thread `patches` through the public build API - -**Files:** -- Modify: `R/build_binaries.R` (`build_binary_package`, `execute_package_builds`, `build_single_tag`, `handle_system_dependencies`) -- Create: `man-roxygen/param-patches.R` -- Test: `tests/testthat/test-patches.R` - -**Interfaces:** -- Produces: `build_binary_package(..., patches = NULL)` and the internal chain each carry `patches` down to `install_pkg_sys_deps()`. `handle_system_dependencies(..., patches = NULL)` passes `patches` and `arch` through. - -- [ ] **Step 1: Write the failing test** - -```r -test_that("handle_system_dependencies forwards patches and arch", { - captured <- list() - local_mocked_bindings( - install_pkg_sys_deps = function(package_name, tag, local_clone_dir_single, - platform, patches = NULL, arch = NULL) { - captured <<- list(patches = patches, arch = arch) - invisible(TRUE) - } - ) - handle_system_dependencies( - "RcppParallel", "5.1.11-2", "ubuntu-2604", tempfile(), "amd64", - NULL, NULL, NULL, NULL, NULL, NULL, NULL, - patches = "local/patches" - ) - expect_identical(captured$patches, "local/patches") - expect_identical(captured$arch, "amd64") -}) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` -Expected: FAIL — `handle_system_dependencies` has no `patches` argument. - -- [ ] **Step 3: Create the roxygen template** - -```r -# man-roxygen/param-patches.R -#' @param patches Optional path to a patch registry directory containing a -#' `registry.json` (and any referenced diff files). When set, matching -#' packages are pre-built as patched binaries and served to `pak` during -#' dependency installation. Defaults to `NULL` (no patching). -``` - -- [ ] **Step 4: Edit the four functions in `R/build_binaries.R`** - -In `build_single_tag()`'s call to `handle_system_dependencies(...)`, add `patches = patches` as the final argument, and add `patches = NULL` to `build_single_tag`'s own signature plus `#' @template param-patches` to its roxygen block. - -Change `handle_system_dependencies` signature to end with `metadata_db_sslmode,` then add `patches = NULL`, and change its inner `install_pkg_sys_deps(...)` call from: - -```r - install_pkg_sys_deps( - package_name, - tag, - local_clone_dir_single, - platform - ) -``` - -to: - -```r - install_pkg_sys_deps( - package_name, - tag, - local_clone_dir_single, - platform, - patches = patches, - arch = arch - ) -``` - -In `execute_package_builds()`, add `patches = NULL` to the signature and pass `patches = patches` into its `build_single_tag(...)` call inside `worker_function`. - -In `build_binary_package()`, add `patches = NULL` to the signature (after `s3_package_cache`), add `#' @template param-patches` to its roxygen, and pass `patches = patches` into the `execute_package_builds(...)` call. - -- [ ] **Step 5: Document, test, and run package check** - -Run: - -```bash -Rscript -e 'devtools::document()' -Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")' -``` - -Expected: PASS for the new test; `man/build_binary_package.Rd` etc. regenerated. - -- [ ] **Step 6: Commit** - -```bash -git add R/build_binaries.R man-roxygen/param-patches.R man NAMESPACE tests/testthat/test-patches.R -git commit -m "feat(patches): thread patches argument through build_binary_package" -``` - ---- - -### Task A8: End-to-end patch test (guarded) and version bump - -**Files:** -- Modify: `tests/testthat/test-patches.R` -- Modify: `DESCRIPTION` (version), `NEWS.md` - -**Interfaces:** -- Produces: a guarded e2e test proving a dependent package builds when its failing dependency is patched. - -- [ ] **Step 1: Add the guarded e2e test** - -```r -test_that("a patched dependency unblocks a dependent build (e2e)", { - skip_if_not(nzchar(Sys.getenv("BINCRAFT_PATCH_E2E"))) - skip_if_offline() - - patches_dir <- withr::local_tempdir() - jsonlite::write_json( - list(list( - package = "RcppParallel", versions = "*", platforms = list("*"), - env = list(RCPP_PARALLEL_USE_TBB = "0"), - reason = "bundled TBB fails on this toolchain" - )), - file.path(patches_dir, "registry.json"), auto_unbox = TRUE - ) - - out <- withr::local_tempdir() - result <- build_binary_package( - "rts2", tag = "latest", local_output_dir_root = out, - upload = FALSE, archive = FALSE, patches = patches_dir - ) - expect_true(isTRUE(result) || identical(result, "skipped")) -}) -``` - -- [ ] **Step 2: Run the e2e test in a container** - -Run: - -```bash -docker run --rm -e BINCRAFT_PATCH_E2E=1 -v "$PWD":/work -w /work \ - reg.devxy.io/rpkgs/build-env-ubuntu:2604 \ - Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")' -``` - -Expected: the e2e test runs (not skipped) and PASSES; build log shows `Applying patch to RcppParallel`. - -- [ ] **Step 3: Bump version and changelog** - -In `DESCRIPTION`, bump `Version:` to `4.3.0.9999`. Prepend to `NEWS.md`: - -```markdown -# bincraft 4.3.0 - -* `build_binary_package()` gains a `patches` argument: a registry of - per-package env / configure / Makevars overrides and source diffs that are - pre-built into patched binaries and served to pak, fixing compiler- and - OS-specific failures (e.g. RcppParallel) including for transitive deps. -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/testthat/test-patches.R DESCRIPTION NEWS.md -git commit -m "test(patches): guarded end-to-end test; bump to 4.3.0.9999" -``` - ---- - -## Phase B — build-cran-binaries registry and wiring - -All Phase B paths are relative to this repo (`build-cran-binaries`). Phase B depends on a bincraft build that includes Phase A (install the updated bincraft in the build containers, or bump the pinned version in `.crow/build-all-versions-install-deps.yaml`). - -### Task B1: Create the patch registry with the RcppParallel entry - -**Files:** -- Create: `local/patches/registry.json` -- Create: `local/patches/README.md` - -**Interfaces:** -- Produces: the curated registry consumed by bincraft's `patches` argument. - -- [ ] **Step 1: Write the registry** - -```json -[ - { - "package": "RcppParallel", - "versions": "*", - "platforms": ["alpine", "ubuntu-2604"], - "env": { "RCPP_PARALLEL_USE_TBB": "0" }, - "configure_args": [], - "makevars": {}, - "patch": null, - "reason": "bundled Intel TBB fails to build on musl and on newer toolchains (e.g. g++ 15 on ubuntu-2604); disabling TBB falls back to TinyThread" - } -] -``` - -- [ ] **Step 2: Write the README** - -`local/patches/README.md` documents the schema (copy the field table from `specs/2026-06-30-package-patching-design.md`), how to add an entry, and that source diffs go in `local/patches//.patch` referenced by the `patch` field. - -- [ ] **Step 3: Verify it parses** - -Run: `Rscript -e 'x <- jsonlite::fromJSON("local/patches/registry.json", simplifyVector = FALSE); stopifnot(length(x) == 1L, x[[1]]$package == "RcppParallel"); cat("ok\n")'` -Expected: `ok`. - -- [ ] **Step 4: Commit** - -```bash -git add local/patches/registry.json local/patches/README.md -git commit -m "feat(patches): add patch registry with RcppParallel TBB workaround" -``` - ---- - -### Task B2: Registry validator script - -**Files:** -- Create: `local/validate-patches.R` - -**Interfaces:** -- Consumes: `local/patches/registry.json`. -- Produces: a script that exits non-zero on schema violations, missing patch files, or ambiguous overlapping entries. - -- [ ] **Step 1: Write the validator** - -```r -#!/usr/bin/env Rscript -# Validate local/patches/registry.json: schema, referenced patch files, and -# ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. - -dir <- "local/patches" -registry_file <- file.path(dir, "registry.json") -if (!file.exists(registry_file)) { - cat("No registry.json found; nothing to validate.\n") - quit(status = 0L) -} - -reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) -required <- c("package", "versions", "platforms", "reason") -errs <- character(0L) - -for (i in seq_along(reg)) { - e <- reg[[i]] - missing <- setdiff(required, names(e)) - if (length(missing) > 0L) { - errs <- c(errs, sprintf( - "entry %d (%s): missing %s", i, - if (is.null(e$package)) "?" else e$package, toString(missing) - )) - } - if (!is.null(e$patch)) { - p <- file.path(dir, e$patch) - if (!file.exists(p)) { - errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing", - i, e$package, p)) - } - } -} - -# Ambiguous overlap: two entries for the same package with identical platforms -# and versions. -keys <- vapply(reg, function(e) { - sprintf("%s|%s|%s", e$package, - paste(sort(as.character(unlist(e$platforms))), collapse = ","), - e$versions) -}, character(1L)) -dups <- keys[duplicated(keys)] -if (length(dups) > 0L) { - errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups)))) -} - -if (length(errs) > 0L) { - cat("Patch registry validation FAILED:\n") - cat(paste0(" - ", errs, "\n")) - quit(status = 1L) -} -cat(sprintf("Patch registry OK (%d entrie(s)).\n", length(reg))) -``` - -- [ ] **Step 2: Run it (expect success on the B1 registry)** - -Run: `Rscript local/validate-patches.R` -Expected: `Patch registry OK (1 entrie(s)).` and exit 0. - -- [ ] **Step 3: Run it against a broken registry (expect failure)** - -Run: - -```bash -cp local/patches/registry.json /tmp/reg.bak -Rscript -e 'writeLines("[{\"package\":\"X\"}]", "local/patches/registry.json")' -Rscript local/validate-patches.R; echo "exit=$?" -cp /tmp/reg.bak local/patches/registry.json -``` - -Expected: prints `validation FAILED` with a missing-field message and `exit=1`. - -- [ ] **Step 4: Commit** - -```bash -git add local/validate-patches.R -git commit -m "feat(patches): add registry validator script" -``` - ---- - -### Task B3: Hook the validator into pre-commit - -**Files:** -- Modify: `.pre-commit-config.yaml` - -**Interfaces:** -- Produces: a local hook that runs `local/validate-patches.R` when the registry or patch files change. - -- [ ] **Step 1: Add the hook** - -Add a `repo: local` hook entry to `.pre-commit-config.yaml`: - -```yaml - - repo: local - hooks: - - id: validate-patches - name: validate patch registry - entry: Rscript local/validate-patches.R - language: system - files: ^local/patches/ - pass_filenames: false -``` - -- [ ] **Step 2: Verify the hook runs** - -Run: `pre-commit run validate-patches --all-files` -Expected: hook passes (`Patch registry OK`). - -- [ ] **Step 3: Commit** - -```bash -git add .pre-commit-config.yaml -git commit -m "ci(patches): validate patch registry in pre-commit" -``` - ---- - -### Task B4: Pass `patches` through the build entry points - -**Files:** -- Modify: `local/build-one.R:99-119` (the `build_binary_package` call) -- Modify: `local/build-all.R:125-145` (the `build_binary_package` call) - -**Interfaces:** -- Consumes: the new bincraft `patches` argument (Phase A) and `local/patches/`. -- Produces: both entry points pass `patches = "local/patches"`. - -- [ ] **Step 1: Edit `local/build-one.R`** - -In the `bincraft::build_binary_package(` call, add as a new argument (e.g. after `archive = TRUE,`): - -```r - patches = "local/patches", -``` - -- [ ] **Step 2: Edit `local/build-all.R`** - -In the `bincraft::build_binary_package(` call, add: - -```r - patches = "local/patches", -``` - -- [ ] **Step 3: Verify the scripts still parse** - -Run: `Rscript -e 'invisible(parse("local/build-one.R")); invisible(parse("local/build-all.R")); cat("parse ok\n")'` -Expected: `parse ok`. - -- [ ] **Step 4: Commit** - -```bash -git add local/build-one.R local/build-all.R -git commit -m "feat(patches): pass patch registry to bincraft build calls" -``` - ---- - -### Task B5: Document the feature in the README - -**Files:** -- Modify: `README.md` - -**Interfaces:** -- Produces: a short "Patching packages" section explaining the registry and linking the design spec. - -- [ ] **Step 1: Add a README section** - -Add a `## Patching packages` section after the "Build Process" section describing: why patching exists (compiler/OS-specific failures cascading via shared deps like RcppParallel), where the registry lives (`local/patches/registry.json`), the two tiers (env/configure/Makevars overrides vs source diffs), and that bincraft pre-builds patched binaries served to pak. Link `specs/2026-06-30-package-patching-design.md`. - -- [ ] **Step 2: Commit** - -```bash -git add README.md -git commit -m "docs(patches): document the package patching workflow" -``` - ---- - -## Self-Review - -**Spec coverage:** -- Registry in this repo, passed to bincraft → Task A7 (`patches` arg), B1 (registry), B4 (wiring). ✓ -- Mechanism in bincraft (resolve → pre-build → prepend repo → install) → A4–A6. ✓ -- Both tiers, env-first → A4 (`env`, `configure_args`, `makevars`, `patch` all applied; env is the cheap default and the RcppParallel entry uses only env). ✓ -- Pre-built patched binary served from a prepended local repo → A0 proof + A5/A6. ✓ -- Cache keyed by pkg/version/platform/arch/rminor/patchhash → A3 (`patch_cache_key`) + A5 (cache use). ✓ -- S3 upload of patched binaries → **partial**: A5 caches locally only. Cross-machine S3 reuse is deferred (see note below) to keep the first cut shippable; local `/mnt/cache` reuse already removes per-dependent rebuilds within a container. Recorded as out-of-first-cut, not dropped. -- Error handling (diff fails → skip+warn; build fails → skip+warn; no match → skip; ambiguous overlap → validation error) → A4 (`apply_source_patch` FALSE path, build tryCatch), A5 (`resolve_patch_version` NA path), B2 (overlap validation). ✓ -- Observability (log line per applied patch) → A5 (`log_info` with `describe_patch`). DB metadata recording of applied patches is deferred with S3 (same note). -- Testing (registry parse/match, cache key, integration, failure path) → A1–A5 unit tests, A8 e2e, A4 patch-fail test. ✓ - -**Deferred from spec (call out to user):** S3 upload/reuse of patched binaries and recording applied patches in the Postgres build-metadata row. The local `/mnt/cache/patched-binaries` cache already prevents repeated rebuilds within a container; S3 reuse across CI jobs is a follow-up. If you want it in the first cut, add a Task A5b (upload `cached` tarball to a `…/patched/` S3 slot and check there before building) and a metadata column — say so and I'll insert them. - -**Placeholder scan:** No TBD/TODO; every code step shows complete code. README/registry-README prose steps describe exact content to write (acceptable for docs). - -**Type consistency:** `prepare_patched_repo` signature is identical across A5 (definition) and A6 (call site, via defaults). `patch_cache_key(entry, version, platform, arch, r_minor)` argument order matches between A3 and A5. `run_pak_install_with_mutex(..., patched_repo)` matches between A6 definition and the A6 test. `handle_system_dependencies(..., patches)` matches between A7 edit and A7 test. `build_patched_binary(entry, version, dest_dir)` matches A4 and A5. - -**Open risk:** Task A0 gates the whole approach; if it fails, the documented source-serving contingency keeps every later task valid with a localized change in A4. diff --git a/provider.tf b/provider.tf index b267fcc..0a9ba15 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.15" + version = "~> 0.14" } } } diff --git a/specs/2026-06-30-package-patching-design.md b/specs/2026-06-30-package-patching-design.md deleted file mode 100644 index 1344173..0000000 --- a/specs/2026-06-30-package-patching-design.md +++ /dev/null @@ -1,157 +0,0 @@ -# Design: Dynamic per-package patching during binary builds - -Date: 2026-06-30 -Status: Approved (pending spec review) - -## Problem - -Some CRAN packages fail to compile on specific build platforms due to compiler- or OS-specific issues that have nothing to do with the package being built. -The canonical example is `RcppParallel`: its bundled Intel TBB sources fail to build on musl (Alpine) and on newer OS/compiler combinations. -Observed failure on `ubuntu-2604` ("resolute") with `g++ 15.2.0`: - -``` -../build/common.inc:74: *** "" is not supported. Add build/.inc file with os-specific settings . Stop. -make: *** [Makevars:163: tbb] Error 2 -ERROR: compilation failed for package 'RcppParallel' -``` - -Because `RcppParallel` is a dependency of many packages, a single such failure cascades: every dependent package (e.g. `rts2`) also fails, even though nothing is wrong with the dependent itself. - -Today there is no way to intervene. -A package can only be **excluded** (`local/excluded-packages.json`), which is all-or-nothing and does not help dependents. - -## Goal - -Allow a curated set of packages to be "patched" — via lightweight build-time overrides or, when necessary, real source diffs — **before** they are installed, whether the package is a direct build target or a transitive dependency pulled in by `pak`. - -## Key constraint that drives the design - -When `RcppParallel` fails here, it is being installed as a **transitive dependency** by `pak`, inside `bincraft::build_binary_package()`. -`pak` downloads, configures, and compiles it in one subprocess; this repo never touches that source. -For a fix to reach a dependency-of-a-dependency, the fixed package must be visible to `pak` itself, where `pak`'s repositories/sources are configured — which is inside `bincraft`. - -Decisions taken during brainstorming: - -- **Mechanism lives in `bincraft`** (the engine), because only there can transitive deps be influenced. -- **Patch tiers: both, env-overrides first.** Support cheap per-package env vars / configure args / Makevars (version-independent) *and* true source diffs (version-pinned), preferring the lightweight override. -- **Registry data lives in this repo** (`build-cran-binaries`) and is passed into `bincraft`, keeping `bincraft` as pure mechanism and the frequently-changing policy data with operational config. - -## Approaches considered - -| Approach | How pak sees the fix | Verdict | -|---|---|---| -| A. Patched **source** repo — drop patched `.tar.gz` source into a local repo, prepend it | pak recompiles from your source | Simple, but env-tier overrides leak globally (one subprocess builds everything) and the dep recompiles on every dependent build | -| **B. Pre-built patched binary repo (chosen)** | pak installs a ready binary by repo priority | Per-package scoping is free; no recompile; the binary is a cacheable/uploadable artifact that fits the existing system | -| C. pkgdepends per-build hook | intercept each build | No clean per-package pre-compile hook exists; fragile | - -Chosen: **B**. - -## Architecture - -### Registry (this repo) - -``` -local/patches/ - registry.json # the manifest - RcppParallel/ - fix.patch # optional source diff, referenced by an entry -``` - -`registry.json` is an array of entries: - -```json -[ - { - "package": "RcppParallel", - "versions": "*", - "platforms": ["alpine", "ubuntu-2604"], - "env": { "RCPP_PARALLEL_USE_TBB": "0" }, - "configure_args": [], - "makevars": {}, - "patch": null, - "reason": "bundled TBB fails to build on musl / newer compilers" - } -] -``` - -Field semantics: - -- `package` (string, required): CRAN package name. -- `versions` (string, required): `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. - Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. -- `platforms` (array of strings, required): matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). - An entry matches if any listed token matches any build token. - `["*"]` matches all platforms. -- `env` (object, optional): environment variables exported only for this package's isolated build. -- `configure_args` (array, optional): passed as `--configure-args` to the isolated build. -- `makevars` (object, optional): key/value pairs written into a package-local Makevars for the isolated build. -- `patch` (string or null, optional): path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. -- `reason` (string, required): human explanation, surfaced in logs and metadata. - -A fix is any combination of `env`, `configure_args`, `makevars`, and `patch`. -"Env-first" is an authoring guideline (prefer the lightweight override) and an ordering of effort, not a runtime branch — all present fields are applied together for the isolated build. - -### Flow (inside bincraft, around existing pak resolution) - -1. **Resolve** the dependency set (dry-run) to learn the concrete versions `pak` will install. - Reuse bincraft's existing resolution where possible (e.g. a `pkgdepends` proposal: `$resolve()` → inspect resolution → ... → `$solve()` / `$install()` after the local repo is prepended). -2. For each resolved package that matches a registry entry (name + `versions` + `platforms`): obtain a **patched binary** for the exact `version × platform × arch × R-minor`: - - **Cache hit** (local `/mnt/cache/patched-binaries/` or S3): fetch it into the local repo. - - **Cache miss**: download the CRAN **source** for that version, apply the source `patch` (if any) to the unpacked tree, build the binary in isolation with `env` / `configure_args` / `makevars` applied, then place the binary in the local repo and write it to the cache (and S3 if uploading is enabled). -3. **Prepend** the local binary repo (`file://…`) to `pak`'s repo list, and regenerate its `PACKAGES` index. -4. Run the **normal install**. - `pak` resolves the patched binary for the matched package — direct or transitive — because it wins on repo priority for an equal version, and installs it without recompiling. - -### Caching (essential) - -`RcppParallel` is a dependency of dozens of packages; without caching the fix would be rebuilt on every dependent build. -Patched binaries are keyed by: - -``` -_____ -``` - -`patchhash` is a hash of the normalized registry entry plus the referenced diff file contents. -Editing a patch therefore changes the hash and auto-invalidates stale cached binaries. - -- Local cache: `/mnt/cache/patched-binaries/`. -- Optional S3 cache for cross-build reuse: a dedicated `…/patched/` slot under the existing arch/codename structure, mirroring how normal binaries are stored. - -### S3 upload - -Patched binaries **are** uploaded to S3 (in addition to the local cache) so they are reused across CI jobs and machines, not just within one container. -They live in a separate `patched/` slot and are not published into the user-facing `src/contrib` index — they are an internal build accelerator, not a distributed artifact. - -## Error handling - -- **Source diff fails to apply** (CRAN moved past the pinned version): log a clear warning, skip that entry, and proceed. - The package builds unpatched (status quo) and may fail. - The skipped/failed-to-apply patch is surfaced in build metadata. -- **Pre-build of the patched binary fails**: log a warning, skip, proceed. -- **No version or platform match**: skip silently (the entry simply does not apply to this build). -- **Overlapping entries for one package**: the most specific entry wins (a concrete `platforms`/`versions` beats `"*"`). - Genuine ambiguity (two equally specific, conflicting entries) is a validation error reported before the build. - -## Observability - -- One log line per applied patch, e.g.: `Applying patch to RcppParallel 5.1.11-2 [env: RCPP_PARALLEL_USE_TBB=0]: bundled TBB fails on musl / newer compilers` -- The set of applied patches (package, version, `patchhash`) is recorded in the Postgres build-metadata row for the build, so it is queryable later. - -## Testing - -- **Unit (registry):** parsing and matching — version constraints, platform token matching, precedence/specificity, and detection of ambiguous overlaps. -- **Unit (cache key):** `patchhash` changes when the entry or diff changes; is stable otherwise. -- **Integration:** `RcppParallel` on `resolute` (and/or Alpine) fails to build without a registry entry and succeeds with one; a dependent package such as `rts2` succeeds once the dependency is patched. -- **Failure path:** an entry pinned to an old version against a newer CRAN release → graceful skip with a warning, build continues. - -## Out of scope - -- Shipping a default registry inside `bincraft` (registry is repo-local for now; a baseline-in-engine + repo-override model can come later if needed). -- Publishing patched binaries into the public `src/contrib` index. -- Automatic detection of which packages need patches — entries are curated by hand. - -## Split of work - -- **bincraft:** the mechanism — registry ingestion, resolution hook, isolated patched-binary build, caching/upload, local-repo prepend, logging, metadata recording. - A new `patches` argument on `build_binary_package()`. -- **build-cran-binaries (this repo):** the `local/patches/` registry and diffs, passing `patches = "local/patches"` through `build-one.R` / `build-all.R`, and documentation.