From 346f5629eb1073c4c0edc84e3c07a2ba82b74880 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 28 May 2026 15:52:28 +0200 Subject: [PATCH 001/125] refactor: collapse cdn.tf pullzones into for_each bunnynet_pullzone.devxy-r-binaries (cran.devxy.io) and bunnynet_pullzone.cran_rpkgs_com (cran.rpkgs.com) were byte-for-byte identical except for the zone name and hostname. Same for the two pullzone_hostname resources. Use a single local.pullzones map and for_each on both resources, so shared knobs (block_ips, limit_bandwidth, s3_auth_*, ...) only have to be edited once. State migration is handled by moved {} blocks so the existing pullzones get re-addressed under the new for_each keys instead of being destroyed+recreated. Run terraform plan to confirm a zero- change apply before applying. --- cdn.tf | 102 ++++++++++++++++++++++++--------------------------------- 1 file changed, 43 insertions(+), 59 deletions(-) diff --git a/cdn.tf b/cdn.tf index ea1ab2a..30ee1e2 100644 --- a/cdn.tf +++ b/cdn.tf @@ -1,7 +1,20 @@ # https://registry.terraform.io/providers/BunnyWay/bunnynet/latest/docs/resources/pullzone -# terraform import bunnynet_pullzone.devxy-r-binaries cran -resource "bunnynet_pullzone" "devxy-r-binaries" { - name = "cran" + +locals { + pullzones = { + "cran" = { + hostname = "cran.devxy.io" + } + "cran-rpkgs" = { + hostname = "cran.rpkgs.com" + } + } +} + +resource "bunnynet_pullzone" "this" { + for_each = local.pullzones + + name = each.key origin { type = "OriginUrl" @@ -45,63 +58,11 @@ resource "bunnynet_pullzone" "devxy-r-binaries" { block_root_path = true } -resource "bunnynet_pullzone_hostname" "devxy-r-binaries" { - pullzone = bunnynet_pullzone.devxy-r-binaries.id - name = "cran.devxy.io" - force_ssl = true - tls_enabled = true -} +resource "bunnynet_pullzone_hostname" "this" { + for_each = local.pullzones -### cran.rpkgs.com - -resource "bunnynet_pullzone" "cran_rpkgs_com" { - name = "cran-rpkgs" - - origin { - type = "OriginUrl" - url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" - } - - routing { - tier = "Standard" - } - - s3_auth_enabled = true - s3_auth_key = var.B2_S3_ACCESS_KEY - s3_auth_secret = var.B2_S3_SECRET_KEY - s3_auth_region = "eu-central-003" - - cache_enabled = true - cache_errors = true - request_coalescing_enabled = true - block_post_requests = true - - limit_requests = 60 - limit_connections = 10 - - safehop_enabled = true - - add_canonical_header = true - - cache_stale = ["offline", "updating"] - use_background_update = true - - block_ips = [ - "185.172.53.0" - ] - - # 50 TB - limit_bandwidth = 50000000000000 - - permacache_storagezone = bunnynet_storage_zone.devxy-r-binaries.id - - # rec from docs: https://www.backblaze.com/docs/cloud-storage-integrate-bunnynet-with-backblaze-b2 - block_root_path = true -} - -resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { - pullzone = bunnynet_pullzone.cran_rpkgs_com.id - name = "cran.rpkgs.com" + pullzone = bunnynet_pullzone.this[each.key].id + name = each.value.hostname force_ssl = true tls_enabled = true } @@ -113,3 +74,26 @@ resource "bunnynet_storage_zone" "devxy-r-binaries" { # Los Angeles and Singapore replication_regions = ["LA", "SG"] } + +# State-migration markers so the previously-singleton resources slide into +# the for_each map addresses without a destroy/create. Safe to delete after +# a successful apply on every workspace. +moved { + from = bunnynet_pullzone.devxy-r-binaries + to = bunnynet_pullzone.this["cran"] +} + +moved { + from = bunnynet_pullzone.cran_rpkgs_com + to = bunnynet_pullzone.this["cran-rpkgs"] +} + +moved { + from = bunnynet_pullzone_hostname.devxy-r-binaries + to = bunnynet_pullzone_hostname.this["cran"] +} + +moved { + from = bunnynet_pullzone_hostname.cran_rpkgs_com + to = bunnynet_pullzone_hostname.this["cran-rpkgs"] +} From 2ba6ab28f782bc54782cbf79ee63af500f94750b Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 31 May 2026 09:29:44 +0200 Subject: [PATCH 002/125] chore: bump to bincraft 4.1.1 --- .crow/archive-missed-packages.yaml | 2 +- .crow/process-updates-alpine-322-amd64.yaml | 2 +- .crow/process-updates-alpine-322-arm64.yaml | 2 +- .crow/process-updates-alpine-323-amd64.yaml | 2 +- .crow/process-updates-alpine-323-arm64.yaml | 2 +- .crow/process-updates-redhat-10-amd64.yaml | 2 +- .crow/process-updates-redhat-10-arm64.yaml | 2 +- .crow/process-updates-redhat-8-amd64.yaml | 2 +- .crow/process-updates-redhat-8-arm64.yaml | 2 +- .crow/process-updates-redhat-9-amd64.yaml | 2 +- .crow/process-updates-redhat-9-arm64.yaml | 2 +- .crow/process-updates-ubuntu-2204-amd64.yaml | 2 +- .crow/process-updates-ubuntu-2204-arm64.yaml | 2 +- .crow/process-updates-ubuntu-2404-amd64.yaml | 2 +- .crow/process-updates-ubuntu-2404-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-322-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-322-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-323-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-323-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-10-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-10-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-8-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-8-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-9-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-9-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index b1a3a8a..c9e1ce5 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -55,7 +55,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), workers = 2L)' backend_options: diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index ef51448..7b5a7dc 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index d7f3664..474a74c 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index 2bf897b..e345216 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index 1d18c9c..e4b3adf 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-10-amd64.yaml b/.crow/process-updates-redhat-10-amd64.yaml index 6fcd6ab..2b96b60 100644 --- a/.crow/process-updates-redhat-10-amd64.yaml +++ b/.crow/process-updates-redhat-10-amd64.yaml @@ -53,7 +53,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-10-arm64.yaml b/.crow/process-updates-redhat-10-arm64.yaml index 8594fd3..a14852e 100644 --- a/.crow/process-updates-redhat-10-arm64.yaml +++ b/.crow/process-updates-redhat-10-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml index 446c8c4..bb658b1 100644 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ b/.crow/process-updates-redhat-8-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml index 248693b..0695ec9 100644 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ b/.crow/process-updates-redhat-8-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml index 045a370..86c4669 100644 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ b/.crow/process-updates-redhat-9-amd64.yaml @@ -53,7 +53,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml index 1d1e04f..82cd059 100644 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ b/.crow/process-updates-redhat-9-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml index be7da88..2d408cd 100644 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ b/.crow/process-updates-ubuntu-2204-amd64.yaml @@ -53,7 +53,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml index 3726145..ca93998 100644 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ b/.crow/process-updates-ubuntu-2204-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml index f52137a..168c17b 100644 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ b/.crow/process-updates-ubuntu-2404-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml index 5f8f3af..057d091 100644 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ b/.crow/process-updates-ubuntu-2404-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml index 2df97fb..ba7735c 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml index 11eeca5..076b267 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml index a111e34..e6f95c2 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml index a30cf76..5c1e732 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml index 93054bd..6a70915 100644 --- a/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml index 79a3a11..dae5001 100644 --- a/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml index 26bf277..33326d2 100644 --- a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml index 6b7582f..fee0bb3 100644 --- a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml index c10ac34..b78dd37 100644 --- a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml index b693252..d339587 100644 --- a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml index 839d1ed..ee5c094 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml index d4fadf0..40b3b71 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml index 8b3edc9..973b7c9 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml index df4f3f0..eaf0a94 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' From 35dafab737da8d9d27b8caa62d92f7470818eb3f Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 8 Jun 2026 08:29:06 +0000 Subject: [PATCH 003/125] refactor: hoist CDN block_ips to a variable (#81) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `block_ips = ["185.172.53.0"]` was duplicated in both `bunnynet_pullzone` resources in `cdn.tf`. Move it to a typed `cdn_block_ips` variable in `vars.tf` with the existing IP as the default. - No plan diff on apply — same value, just sourced from `var.cdn_block_ips` instead of a literal. - Adding/removing IPs is now a one-line var override (or a default change) instead of two edits in the resource bodies. ## Interaction with #80 (for_each refactor) `#80` collapses the two pullzones to `bunnynet_pullzone.this[for_each]`. Either order works; whichever lands second is a trivial one-line rebase on the surviving `block_ips =` line. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/81 --- cdn.tf | 8 ++------ vars.tf | 8 ++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cdn.tf b/cdn.tf index ea1ab2a..c41fdb4 100644 --- a/cdn.tf +++ b/cdn.tf @@ -32,9 +32,7 @@ resource "bunnynet_pullzone" "devxy-r-binaries" { cache_stale = ["offline", "updating"] use_background_update = true - block_ips = [ - "185.172.53.0" - ] + block_ips = var.cdn_block_ips # 50 TB limit_bandwidth = 50000000000000 @@ -86,9 +84,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { cache_stale = ["offline", "updating"] use_background_update = true - block_ips = [ - "185.172.53.0" - ] + block_ips = var.cdn_block_ips # 50 TB limit_bandwidth = 50000000000000 diff --git a/vars.tf b/vars.tf index aad56ff..b831269 100644 --- a/vars.tf +++ b/vars.tf @@ -17,3 +17,11 @@ variable "B2_S3_SECRET_KEY" { type = string sensitive = true } + +variable "cdn_block_ips" { + description = "IP addresses to block at the BunnyCDN pullzones (cran.devxy.io, cran.rpkgs.com)." + type = list(string) + default = [ + "185.172.53.0", + ] +} From fb95042d16b1da465ced1a6f191c782874e2aea1 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 8 Jun 2026 08:29:40 +0000 Subject: [PATCH 004/125] chore: drop stale Hetzner-pointing helpers (#77) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Both helpers still point at the old Hetzner storage: - `Justfile` — all 3 recipes (`build-all`, `build-single`, `process-updates`) hit `hel1.your-objectstorage.com` / bucket `devxy-r-package-binaries-hel1` using `HETZNER_S3_*_K3S` env vars. - `local/manual-package-index-update.R` — same endpoint + bucket, hardcoded into `s3fs::s3_dir_ls()` / `s3_file_delete()` calls. Storage moved to Backblaze (`s3.eu-central-003.backblazeb2.com` / `devxy-rpkgs-binaries`) a while back, so running either of these today would write to the wrong bucket or fail outright. No callers reference them in-tree, deleting outright. A one-off rebuild can just call `bincraft::upload_package_index()` directly with current Backblaze settings. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/77 --- Justfile | 13 ---------- local/manual-package-index-update.R | 39 ----------------------------- 2 files changed, 52 deletions(-) delete mode 100644 Justfile delete mode 100644 local/manual-package-index-update.R diff --git a/Justfile b/Justfile deleted file mode 100644 index 0c960d9..0000000 --- a/Justfile +++ /dev/null @@ -1,13 +0,0 @@ -# just build-all alpine 3.22 arm64 4.5.3 odbc 1 -build-all OS OS_VERSION ARCH R_VERSION PACKAGE NCPUS: - docker run --rm -it --platform linux/{{ARCH}} -v ./:/package -e AWS_ACCESS_KEY_ID="$HETZNER_S3_ACCESS_KEY_K3S" -e AWS_SECRET_ACCESS_KEY="$HETZNER_S3_SECRET_KEY_K3S" -e PGPASS="$PGPASS" -e NCPUS={{NCPUS}} --pull=always reg.devxy.io/rpkgs/build-env-{{OS}}:{{OS_VERSION}} bash -c '/opt/R/{{R_VERSION}}/bin/R -q -e "install.packages(\"pak\", repos = sprintf(\"https://r-lib.github.io/p/pak/stable/%s/%s/%s\", .Platform\$pkgType, R.Version()\$os, R.Version()\$arch))" && /opt/R/{{R_VERSION}}/bin/R -q -e "pak::pak(\"git::https://codefloe.com/rpkgs/bincraft.git\")" && /opt/R/{{R_VERSION}}/bin/R -q -e "bincraft::build_binary_package(\"{{PACKAGE}}\", platform = \"{{OS}}\", force=TRUE, s3_endpoint = \"https://hel1.your-objectstorage.com\", s3_region = \"hel1\", s3_bucket = \"devxy-r-package-binaries-hel1\", s3_access_key_id = Sys.getenv(\"HETZNER_S3_ACCESS_KEY_K3S\"), s3_secret_access_key = Sys.getenv(\"HETZNER_S3_SECRET_KEY_K3S\"), metadata_db_host = \"r-binaries.devxy.io\", metadata_db_name = \"build_metadata\", metadata_db_table = \"single_builds\", metadata_db_user = \"rpkgs\", metadata_db_password = Sys.getenv(\"PGPASS\"), metadata_db_sslmode = \"require\", metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"' - -# just build-single alpine 3.22 arm64 4.5.3 odbc 1.5.0 1 -# just build-single alpine 3.22 arm64 4.5.3 sf latest 1 -# just build-single ubuntu noble arm64 4.4.3 rlang 1.1.6 1 -build-single OS OS_VERSION ARCH R_VERSION PACKAGE TAG NCPUS: - docker run --rm -it --platform linux/{{ARCH}} -v ./:/package -e AWS_ACCESS_KEY_ID="$HETZNER_S3_ACCESS_KEY_K3S" -e AWS_SECRET_ACCESS_KEY="$HETZNER_S3_SECRET_KEY_K3S" -e PGPASS="$PGPASS" -e NCPUS={{NCPUS}} --pull=always reg.devxy.io/rpkgs/build-env-{{OS}}:{{OS_VERSION}} bash -c '/opt/R/{{R_VERSION}}/bin/R -q -e "install.packages(\"pak\", repos = sprintf(\"https://r-lib.github.io/p/pak/stable/%s/%s/%s\", .Platform\$pkgType, R.Version()\$os, R.Version()\$arch))" && /opt/R/{{R_VERSION}}/bin/R -q -e "pak::pak(\"git::https://codefloe.com/rpkgs/bincraft.git\")" && /opt/R/{{R_VERSION}}/bin/R -q -e "bincraft::build_binary_package(\"{{PACKAGE}}\", tag = \"{{TAG}}\", platform = \"{{OS}}\", force=TRUE, s3_endpoint = \"https://hel1.your-objectstorage.com\", s3_region = \"hel1\", s3_bucket = \"devxy-r-package-binaries-hel1\", s3_access_key_id = Sys.getenv(\"HETZNER_S3_ACCESS_KEY_K3S\"), s3_secret_access_key = Sys.getenv(\"HETZNER_S3_SECRET_KEY_K3S\"), metadata_db_host = \"r-binaries.devxy.io\", metadata_db_name = \"build_metadata\", metadata_db_table = \"single_builds\", metadata_db_user = \"rpkgs\", metadata_db_password = Sys.getenv(\"PGPASS\"), metadata_db_sslmode = \"require\", metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"' - -# just process-updates redhat 9 arm64 4.4.3 'lubridate::interval(lubridate::today() - 4, lubridate::today() - 4)' -process-updates OS OS_VERSION ARCH R_VERSION interval: - docker run --rm -it --platform linux/{{ARCH}} -e AWS_ACCESS_KEY_ID="$HETZNER_S3_ACCESS_KEY_K3S" -e AWS_SECRET_ACCESS_KEY="$HETZNER_S3_SECRET_KEY_K3S" -e PGPASS="$PGPASS" --pull=always reg.devxy.io/rpkgs/build-env-{{OS}}:{{OS_VERSION}} /opt/R/{{R_VERSION}}/bin/R -q -e "bincraft::process_cran_updates(interval = {{interval}}, platform = \"{{OS}}\", s3_endpoint = \"https://hel1.your-objectstorage.com\", s3_region = \"hel1\", s3_bucket = \"devxy-r-package-binaries-hel1\", s3_access_key_id = Sys.getenv(\"HETZNER_S3_ACCESS_KEY_K3S\"), s3_secret_access_key = Sys.getenv(\"HETZNER_S3_SECRET_KEY_K3S\"), metadata_db_host = \"r-binaries.devxy.io\", metadata_db_name = \"build_metadata\", metadata_db_table = \"single_builds\", metadata_db_user = \"rpkgs\", metadata_db_password = Sys.getenv(\"PGPASS\"), metadata_db_sslmode = \"require\", metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" diff --git a/local/manual-package-index-update.R b/local/manual-package-index-update.R deleted file mode 100644 index f100217..0000000 --- a/local/manual-package-index-update.R +++ /dev/null @@ -1,39 +0,0 @@ -# R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' -# remotes::install_gitlab("devxy/r-package-binaries/bincraft") - -library(bincraft) -library(dplyr) -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), - aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"), - endpoint = "https://hel1.your-objectstorage.com", - region_name = "hel1", - refresh = TRUE -) - -s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES.db") -s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES.gz") -s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES.rds") -s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES") - -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/duckdbfs_0.0.9.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/Archive/duckplyr/duckplyr_0.4.1.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/duckplyr_1.0.0.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/duckduckr_1.0.0.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/getDTeval_0.0.2.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/dualtrees_0.1.5.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/arm64/alpine321/latest/src/contrib/tidyselect_1.2.1.tar.gz") -s3fs::s3_file_info("devxy-r-package-binaries-hel1/arm64/alpine321/latest/src/contrib/tidysq_1.2.3.tar.gz") -# s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib/Archive/curl/curl_6.0.1.tar.gz") - -s3fs::s3_file_download("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db") - -upload_package_index(codename = "noble", arch = "arm64") - -fs::file_delete("PACKAGES.db") - - -con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES-debug.db") - -DBI::dbListTables(con) -df = DBI::dbReadTable(con, "packages") From 53da56874913131212cf45836aeb17cabb03bfe0 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 8 Jun 2026 08:30:19 +0000 Subject: [PATCH 005/125] refactor: single source of truth for excluded packages (#79) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `local/build-all.R` had a 75-name hardcoded `exclude <- c(...)` vector that had drifted from `local/excluded-packages.json` — `RcmdrPlugin.ROC` was in the R vector but missing from the JSON. `.crow/weekly-rebuild-missing-*.yaml` already reads the JSON via `jsonlite::fromJSON(...)[["package"]]`. This brings `build-all.R` in line with that pattern. Changes: - **`local/excluded-packages.json`**: add the missing `RcmdrPlugin.ROC` entry (reason `"hang"`, matching siblings). - **`local/build-all.R`**: replace the 16-line hardcoded vector with one `jsonlite::fromJSON(...)` call. - **`.crow/build-all-versions-install-deps-{amd,arm}64.yaml`**: add `jsonlite` to the install-deps `pak::pak()` list so it's available in `/mnt/cache/R-pkgs` for the build step. - **`.crow/build-all-versions-arm64.yaml`**: drop the dead base64-encoded `SKIP_PKGS` docs comment that nobody was passing as a `--var` anyway; replace with a one-line pointer to the JSON. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/79 --- .crow/build-all-versions-arm64.yaml | 4 ++-- ...build-all-versions-install-deps-amd64.yaml | 2 +- ...build-all-versions-install-deps-arm64.yaml | 2 +- local/build-all.R | 19 ++----------------- local/excluded-packages.json | 1 + 5 files changed, 7 insertions(+), 21 deletions(-) diff --git a/.crow/build-all-versions-arm64.yaml b/.crow/build-all-versions-arm64.yaml index afa6e89..37992df 100644 --- a/.crow/build-all-versions-arm64.yaml +++ b/.crow/build-all-versions-arm64.yaml @@ -3,8 +3,8 @@ # OS (alpine) # OS_VERSION (3.22) # R_VERSION (4.5.3) -# echo "c('RInno','KeyboardSimulator','R2PPT','RWinEdt','blatr','excel.link','spectrino','taskscheduleR','MDSGUI','BiplotGUI','R2wd','rFUSION','MediaNews','doBy','IDPmisc','frailtypack','afex','FrF2','DoE.base','agricolae','doFuture','fscaret','PHYLOGR','seewave','pls','relaimpo','geepack','gggenes','NPCirc','repmis','PNDSIBGE','lidR','poismf','neonstore','MachineShop','mvst','MacBehaviour','mcmcderive','RGIFT','KnowBR','netmeta','spdep','sf','Rfast','compareGroups','ff','GsymPoint','RcppDynProg','comtradr','RcppDynProg','FD','PearsonDS','DCluster','gRc','mixlm','geospt','fdth','ffmanova','fiery','ffscrapr','cold','fiery','RcmdrPlugin.DoE','RcmdrPlugin.NMBU')" | base64 -# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.22 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --branch=main --var SKIP_PKGS=YygnUklubm8nLCdLZXlib2FyZFNpbXVsYXRvcicsJ1IyUFBUJywnUldpbkVkdCcsJ2JsYXRyJywnZXhjZWwubGluaycsJ3NwZWN0cmlubycsJ3Rhc2tzY2hlZHVsZVInLCdNRFNHVUknLCdCaXBsb3RHVUknLCdSMndkJywnckZVU0lPTicsJ01lZGlhTmV3cycsJ2RvQnknLCdJRFBtaXNjJywnZnJhaWx0eXBhY2snLCdhZmV4JywnRnJGMicsJ0RvRS5iYXNlJywnYWdyaWNvbGFlJywnZG9GdXR1cmUnLCdmc2NhcmV0JywnUEhZTE9HUicsJ3NlZXdhdmUnLCdwbHMnLCdyZWxhaW1wbycsJ2dlZXBhY2snLCdnZ2dlbmVzJywnTlBDaXJjJywncmVwbWlzJywnUE5EU0lCR0UnLCdsaWRSJywncG9pc21mJywnbmVvbnN0b3JlJywnTWFjaGluZVNob3AnLCdtdnN0JywnTWFjQmVoYXZpb3VyJywnbWNtY2Rlcml2ZScsJ1JHSUZUJywnS25vd0JSJywnbmV0bWV0YScsJ3NwZGVwJywnc2YnLCdSZmFzdCcsJ2NvbXBhcmVHcm91cHMnLCdmZicsJ0dzeW1Qb2ludCcsJ1JjcHBEeW5Qcm9nJywnY29tdHJhZHInLCdSY3BwRHluUHJvZycsJ0ZEJywnUGVhcnNvbkRTJywnRENsdXN0ZXInLCdnUmMnLCdtaXhsbScsJ2dlb3NwdCcsJ2ZkdGgnLCdmZm1hbm92YScsJ2ZpZXJ5JywnZmZzY3JhcHInLCdjb2xkJywnZmllcnknLCdSY21kclBsdWdpbi5Eb0UnLCdSY21kclBsdWdpbi5OTUJVJykK --log-level=info 7 +# Skip list lives in local/excluded-packages.json (read by local/build-all.R). +# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.22 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --branch=main --log-level=info 7 # crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.23 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --branch=main --log-level=info 5 # crow pipeline create --var ARCH=arm64 --var OS=redhat --var OS_VERSION=10 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 when: diff --git a/.crow/build-all-versions-install-deps-amd64.yaml b/.crow/build-all-versions-install-deps-amd64.yaml index d5729be..4d50ddf 100644 --- a/.crow/build-all-versions-install-deps-amd64.yaml +++ b/.crow/build-all-versions-install-deps-amd64.yaml @@ -34,7 +34,7 @@ steps: - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Precomputed %s package versions to build', nrow(pkgs))" backend_options: docker: diff --git a/.crow/build-all-versions-install-deps-arm64.yaml b/.crow/build-all-versions-install-deps-arm64.yaml index 1242172..2b3b881 100644 --- a/.crow/build-all-versions-install-deps-arm64.yaml +++ b/.crow/build-all-versions-install-deps-arm64.yaml @@ -33,7 +33,7 @@ steps: - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Precomputed %s package versions to build', nrow(pkgs))" backend_options: docker: diff --git a/local/build-all.R b/local/build-all.R index 651a0c7..102d18c 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -21,23 +21,8 @@ chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE)) chunk <- chunks[[split_index]] sprintf("# of package versions for this job: %s", nrow(chunk)) -# Exclude known problematic packages -exclude <- c( - "RInno", "KeyboardSimulator", "R2PPT", "RWinEdt", "blatr", "excel.link", - "spectrino", "taskscheduleR", "MDSGUI", "BiplotGUI", "R2wd", "rFUSION", - "MediaNews", "doBy", "IDPmisc", "frailtypack", "afex", "FrF2", "DoE.base", - "agricolae", "doFuture", "fscaret", "PHYLOGR", "seewave", "pls", "relaimpo", - "geepack", "gggenes", "NPCirc", "repmis", "PNDSIBGE", "lidR", "poismf", - "neonstore", "MachineShop", "mvst", "MacBehaviour", "mcmcderive", "RGIFT", - "KnowBR", "netmeta", "spdep", "Rfast", "compareGroups", "ff", "GsymPoint", - "RcppDynProg", "comtradr", "FD", "PearsonDS", "DCluster", "gRc", "mixlm", - "geospt", "fdth", "ffmanova", "fiery", "ffscrapr", "cold", - "RcmdrPlugin.DoE", "RcmdrPlugin.NMBU", "RcmdrPlugin.RiskDemo", - "RcmdrPlugin.ROC", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", - "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.aRnova", - "RcmdrPlugin.depthTools", "RcmdrPlugin.orloca", "RcmdrPlugin.sos", - "RcmdrPlugin.survival", "RcmdrPlugin.temis", "GWlasso", "GWmodelVis" -) +# Exclude known problematic packages (single source of truth) +exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] chunk <- chunk[!chunk$Package %in% exclude, ] # Read pre-computed S3 listing from install-deps step diff --git a/local/excluded-packages.json b/local/excluded-packages.json index 4bfa2de..dccfc0b 100644 --- a/local/excluded-packages.json +++ b/local/excluded-packages.json @@ -61,6 +61,7 @@ {"package": "RcmdrPlugin.DoE", "reason": "hang"}, {"package": "RcmdrPlugin.NMBU", "reason": "hang"}, {"package": "RcmdrPlugin.RiskDemo", "reason": "hang"}, + {"package": "RcmdrPlugin.ROC", "reason": "hang"}, {"package": "RcmdrPlugin.TeachStat", "reason": "hang"}, {"package": "RcmdrPlugin.TeachingDemos", "reason": "hang"}, {"package": "RcmdrPlugin.UCA", "reason": "hang"}, From 4425904eebc9f332c9cd30dafcf6e7439696a722 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 8 Jun 2026 08:30:38 +0000 Subject: [PATCH 006/125] feat(ci): archive missed packages for redhat-10 and alpine323 (#75) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `archive-missed-packages.yaml`'s matrix was last touched when redhat-9 and alpine322 were the newest targets. redhat-10 and alpine323 builds are now wired up across `process-updates-*` and `weekly-rebuild-missing-*` workflows, but the daily archive sweep wasn't catching them — so missed-archive packages on those platforms accumulate untouched. Adds the 4 missing entries: - `redhat-10` × `{amd64, arm64}` - `alpine323` × `{amd64, arm64}` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/75 --- .crow/archive-missed-packages.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index c9e1ce5..bb9ecb4 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -25,6 +25,10 @@ matrix: ARCH: amd64 - CODENAME: redhat-9 ARCH: arm64 + - CODENAME: redhat-10 + ARCH: amd64 + - CODENAME: redhat-10 + ARCH: arm64 - CODENAME: alpine321 ARCH: amd64 - CODENAME: alpine321 @@ -33,6 +37,10 @@ matrix: ARCH: amd64 - CODENAME: alpine322 ARCH: arm64 + - CODENAME: alpine323 + ARCH: amd64 + - CODENAME: alpine323 + ARCH: arm64 steps: - name: 'Archive missed packages' From 228459c0bae107eac71747930bee28317c178017 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 8 Jun 2026 08:31:03 +0000 Subject: [PATCH 007/125] fix(ci): actually purge BunnyCDN cache in purge_cdn_cache.sh (#78) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary While cleaning this up I noticed the script **defined** `purge_cdn_cache()` but never **called** it. Every CI run only declared the helper and exited cleanly without issuing a single curl. The PACKAGES freshness on cran.devxy.io / cran.rpkgs.com has been carried entirely by R's `Cache-Control: no-cache` header on the index files. Fixes in one shot: 1. **Actually run the purge.** The shifted args (`api_key`, `arch`, `os_id`, `domain…`) are now consumed inline and a curl POST is issued per (domain × resource). 2. **Drop `set -x`** — leaked the `AccessKey:` header into job logs. 3. **Drop the Python URL-encoder.** `curl -G --data-urlencode "url=…" --data "async=false" https://api.bunny.net/purge` does the same with no Python. The workflow purge steps can later switch from `alpine:3.23 + apk add bash curl` to a slimmer curl-only image. 4. **Add `src/contrib/Meta/archive.rds`** to the purged resource list. It's rewritten on every `process_cran_updates` run (see README) and was being served stale. ## Risks - This is the **first time** the script actually purges anything. If anything else (e.g. a downstream service) relied on the no-op behavior, this PR is the moment it stops being silent. I don't see any such caller. - Edge cache miss right after a purge means an origin S3 fetch — minor latency uptick on the first request per region per resource. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/78 --- scripts/purge_cdn_cache.sh | 96 +++++++++++++++----------------------- 1 file changed, 38 insertions(+), 58 deletions(-) diff --git a/scripts/purge_cdn_cache.sh b/scripts/purge_cdn_cache.sh index a4e34c3..b525a44 100644 --- a/scripts/purge_cdn_cache.sh +++ b/scripts/purge_cdn_cache.sh @@ -1,64 +1,44 @@ -#!/bin/bash +#!/usr/bin/env bash +# +# Purge BunnyCDN edge cache for the index files of one ARCH+OS_ID +# combination across one or more domains. +# +# Usage: +# purge_cdn_cache.sh [...] +# +set -euo pipefail -set -o pipefail -set -x +if (( $# < 4 )); then + echo "usage: $0 [...]" >&2 + exit 2 +fi -# Function to URL-encode a string using Python -urlencode() { - python3 -c "import urllib.parse, sys; print(urllib.parse.quote(sys.argv[1], safe=''))" "$1" +api_key="$1" +arch="$2" +os_id="$3" +shift 3 + +resources=( + "src/contrib/PACKAGES" + "src/contrib/PACKAGES.gz" + "src/contrib/PACKAGES.rds" + "src/contrib/PACKAGES.db" + "src/contrib/Meta/archive.rds" +) + +purge_one() { + local target="$1" + # --data-urlencode encodes value of `url=` for us; -G forces GET. + curl -sS -G -X POST \ + -H "AccessKey: ${api_key}" \ + --data-urlencode "url=${target}" \ + --data "async=false" \ + "https://api.bunny.net/purge" + echo } -# Function to send the purge request using the provided API key -purge() { - local api_key="$1" - local url="$2" - curl -vL -X POST -H "AccessKey: $api_key" "$url" -} - -# Function to generate the purge URLs for a given domain, arch, and OS ID -generate_urls() { - local domain="$1" - local arch="$2" - local os_id="$3" - - # Array of resource paths (can be modified or extended as needed) - local resources=( - "src/contrib/PACKAGES" - "src/contrib/PACKAGES.gz" - "src/contrib/PACKAGES.rds" - "src/contrib/PACKAGES.db" - ) - - local urls=() +for domain in "$@"; do for resource in "${resources[@]}"; do - local target_url="https://${domain}/${arch}/${os_id}/latest/${resource}" - local encoded - encoded=$(urlencode "$target_url") - urls+=( "https://api.bunny.net/purge?url=${encoded}&async=false" ) + purge_one "https://${domain}/${arch}/${os_id}/latest/${resource}" done - - echo "${urls[@]}" -} - -# Function to process multiple domains. -# Usage: process_domains domain1 [domain2 ... domainN] -purge_cdn_cache() { - local api_key="$1" - local arch="$2" - local os_id="$3" - shift 3 # Remove the first three arguments, leaving the domains - - for domain in "$@"; do - # Generate the list of URLs for the current domain - local urls - urls=($(generate_urls "$domain" "$arch" "$os_id")) - - # Purge each URL - for url in "${urls[@]}"; do - purge "$api_key" "$url" - done - done -} - -# Example usage: -# purge_cdn_cache "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "domain1.com" "domain2.com" +done From 81f585f7f516bdd493d0419e884b79a451714527 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Wed, 10 Jun 2026 00:31:30 +0000 Subject: [PATCH 008/125] chore(deps): update reg.devxy.io/docker.io/library/alpine docker tag to v3.24 --- .crow/process-updates-alpine-322-amd64.yaml | 2 +- .crow/process-updates-alpine-322-arm64.yaml | 2 +- .crow/process-updates-alpine-323-amd64.yaml | 2 +- .crow/process-updates-alpine-323-arm64.yaml | 2 +- .crow/process-updates-redhat-10-amd64.yaml | 2 +- .crow/process-updates-redhat-10-arm64.yaml | 2 +- .crow/process-updates-redhat-8-amd64.yaml | 2 +- .crow/process-updates-redhat-8-arm64.yaml | 2 +- .crow/process-updates-redhat-9-amd64.yaml | 2 +- .crow/process-updates-redhat-9-arm64.yaml | 2 +- .crow/process-updates-ubuntu-2204-amd64.yaml | 2 +- .crow/process-updates-ubuntu-2204-arm64.yaml | 2 +- .crow/process-updates-ubuntu-2404-amd64.yaml | 2 +- .crow/process-updates-ubuntu-2404-arm64.yaml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index 7b5a7dc..1927d57 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -77,7 +77,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index 474a74c..c89a22f 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -59,7 +59,7 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index e345216..3a73df5 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -77,7 +77,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index e4b3adf..2401859 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -59,7 +59,7 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-redhat-10-amd64.yaml b/.crow/process-updates-redhat-10-amd64.yaml index 2b96b60..a4f0cd1 100644 --- a/.crow/process-updates-redhat-10-amd64.yaml +++ b/.crow/process-updates-redhat-10-amd64.yaml @@ -79,7 +79,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-redhat-10-arm64.yaml b/.crow/process-updates-redhat-10-arm64.yaml index a14852e..a786070 100644 --- a/.crow/process-updates-redhat-10-arm64.yaml +++ b/.crow/process-updates-redhat-10-arm64.yaml @@ -60,7 +60,7 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml index bb658b1..09e5d47 100644 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ b/.crow/process-updates-redhat-8-amd64.yaml @@ -77,7 +77,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml index 0695ec9..aa51437 100644 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ b/.crow/process-updates-redhat-8-arm64.yaml @@ -59,7 +59,7 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml index 86c4669..bf2b54e 100644 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ b/.crow/process-updates-redhat-9-amd64.yaml @@ -78,7 +78,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml index 82cd059..0356998 100644 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ b/.crow/process-updates-redhat-9-arm64.yaml @@ -59,7 +59,7 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml index 2d408cd..8940c70 100644 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ b/.crow/process-updates-ubuntu-2204-amd64.yaml @@ -78,7 +78,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml index ca93998..2757c68 100644 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ b/.crow/process-updates-ubuntu-2204-arm64.yaml @@ -59,7 +59,7 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml index 168c17b..3dd25c9 100644 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ b/.crow/process-updates-ubuntu-2404-amd64.yaml @@ -77,7 +77,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml index 057d091..69027ae 100644 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ b/.crow/process-updates-ubuntu-2404-arm64.yaml @@ -59,7 +59,7 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/docker.io/library/alpine:3.24 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY From 44482b18f9cb5ac0ca997ac446057811d606d1c6 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 11 Jun 2026 11:16:55 +0200 Subject: [PATCH 009/125] chore: cleanup and adjust cdn definitions --- cdn.tf | 111 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/cdn.tf b/cdn.tf index c41fdb4..2aff3dc 100644 --- a/cdn.tf +++ b/cdn.tf @@ -1,67 +1,76 @@ # https://registry.terraform.io/providers/BunnyWay/bunnynet/latest/docs/resources/pullzone # terraform import bunnynet_pullzone.devxy-r-binaries cran -resource "bunnynet_pullzone" "devxy-r-binaries" { - name = "cran" +# resource "bunnynet_pullzone" "devxy-r-binaries" { +# name = "cran" - origin { - type = "OriginUrl" - url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" - } +# origin { +# type = "OriginUrl" +# url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" +# } - routing { - tier = "Standard" - } +# routing { +# tier = "Standard" +# } - s3_auth_enabled = true - s3_auth_key = var.B2_S3_ACCESS_KEY - s3_auth_secret = var.B2_S3_SECRET_KEY - s3_auth_region = "eu-central-003" +# s3_auth_enabled = true +# s3_auth_key = var.B2_S3_ACCESS_KEY +# s3_auth_secret = var.B2_S3_SECRET_KEY +# s3_auth_region = "eu-central-003" - cache_enabled = true - cache_errors = true - request_coalescing_enabled = true - block_post_requests = true +# cache_enabled = true +# cache_errors = true +# request_coalescing_enabled = true +# block_post_requests = true - limit_requests = 60 - limit_connections = 10 +# limit_requests = 500 +# limit_connections = 50 - safehop_enabled = true +# safehop_enabled = true - add_canonical_header = true +# add_canonical_header = true - cache_stale = ["offline", "updating"] - use_background_update = true +# cache_stale = ["offline", "updating"] +# use_background_update = true - block_ips = var.cdn_block_ips + # block_ips = var.cdn_block_ips - # 50 TB - limit_bandwidth = 50000000000000 +# # 50 TB +# limit_bandwidth = 50000000000000 - permacache_storagezone = bunnynet_storage_zone.devxy-r-binaries.id +# permacache_storagezone = bunnynet_storage_zone.devxy-r-binaries.id - # rec from docs: https://www.backblaze.com/docs/cloud-storage-integrate-bunnynet-with-backblaze-b2 - block_root_path = true -} +# # rec from docs: https://www.backblaze.com/docs/cloud-storage-integrate-bunnynet-with-backblaze-b2 +# block_root_path = true +# } -resource "bunnynet_pullzone_hostname" "devxy-r-binaries" { - pullzone = bunnynet_pullzone.devxy-r-binaries.id - name = "cran.devxy.io" - force_ssl = true - tls_enabled = true -} +# resource "bunnynet_pullzone_hostname" "devxy-r-binaries" { +# pullzone = bunnynet_pullzone.devxy-r-binaries.id +# name = "cran.devxy.io" +# force_ssl = true +# tls_enabled = true +# } ### cran.rpkgs.com resource "bunnynet_pullzone" "cran_rpkgs_com" { name = "cran-rpkgs" + cache_errors = false + + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + origin { - type = "OriginUrl" - url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = 29277 } routing { - tier = "Standard" + filters = [ + "scripting", + ] } s3_auth_enabled = true @@ -70,26 +79,24 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { s3_auth_region = "eu-central-003" cache_enabled = true - cache_errors = true request_coalescing_enabled = true block_post_requests = true - limit_requests = 60 - limit_connections = 10 + limit_requests = 500 + limit_connections = 50 safehop_enabled = true add_canonical_header = true - cache_stale = ["offline", "updating"] - use_background_update = true + cache_stale = ["offline", "updating"] block_ips = var.cdn_block_ips # 50 TB limit_bandwidth = 50000000000000 - permacache_storagezone = bunnynet_storage_zone.devxy-r-binaries.id + # permacache_storagezone = bunnynet_storage_zone.devxy-r-binaries.id # rec from docs: https://www.backblaze.com/docs/cloud-storage-integrate-bunnynet-with-backblaze-b2 block_root_path = true @@ -102,10 +109,10 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } -resource "bunnynet_storage_zone" "devxy-r-binaries" { - name = "devxy-r-binaries-storage" - region = "DE" - zone_tier = "Standard" - # Los Angeles and Singapore - replication_regions = ["LA", "SG"] -} +# resource "bunnynet_storage_zone" "devxy-r-binaries" { +# name = "devxy-r-binaries-storage" +# region = "DE" +# zone_tier = "Standard" +# # Los Angeles and Singapore +# replication_regions = ["LA", "SG"] +# } From a02bd5c4ff1da8a69f30bd76eab39e6c48ee9b64 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 11 Jun 2026 14:42:51 +0200 Subject: [PATCH 010/125] chore: adjust cdn limits --- cdn.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdn.tf b/cdn.tf index 2aff3dc..8a950f5 100644 --- a/cdn.tf +++ b/cdn.tf @@ -82,8 +82,8 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { request_coalescing_enabled = true block_post_requests = true - limit_requests = 500 - limit_connections = 50 + limit_requests = 5000 + limit_connections = 1000 safehop_enabled = true From 4242e63f3031268659764327123143d77271b627 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 12 Jun 2026 13:41:53 +0000 Subject: [PATCH 011/125] refactor(ci): simplify process-updates cache cleanup (#76) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The `rm -rf /mnt/cache/...` block in 14 `process-updates-*` files had two real issues: 1. `/mnt/cache/pkgcache/R/pkgcache` listed twice — harmless duplicate. 2. `/mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres` — missing a space between two paths, so `rm` walks a nested nonsense path that never exists. The `00LOCK-RPostgres` lock cleanup never actually ran. Collapses to the same shape `weekly-rebuild-missing-*` already uses: ``` rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/{bincraft,pkgcache} /mnt/cache/pkgcache/R/pkgcache ``` `00LOCK-*` covers every stale lock (pak, bincraft, RPostgres, r-pkg-binaries, …) without enumerating them, so adding a new locked package later doesn't need another YAML touch. 14 files, 1 line changed each. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/76 --- .crow/process-updates-alpine-322-amd64.yaml | 2 +- .crow/process-updates-alpine-322-arm64.yaml | 2 +- .crow/process-updates-alpine-323-amd64.yaml | 2 +- .crow/process-updates-alpine-323-arm64.yaml | 2 +- .crow/process-updates-redhat-10-amd64.yaml | 2 +- .crow/process-updates-redhat-10-arm64.yaml | 2 +- .crow/process-updates-redhat-8-amd64.yaml | 2 +- .crow/process-updates-redhat-8-arm64.yaml | 2 +- .crow/process-updates-redhat-9-amd64.yaml | 2 +- .crow/process-updates-redhat-9-arm64.yaml | 2 +- .crow/process-updates-ubuntu-2204-amd64.yaml | 2 +- .crow/process-updates-ubuntu-2204-arm64.yaml | 2 +- .crow/process-updates-ubuntu-2404-amd64.yaml | 2 +- .crow/process-updates-ubuntu-2404-arm64.yaml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index 1927d57..70f8418 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -51,7 +51,7 @@ steps: # - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index c89a22f..d0d1d93 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -50,7 +50,7 @@ steps: # - arm64-binaries-r-dep-cache-alpine-321:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index 3a73df5..8847e9f 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -51,7 +51,7 @@ steps: # - amd64-binaries-r-dep-cache-alpine-323:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index 2401859..1352d4a 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -50,7 +50,7 @@ steps: # - arm64-binaries-r-dep-cache-alpine-321:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-redhat-10-amd64.yaml b/.crow/process-updates-redhat-10-amd64.yaml index a4f0cd1..d7900f8 100644 --- a/.crow/process-updates-redhat-10-amd64.yaml +++ b/.crow/process-updates-redhat-10-amd64.yaml @@ -52,7 +52,7 @@ steps: # - amd64-binaries-r-dep-cache-redhat-10:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-redhat-10-arm64.yaml b/.crow/process-updates-redhat-10-arm64.yaml index a786070..cbe15ce 100644 --- a/.crow/process-updates-redhat-10-arm64.yaml +++ b/.crow/process-updates-redhat-10-arm64.yaml @@ -50,7 +50,7 @@ steps: # - arm64-binaries-r-dep-cache-redhat-10:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml index 09e5d47..81afe3f 100644 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ b/.crow/process-updates-redhat-8-amd64.yaml @@ -51,7 +51,7 @@ steps: # - amd64-binaries-r-dep-cache-redhat-8:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml index aa51437..54c9c40 100644 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ b/.crow/process-updates-redhat-8-arm64.yaml @@ -50,7 +50,7 @@ steps: # - arm64-binaries-r-dep-cache-redhat-8:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml index bf2b54e..d2dfdcc 100644 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ b/.crow/process-updates-redhat-9-amd64.yaml @@ -52,7 +52,7 @@ steps: # - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml index 0356998..bd182bf 100644 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ b/.crow/process-updates-redhat-9-arm64.yaml @@ -50,7 +50,7 @@ steps: # - arm64-binaries-r-dep-cache-redhat-9:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml index 8940c70..79ce2df 100644 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ b/.crow/process-updates-ubuntu-2204-amd64.yaml @@ -52,7 +52,7 @@ steps: # - amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml index 2757c68..fd31231 100644 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ b/.crow/process-updates-ubuntu-2204-arm64.yaml @@ -50,7 +50,7 @@ steps: # - arm64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml index 3dd25c9..398f931 100644 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ b/.crow/process-updates-ubuntu-2404-amd64.yaml @@ -51,7 +51,7 @@ steps: # - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml index 69027ae..a9fc8ac 100644 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ b/.crow/process-updates-ubuntu-2404-arm64.yaml @@ -50,7 +50,7 @@ steps: # - arm64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages From d018f3137627845fd51fddc89bc6dcd81ccc7da7 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sun, 14 Jun 2026 00:32:46 +0000 Subject: [PATCH 012/125] chore(deps): update dependency bincraft to v4.2.0 --- .crow/archive-missed-packages.yaml | 2 +- .crow/process-updates-alpine-322-amd64.yaml | 2 +- .crow/process-updates-alpine-322-arm64.yaml | 2 +- .crow/process-updates-alpine-323-amd64.yaml | 2 +- .crow/process-updates-alpine-323-arm64.yaml | 2 +- .crow/process-updates-redhat-10-amd64.yaml | 2 +- .crow/process-updates-redhat-10-arm64.yaml | 2 +- .crow/process-updates-redhat-8-amd64.yaml | 2 +- .crow/process-updates-redhat-8-arm64.yaml | 2 +- .crow/process-updates-redhat-9-amd64.yaml | 2 +- .crow/process-updates-redhat-9-arm64.yaml | 2 +- .crow/process-updates-ubuntu-2204-amd64.yaml | 2 +- .crow/process-updates-ubuntu-2204-arm64.yaml | 2 +- .crow/process-updates-ubuntu-2404-amd64.yaml | 2 +- .crow/process-updates-ubuntu-2404-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-322-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-322-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-323-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-323-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-10-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-10-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-8-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-8-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-9-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-9-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index bb9ecb4..b4bc8e2 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -63,7 +63,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), workers = 2L)' backend_options: diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index 70f8418..6dbda5a 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index d0d1d93..174d062 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index 8847e9f..042e3a8 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index 1352d4a..1de153e 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-10-amd64.yaml b/.crow/process-updates-redhat-10-amd64.yaml index d7900f8..9b1fc27 100644 --- a/.crow/process-updates-redhat-10-amd64.yaml +++ b/.crow/process-updates-redhat-10-amd64.yaml @@ -53,7 +53,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-10-arm64.yaml b/.crow/process-updates-redhat-10-arm64.yaml index cbe15ce..89793e7 100644 --- a/.crow/process-updates-redhat-10-arm64.yaml +++ b/.crow/process-updates-redhat-10-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml index 81afe3f..87ed625 100644 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ b/.crow/process-updates-redhat-8-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml index 54c9c40..d48a357 100644 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ b/.crow/process-updates-redhat-8-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml index d2dfdcc..de49720 100644 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ b/.crow/process-updates-redhat-9-amd64.yaml @@ -53,7 +53,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml index bd182bf..1461526 100644 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ b/.crow/process-updates-redhat-9-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml index 79ce2df..39eed9c 100644 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ b/.crow/process-updates-ubuntu-2204-amd64.yaml @@ -53,7 +53,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml index fd31231..ea9b9b0 100644 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ b/.crow/process-updates-ubuntu-2204-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml index 398f931..88e1432 100644 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ b/.crow/process-updates-ubuntu-2404-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml index a9fc8ac..ecb9733 100644 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ b/.crow/process-updates-ubuntu-2404-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml index ba7735c..e7b3904 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml index 076b267..26624c4 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml index e6f95c2..2b05036 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml index 5c1e732..8c87a02 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml index 6a70915..87e21f0 100644 --- a/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml index dae5001..2b2cf88 100644 --- a/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml index 33326d2..4ef638e 100644 --- a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml index fee0bb3..d02673b 100644 --- a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml index b78dd37..c857116 100644 --- a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml index d339587..c7968ba 100644 --- a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml index ee5c094..6a2d625 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml index 40b3b71..d14c278 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml index 973b7c9..e9002f8 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml index eaf0a94..1f17c45 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.1.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.1.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' From f28ccaf00817006b13288ebe38862d6a2d4f0fb2 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sun, 14 Jun 2026 00:33:22 +0000 Subject: [PATCH 013/125] chore(deps): update reg.devxy.io/rpkgs/build-env-alpine docker tag to v3.24 --- .crow/archive-missed-packages.yaml | 2 +- .crow/process-updates-alpine-322-amd64.yaml | 2 +- .crow/process-updates-alpine-322-arm64.yaml | 2 +- .crow/process-updates-alpine-323-amd64.yaml | 2 +- .crow/process-updates-alpine-323-arm64.yaml | 2 +- .crow/weekly-audit-missing-alpine-321-amd64.yaml | 2 +- .crow/weekly-audit-missing-alpine-321-arm64.yaml | 2 +- .crow/weekly-audit-missing-alpine-322-amd64.yaml | 2 +- .crow/weekly-audit-missing-alpine-322-arm64.yaml | 2 +- .crow/weekly-audit-missing-alpine-323-amd64.yaml | 2 +- .crow/weekly-audit-missing-alpine-323-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-322-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-322-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-323-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-323-arm64.yaml | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index b4bc8e2..e8b63a5 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -44,7 +44,7 @@ matrix: steps: - name: 'Archive missed packages' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index 6dbda5a..a510a53 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -13,7 +13,7 @@ labels: steps: - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.22 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index 174d062..b420330 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -13,7 +13,7 @@ labels: steps: - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.22 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index 042e3a8..7e333e0 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -13,7 +13,7 @@ labels: steps: - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index 1de153e..4112d22 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -13,7 +13,7 @@ labels: steps: - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/weekly-audit-missing-alpine-321-amd64.yaml b/.crow/weekly-audit-missing-alpine-321-amd64.yaml index ef5a200..906ab7b 100644 --- a/.crow/weekly-audit-missing-alpine-321-amd64.yaml +++ b/.crow/weekly-audit-missing-alpine-321-amd64.yaml @@ -8,7 +8,7 @@ skip_clone: true steps: - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: B2_S3_ACCESS_KEY: diff --git a/.crow/weekly-audit-missing-alpine-321-arm64.yaml b/.crow/weekly-audit-missing-alpine-321-arm64.yaml index 8099c57..41ad809 100644 --- a/.crow/weekly-audit-missing-alpine-321-arm64.yaml +++ b/.crow/weekly-audit-missing-alpine-321-arm64.yaml @@ -8,7 +8,7 @@ skip_clone: true steps: - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: B2_S3_ACCESS_KEY: diff --git a/.crow/weekly-audit-missing-alpine-322-amd64.yaml b/.crow/weekly-audit-missing-alpine-322-amd64.yaml index 75cd8c3..2ed0d81 100644 --- a/.crow/weekly-audit-missing-alpine-322-amd64.yaml +++ b/.crow/weekly-audit-missing-alpine-322-amd64.yaml @@ -7,7 +7,7 @@ skip_clone: true steps: - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.22 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: B2_S3_ACCESS_KEY: diff --git a/.crow/weekly-audit-missing-alpine-322-arm64.yaml b/.crow/weekly-audit-missing-alpine-322-arm64.yaml index 290c7aa..ef065e3 100644 --- a/.crow/weekly-audit-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-audit-missing-alpine-322-arm64.yaml @@ -7,7 +7,7 @@ skip_clone: true steps: - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.22 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: B2_S3_ACCESS_KEY: diff --git a/.crow/weekly-audit-missing-alpine-323-amd64.yaml b/.crow/weekly-audit-missing-alpine-323-amd64.yaml index 15c761d..9e613f6 100644 --- a/.crow/weekly-audit-missing-alpine-323-amd64.yaml +++ b/.crow/weekly-audit-missing-alpine-323-amd64.yaml @@ -7,7 +7,7 @@ skip_clone: true steps: - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: B2_S3_ACCESS_KEY: diff --git a/.crow/weekly-audit-missing-alpine-323-arm64.yaml b/.crow/weekly-audit-missing-alpine-323-arm64.yaml index efa41be..e0130c4 100644 --- a/.crow/weekly-audit-missing-alpine-323-arm64.yaml +++ b/.crow/weekly-audit-missing-alpine-323-arm64.yaml @@ -7,7 +7,7 @@ skip_clone: true steps: - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: B2_S3_ACCESS_KEY: diff --git a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml index e7b3904..cb395d9 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml @@ -10,7 +10,7 @@ labels: steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.22 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml index 26624c4..1651b4e 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml @@ -10,7 +10,7 @@ labels: steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.22 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml index 2b05036..94d15af 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml @@ -10,7 +10,7 @@ labels: steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml index 8c87a02..38bc778 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml @@ -10,7 +10,7 @@ labels: steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23 + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 pull: true environment: RED_HAT_DEV_PW: From d558e27c11600c2614bc66bed5cc26a4474a3723 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 08:04:40 +0000 Subject: [PATCH 014/125] feat: R-minor-sensitive binary builds (full + iterative) (#84) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Builds R-minor-sensitive CRAN packages once per installed R minor version (into per-minor S3 slots `…/contrib//`) and everything else once into the generic slot, driven by bincraft 4.2.0's ABI classifier. Both the full and iterative pipelines are covered. ## What's in here **Detection / precompute** - `local/r-minor-helpers.R` — pure `classify_from_metadata()` (NeedsCompilation / risky `LinkingTo`) + `parse_build_args()`, with unit tests. - `local/packages-to-build.R` — adds a per-package `r_minor_sensitive` flag: cheap CRAN-metadata rules first, source download + `bincraft::needs_per_minor_recompile()` only for the ambiguous compiled subset (fail-safe to sensitive). Classified once per package, applied to all versions. **Full build** - `local/build-all.R` — passes the per-row `is_r_minor_sensitive` flag; new `--sensitive-only` mode builds just the risky subset. - `.crow/build-all-versions-{amd64,arm64}.yaml` — install-deps persists the sensitive subset; build step runs a sensitive-only pass under each non-primary `/opt/R/*` minor; new step uploads the generic index plus a per-minor index for each minor. **Iterative build** - All 14 `.crow/process-updates-*.yaml` — primary pass uses `r_minor_detection = 'classifier'`; a sensitive-only multi-R pass builds risky updates under each other minor; per-minor index upload added. **Tooling / housekeeping** - Pins bincraft `v4.1.1` → `v4.2.0` across all workflows; removes the superseded standalone `build-r-minor-sensitive-packages.yaml`. - Adds prek/pre-commit hooks (prettier, markdownlint, editorconfig-checker, yamllint, air) and applies them repo-wide; excludes the verbatim GPL `LICENSE.md` and auxiliary shell scripts. - Design + implementation docs under `docs/superpowers/`. ## Requires before merge - A `v4.2.0` git tag must be pushed on the bincraft repo (codefloe.com/rpkgs/bincraft) — the workflow install steps pin `@v4.2.0`. The full-build install-deps clones `main`, so it is unaffected. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/84 --- .crow/archive-missed-packages.yaml | 1 - .crow/build-all-versions-amd64.yaml | 57 ++++- .crow/build-all-versions-arm64.yaml | 55 ++++- ...build-all-versions-install-deps-amd64.yaml | 4 +- ...build-all-versions-install-deps-arm64.yaml | 4 +- .crow/build-r-minor-sensitive-packages.yaml | 80 ------ .crow/process-updates-alpine-322-amd64.yaml | 16 +- .crow/process-updates-alpine-322-arm64.yaml | 16 +- .crow/process-updates-alpine-323-amd64.yaml | 16 +- .crow/process-updates-alpine-323-arm64.yaml | 16 +- .crow/process-updates-redhat-10-amd64.yaml | 16 +- .crow/process-updates-redhat-10-arm64.yaml | 16 +- .crow/process-updates-redhat-8-amd64.yaml | 16 +- .crow/process-updates-redhat-8-arm64.yaml | 16 +- .crow/process-updates-redhat-9-amd64.yaml | 16 +- .crow/process-updates-redhat-9-arm64.yaml | 16 +- .crow/process-updates-ubuntu-2204-amd64.yaml | 16 +- .crow/process-updates-ubuntu-2204-arm64.yaml | 16 +- .crow/process-updates-ubuntu-2404-amd64.yaml | 16 +- .crow/process-updates-ubuntu-2404-arm64.yaml | 16 +- .editorconfig | 4 + .editorconfig-checker.json | 3 + .markdownlint-cli2.yaml | 4 + .pre-commit-config.yaml | 45 ++++ .prettierignore | 1 + .yamllint.yaml | 19 ++ README.md | 46 ++-- local/archive-missed-pkgs.R | 26 +- local/build-all.R | 79 +++--- local/check-NA.R | 9 +- local/debug-packages-writing.R | 70 ++++-- local/detect-duplicates.R | 7 +- local/excluded-packages.json | 150 ++++++------ local/fetch-rebuild-packages-from-issue.R | 52 ++-- local/label-removed-cran-packages.R | 14 +- local/last-processed-by-platform.R | 2 +- local/missing-cran-packages-db.R | 48 +++- local/missing-cran-packages.R | 48 +++- local/missing-packages-in-index.R | 117 +++++++-- local/packages-to-build.R | 77 +++++- local/packages-without-any-binary.R | 25 +- local/r-minor-helpers.R | 33 +++ local/tests/test-build-all-args.R | 13 + local/tests/test-r-minor-helpers.R | 28 +++ local/weekly-missing-binaries-audit.R | 229 +++++++++++++----- renovate.json | 8 +- 46 files changed, 1170 insertions(+), 412 deletions(-) delete mode 100644 .crow/build-r-minor-sensitive-packages.yaml create mode 100644 .editorconfig-checker.json create mode 100644 .markdownlint-cli2.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 .prettierignore create mode 100644 .yamllint.yaml create mode 100644 local/r-minor-helpers.R create mode 100644 local/tests/test-build-all-args.R create mode 100644 local/tests/test-r-minor-helpers.R diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index e8b63a5..e18a457 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -6,7 +6,6 @@ when: - event: cron cron: archive-missed-packages - matrix: include: - CODENAME: jammy diff --git a/.crow/build-all-versions-amd64.yaml b/.crow/build-all-versions-amd64.yaml index 796c812..0c74ef7 100644 --- a/.crow/build-all-versions-amd64.yaml +++ b/.crow/build-all-versions-amd64.yaml @@ -39,7 +39,7 @@ depends_on: steps: - name: 'Build binaries' - image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: RED_HAT_DEV_PW: @@ -57,7 +57,7 @@ steps: # normal env vars GIT_USER: pat-s # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: "" + R_PKG_CACHE_DIR: '' R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache NCPUS: 2 @@ -70,6 +70,15 @@ steps: # pkgs = readRDS('/mnt/cache/pkgs_amd64.rds'); - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive pass under R $RV ===" + $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true + done # archive missed packages - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" backend_options: @@ -97,3 +106,47 @@ steps: operator: 'Equal' value: 'true' effect: 'NoSchedule' + - name: 'Upload package indexes' + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + R_LIBS_USER: /mnt/cache/R-pkgs + volumes: + - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache + commands: + - | + CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e "cat(bincraft::set_codename(NULL))") + /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m + kubernetes: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: ${ARCH} + node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE} + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/build-all-versions-arm64.yaml b/.crow/build-all-versions-arm64.yaml index 37992df..aac08c3 100644 --- a/.crow/build-all-versions-arm64.yaml +++ b/.crow/build-all-versions-arm64.yaml @@ -54,7 +54,7 @@ depends_on: steps: - name: 'Build binaries' - image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: RED_HAT_DEV_PW: @@ -83,6 +83,15 @@ steps: - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive pass under R $RV ===" + $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true + done # archive missed packages - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" backend_options: @@ -110,3 +119,47 @@ steps: operator: 'Equal' value: 'true' effect: 'NoSchedule' + - name: 'Upload package indexes' + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + R_LIBS_USER: /mnt/cache/R-pkgs + volumes: + - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache + commands: + - | + CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e "cat(bincraft::set_codename(NULL))") + /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m + kubernetes: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: ${ARCH} + node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE} + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/build-all-versions-install-deps-amd64.yaml b/.crow/build-all-versions-install-deps-amd64.yaml index 4d50ddf..285a4e2 100644 --- a/.crow/build-all-versions-install-deps-amd64.yaml +++ b/.crow/build-all-versions-install-deps-amd64.yaml @@ -10,7 +10,7 @@ labels: steps: - name: 'Install deps and bincraft' - image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: REPO_RO_TOKEN: @@ -35,7 +35,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Precomputed %s package versions to build', nrow(pkgs))" + - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: resources: diff --git a/.crow/build-all-versions-install-deps-arm64.yaml b/.crow/build-all-versions-install-deps-arm64.yaml index 2b3b881..98decea 100644 --- a/.crow/build-all-versions-install-deps-arm64.yaml +++ b/.crow/build-all-versions-install-deps-arm64.yaml @@ -9,7 +9,7 @@ labels: steps: - name: 'Install deps and bincraft' - image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: REPO_RO_TOKEN: @@ -34,7 +34,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Precomputed %s package versions to build', nrow(pkgs))" + - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: resources: diff --git a/.crow/build-r-minor-sensitive-packages.yaml b/.crow/build-r-minor-sensitive-packages.yaml deleted file mode 100644 index a0c3c2d..0000000 --- a/.crow/build-r-minor-sensitive-packages.yaml +++ /dev/null @@ -1,80 +0,0 @@ -when: - - event: manual - evaluate: 'build == "r-minor"' -# - event: manual -# evaluate: 'build == "alpine-322-arm64"' -# - event: manual -# evaluate: 'build == "alpine-322-arm64-1"' -# - event: manual -# evaluate: 'build == "all-arm64"' - -# skip_clone: true - -# depends_on: -# - alpine-322-arm64-install-deps - -matrix: - include: - - os: alpine - os_version: 3.21 - r_version: 4.5.3 - - os: alpine - os_version: 3.21 - r_version: 4.4.3 - # - os: alpine - # os_version: 3.21 - # r_version: 4.3 -steps: - - name: 'Build R-minor-sensitive binaries' - image: "docker.io/devxygmbh/rpkgs-build-env-${os}:${os_version}" - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /tmp/R-libs:/mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - NCPUS: 2 - STRATEGY: sequential - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - mkdir -p /tmp/R-libs - - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft && /opt/R/$R_VERSION/bin/R CMD INSTALL --library=/tmp/R-libs /tmp/bincraft && /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run) - - $XVFB -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = bincraft::get_r_minor_sensitive_packages(r_minor_packages_issue_url = 'https://git.devxy.io/api/v1/repos/devxy/build-cran-binaries/issues/29'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, is_r_minor_sensitive = TRUE, force = FALSE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE))" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index a510a53..d1f67c3 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -56,8 +56,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine322', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done backend_options: kubernetes: resources: diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index b420330..617ed42 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -55,8 +55,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine322', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index 7e333e0..86bfc10 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -56,8 +56,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine323', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done backend_options: kubernetes: resources: diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index 4112d22..d8f9260 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -55,8 +55,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine323', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 diff --git a/.crow/process-updates-redhat-10-amd64.yaml b/.crow/process-updates-redhat-10-amd64.yaml index 9b1fc27..307c814 100644 --- a/.crow/process-updates-redhat-10-amd64.yaml +++ b/.crow/process-updates-redhat-10-amd64.yaml @@ -58,8 +58,22 @@ steps: - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel10', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done backend_options: kubernetes: resources: diff --git a/.crow/process-updates-redhat-10-arm64.yaml b/.crow/process-updates-redhat-10-arm64.yaml index 89793e7..e41fd03 100644 --- a/.crow/process-updates-redhat-10-arm64.yaml +++ b/.crow/process-updates-redhat-10-arm64.yaml @@ -56,8 +56,22 @@ steps: - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel10', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml index 87ed625..b1ce39d 100644 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ b/.crow/process-updates-redhat-8-amd64.yaml @@ -56,8 +56,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel8', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done backend_options: kubernetes: resources: diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml index d48a357..e5f8b18 100644 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ b/.crow/process-updates-redhat-8-arm64.yaml @@ -55,8 +55,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel8', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml index de49720..b0525c3 100644 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ b/.crow/process-updates-redhat-9-amd64.yaml @@ -57,8 +57,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel9', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done backend_options: kubernetes: resources: diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml index 1461526..870705a 100644 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ b/.crow/process-updates-redhat-9-arm64.yaml @@ -55,8 +55,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel9', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml index 39eed9c..a7aad25 100644 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ b/.crow/process-updates-ubuntu-2204-amd64.yaml @@ -57,8 +57,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'jammy', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done backend_options: kubernetes: resources: diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml index ea9b9b0..4126add 100644 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ b/.crow/process-updates-ubuntu-2204-arm64.yaml @@ -55,8 +55,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'jammy', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml index 88e1432..954f0ab 100644 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ b/.crow/process-updates-ubuntu-2404-amd64.yaml @@ -56,8 +56,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'noble', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done backend_options: kubernetes: resources: diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml index ecb9733..0ced70a 100644 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ b/.crow/process-updates-ubuntu-2404-arm64.yaml @@ -55,8 +55,22 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'noble', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 diff --git a/.editorconfig b/.editorconfig index 734c5ee..82fac74 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,6 +11,10 @@ insert_final_newline = true [*.md] trim_trailing_whitespace = false +# Markdown code fences hold verbatim content (logs, snippets) whose +# indentation must not be forced to the source indent rules. +indent_style = unset +indent_size = unset # Ignore paths [/collections/ansible_collections/community/**] diff --git a/.editorconfig-checker.json b/.editorconfig-checker.json new file mode 100644 index 0000000..f7abef3 --- /dev/null +++ b/.editorconfig-checker.json @@ -0,0 +1,3 @@ +{ + "Exclude": ["^LICENSE\\.md$", "^benchmark/", "^docker/reprex/"] +} diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml new file mode 100644 index 0000000..66e7cc8 --- /dev/null +++ b/.markdownlint-cli2.yaml @@ -0,0 +1,4 @@ +# Rules live in .markdownlint.yaml; this file only sets cli2 options. +ignores: + - LICENSE.md + - docs/superpowers/** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..c7edc32 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,45 @@ +# cSpell:ignore autofix autoupdate +# Excluded: verbatim GPL license and auxiliary shell scripts with intentional +# in-string formatting (reformatting would corrupt their output). +exclude: | + (?x)^( + LICENSE\.md| + benchmark/| + docker/reprex/ + ) +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + args: + - --markdown-linebreak-ext=md + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.22.1 + hooks: + - id: markdownlint-cli2 + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.8.4 + hooks: + - id: prettier + - repo: https://github.com/posit-dev/air-pre-commit + rev: 0.8.2 + hooks: + - id: air-format + - repo: https://github.com/editorconfig-checker/editorconfig-checker + rev: v3.7.0 + hooks: + - id: editorconfig-checker + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.38.0 + hooks: + - id: yamllint + args: [--strict, -c=.yamllint.yaml] + - repo: local + hooks: + - id: yaml-file-extension + name: Check if YAML files has *.yaml extension. + entry: YAML filenames must have .yaml extension. + language: fail + files: .yml$ diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..0e4b780 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +LICENSE.md diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..d0370a3 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,19 @@ +rules: + comments: + require-starting-space: false + ignore-shebangs: true + min-spaces-from-content: 1 + braces: + min-spaces-inside-empty: 0 + max-spaces-inside-empty: 0 + min-spaces-inside: 0 + max-spaces-inside: 1 + document-start: + present: false + indentation: + spaces: 2 + indent-sequences: true + # .crow/* workflows embed long single-line R commands (>1k chars) + line-length: disable + new-lines: + type: unix diff --git a/README.md b/README.md index fd03609..278a475 100644 --- a/README.md +++ b/README.md @@ -186,19 +186,19 @@ Below is a collection of raw errors observed during the build process:
-``` +```text * installing to library '/tmp/Rtmp7WPw19/temp_libpath114b846b58'\n* installing *source* package 'ade4' ...\n** using staged installation\nERROR: a 'NAMESPACE' file is required\n* removing '/tmp/Rtmp7WPw19/temp_libpath114b846b58/ade4'\n" ``` Tag does not have a NAMESPACE file and hence cannot be built. -``` +```text "* installing to library '/tmp/RtmpLcCitS/temp_libpath1146aabbe92'\nERROR: dependency 'tripack' is not available for package 'alphahull'\n* removing '/tmp/RtmpLcCitS/temp_libpath1146aabbe92/alphahull'\n" ``` Dependency not available: Either because the dependency was not declared or errored itself during installation. -``` +```text In function '\033[01m\033[KRcpp::List solveRRBLUP(const mat&, const mat&, const mat&)\033[m\033[K':\n\033[01m\033[KMME.cpp:162:61:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope\n 162 | double ll = -0.5*(double(optRes[\"objective\"])+df+df*log(2*\033[01;31m\033[KPI\033[m\033[K/df));\n | \033[01;31m\033[K^~\033[m\033[K\n\033[01m\033[KMME.cpp:\033[m\033[K In function '\033[01m\033[KRcpp::List solveRRBLUPMV(const mat&, const mat&, const mat&, int, double)\033[m\033[K':\n\033[01m\033[KMME.cpp:277:31:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope; did you mean '\033[01m\033[KHI\033[m\033[K'?\n 277 | ll -= double(n*m)/2.0*log(2*\033[01;31m\033[KPI\033[m\033[K);\n | \033[01;31m\033[K^~\033[m\033[K\n | \033[32m\033[KHI\033[m\033[K\nmake: *** [/opt/R/4.4.1/lib/R/etc/Makeconf:204: MME.o] Error 1\nERROR: compilation failed for package 'AlphaSimR'\n* removing '/tmp/RtmpclI5CE/temp_libpath11135d215d5/AlphaSimR'\n ``` @@ -230,26 +230,26 @@ For others it might be due to exotic external dependencies which require manual Help in resolving these issues are highly welcome! -| Name | Platform | Arch | Reason | Solved via | Date Created | Date Solved | -| ------------- | -------------- | ------- | -------------------------------- | ---------- | ------------ | ----------- | -| Apollonius | redhat-8 | | gmp missing | 2024-11-15 | | -| doBy | alpine-320 | amd | hangs | 2024-11-23 | | -| later | ubuntu 22 & 24 | amd | hangs for early versions - xfvb? | 2024-11-23 | | -| CoTiMA | ubuntu 22 & 24 | amd | OOM? | 2024-11-23 | | -| FrF2 | alpine | arm | hangs | 2024-11-23 | | -| FrF2.catlg128 | alpine | arm | hangs | 2024-11-23 | | -| DoE.base | alpine | arm/amd | hangs | 2024-11-23 | | -| eha | alpine | amd | hangs | 2024-11-23 | | -| gRain | alpine | amd | hangs | 2024-11-23 | | -| gRbase | alpine | amd | hangs | 2024-11-24 | | -| IDPmisc | alpine | amd | hangs | 2024-11-24 | | -| RVAideMemoire | alpine | amd | hangs | 2024-11-25 | | -| pbkrtest | alpine | arm | hangs | 2024-11-25 | | -| seewave | alpine | arm | hangs | 2024-11-25 | | -| spdep | alpine | arm | hangs | 2024-12-03 | | -| compareGroups | alpine | arm | hangs | 2024-12-11 | | -| SNPassoc | alpine | arm | hangs | 2024-12-11 | | -| surveillance | alpine | arm | hangs | 2024-12-13 | | +| Name | Platform | Arch | Reason | Date Created | Date Solved | +| ------------- | -------------- | ------- | -------------------------------- | ------------ | ----------- | +| Apollonius | redhat-8 | | gmp missing | 2024-11-15 | | +| doBy | alpine-320 | amd | hangs | 2024-11-23 | | +| later | ubuntu 22 & 24 | amd | hangs for early versions - xfvb? | 2024-11-23 | | +| CoTiMA | ubuntu 22 & 24 | amd | OOM? | 2024-11-23 | | +| FrF2 | alpine | arm | hangs | 2024-11-23 | | +| FrF2.catlg128 | alpine | arm | hangs | 2024-11-23 | | +| DoE.base | alpine | arm/amd | hangs | 2024-11-23 | | +| eha | alpine | amd | hangs | 2024-11-23 | | +| gRain | alpine | amd | hangs | 2024-11-23 | | +| gRbase | alpine | amd | hangs | 2024-11-24 | | +| IDPmisc | alpine | amd | hangs | 2024-11-24 | | +| RVAideMemoire | alpine | amd | hangs | 2024-11-25 | | +| pbkrtest | alpine | arm | hangs | 2024-11-25 | | +| seewave | alpine | arm | hangs | 2024-11-25 | | +| spdep | alpine | arm | hangs | 2024-12-03 | | +| compareGroups | alpine | arm | hangs | 2024-12-11 | | +| SNPassoc | alpine | arm | hangs | 2024-12-11 | | +| surveillance | alpine | arm | hangs | 2024-12-13 | | ## CDN Settings diff --git a/local/archive-missed-pkgs.R b/local/archive-missed-pkgs.R index 57e92d7..f817c3e 100644 --- a/local/archive-missed-pkgs.R +++ b/local/archive-missed-pkgs.R @@ -11,16 +11,34 @@ s3fs::s3_file_system( arch = "arm64" os = "alpine321" -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", arch, os)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", + arch, + os +)) length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1]))) -non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) x[1])[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))]) +non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) { + x[1] +})[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))]) # RInno is not built because it only exists for Windows non_archived = setdiff(non_archived, "RInno") # future::plan("sequential", workers = 8) future::plan("sequential", workers = 1) # future.apply::future_lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch)) -lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch, s3_region = "hel1", s3_endpoint = "https://hel1.your-objectstorage.com", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S", s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S")))) +lapply(non_archived, function(x) { + archive_package( + x, + codename = os, + arch = arch, + s3_region = "hel1", + s3_endpoint = "https://hel1.your-objectstorage.com", + s3_bucket = "devxy-r-package-binaries-hel1", + s3_access_key_id = Sys.getenv( + "HETZNER_S3_ACCESS_KEY_K3S", + s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S") + ) + ) +}) # archive_package() - # grep("duckdb_", files, value = T) diff --git a/local/build-all.R b/local/build-all.R index 102d18c..4e7b44e 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -1,10 +1,13 @@ sink(stdout(), type = "message") options(crayon.enabled = TRUE, future.globals.onReference = NULL) +source(file.path("local", "r-minor-helpers.R")) args <- commandArgs(trailingOnly = TRUE) -split_into <- as.integer(args[1]) -split_index <- as.integer(args[2]) -ncpus <- as.integer(args[3]) +parsed <- parse_build_args(args) +split_into <- parsed$split_into +split_index <- parsed$split_index +ncpus <- parsed$ncpus +sensitive_only <- parsed$sensitive_only options(Ncpus = ncpus) # Load bincraft eagerly to avoid lazy-load memory spike during first build call @@ -12,9 +15,20 @@ library(bincraft, quietly = TRUE) library(future) plan("sequential") -# Read precomputed package+version pairs -pkgs <- readRDS("/mnt/cache/packages/pkgs_to_build.rds") -sprintf("Total# of remaining package versions: %s", nrow(pkgs)) +pkgs <- if (sensitive_only) { + readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds") +} else { + readRDS("/mnt/cache/packages/pkgs_to_build.rds") +} +# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive) +if (is.null(pkgs$r_minor_sensitive)) { + pkgs$r_minor_sensitive <- FALSE +} +sprintf( + "Total# of remaining package versions: %s (sensitive_only=%s)", + nrow(pkgs), + sensitive_only +) # Split into chunks for this worker chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE)) @@ -32,26 +46,33 @@ s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") sprintf("S3 cache: %s files", length(s3_cache)) n <- nrow(chunk) -mapply(function(pkg, ver, i) { - cat(sprintf("[%d/%d] %s_%s\n", i, n, pkg, ver)) - bincraft::build_binary_package( - pkg, - tag = ver, - s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", - s3_region = "eu-central-003", - s3_bucket = "devxy-rpkgs-binaries", - s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - s3_package_cache = s3_cache, - metadata_db_host = "r-binaries.devxy.io", - metadata_db_name = "build_metadata", - metadata_db_table = "single_builds", - metadata_db_user = "rpkgs", - metadata_db_password = Sys.getenv("PGPASS"), - metadata_db_sslmode = "require", - metadata_db_port = 15432, - archive = TRUE, - upload = TRUE, - store_build_metadata = TRUE - ) -}, chunk$Package, chunk$Version, seq_len(n)) +mapply( + function(pkg, ver, sens, i) { + cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) + bincraft::build_binary_package( + pkg, + tag = ver, + is_r_minor_sensitive = isTRUE(sens), + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + s3_package_cache = s3_cache, + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + upload = TRUE, + store_build_metadata = TRUE + ) + }, + chunk$Package, + chunk$Version, + chunk$r_minor_sensitive, + seq_len(n) +) diff --git a/local/check-NA.R b/local/check-NA.R index 7405db4..95c158e 100644 --- a/local/check-NA.R +++ b/local/check-NA.R @@ -1,4 +1,3 @@ - ### Scope: Check whether there are any incomplete entries in PACKAGES.rds. These will result in NA when calling available.packages() foo = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib") sum((is.na(foo[, "Version"]))) @@ -17,7 +16,9 @@ foo = available.packages("https://cran.devxy.io/amd64/rhel9/latest/src/contrib") sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) -foo = available.packages("https://cran.devxy.io/amd64/alpine320/latest/src/contrib") +foo = available.packages( + "https://cran.devxy.io/amd64/alpine320/latest/src/contrib" +) sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) @@ -40,6 +41,8 @@ foo = available.packages("https://cran.devxy.io/arm64/rhel9/latest/src/contrib") sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) -foo = available.packages("https://cran.devxy.io/arm64/alpine320/latest/src/contrib") +foo = available.packages( + "https://cran.devxy.io/arm64/alpine320/latest/src/contrib" +) sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) diff --git a/local/debug-packages-writing.R b/local/debug-packages-writing.R index 5a0fd24..533ae2b 100644 --- a/local/debug-packages-writing.R +++ b/local/debug-packages-writing.R @@ -2,42 +2,58 @@ library(future) future::plan(multisession) future::plan(sequential) time = Sys.time() -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive"), recurse = T) +files <- s3fs::s3_dir_ls( + sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive" + ), + recurse = T +) Sys.time() - time - archive <- llply(dirs, function(dir) { - files <- list.files(dir, recursive = FALSE, full.names = TRUE, pattern = "*.tar.gz") + files <- list.files( + dir, + recursive = FALSE, + full.names = TRUE, + pattern = "*.tar.gz" + ) if (length(files) == 0) { print(paste0("Error: Empty directory: ", dir)) return(NULL) } info <- file.info(files) - tryCatch({ - rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) - }, error = function(e) { - print(paste0("Error: Exception catched for Archived directory: ", dir)) - print(e) - return(NULL) - }) + tryCatch( + { + rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) + }, + error = function(e) { + print(paste0("Error: Exception catched for Archived directory: ", dir)) + print(e) + return(NULL) + } + ) info }) -tryCatch({ - rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) -}, error = function(e) { - print(paste0("Error: Exception catched for Archived directory: ", dir)) - print(e) - return(NULL) -}) +tryCatch( + { + rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) + }, + error = function(e) { + print(paste0("Error: Exception catched for Archived directory: ", dir)) + print(e) + return(NULL) + } +) - - -curl::curl_download("https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db") +curl::curl_download( + "https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", + "PACKAGES.db" +) con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db") df = DBI::dbReadTable(con, "packages") @@ -50,6 +66,14 @@ which(grepl("digest", df$Package)) system("cat /tmp/PACKAGES | grep '^Package: digest' | wc -l") -curl::curl_fetch_memory("https://cloud.r-project.org/src/contrib/Meta/archive.rds") -remotes = readRDS(url("https://cloud.r-project.org/src/contrib/Meta/archive.rds", "rb")) -packages = readRDS(url("https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds", "rb")) +curl::curl_fetch_memory( + "https://cloud.r-project.org/src/contrib/Meta/archive.rds" +) +remotes = readRDS(url( + "https://cloud.r-project.org/src/contrib/Meta/archive.rds", + "rb" +)) +packages = readRDS(url( + "https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds", + "rb" +)) diff --git a/local/detect-duplicates.R b/local/detect-duplicates.R index 076499d..f7e2c18 100644 --- a/local/detect-duplicates.R +++ b/local/detect-duplicates.R @@ -37,10 +37,13 @@ future.apply::future_lapply( files <- s3fs::s3_dir_ls( sprintf( "devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", - arch, codename + arch, + codename ) ) - if (length(files) == 0) return(NULL) + if (length(files) == 0) { + return(NULL) + } files_b <- basename(files) pkg_names <- sapply(strsplit(files_b, "_"), `[`, 1) dupes <- unique(pkg_names[duplicated(pkg_names)]) diff --git a/local/excluded-packages.json b/local/excluded-packages.json index dccfc0b..6489998 100644 --- a/local/excluded-packages.json +++ b/local/excluded-packages.json @@ -1,77 +1,77 @@ [ - {"package": "RInno", "reason": "windows-only"}, - {"package": "KeyboardSimulator", "reason": "windows-only"}, - {"package": "R2PPT", "reason": "windows-only"}, - {"package": "RWinEdt", "reason": "windows-only"}, - {"package": "blatr", "reason": "windows-only"}, - {"package": "excel.link", "reason": "windows-only"}, - {"package": "spectrino", "reason": "windows-only"}, - {"package": "taskscheduleR", "reason": "windows-only"}, - {"package": "MDSGUI", "reason": "windows-only"}, - {"package": "BiplotGUI", "reason": "windows-only"}, - {"package": "R2wd", "reason": "windows-only"}, - {"package": "rFUSION", "reason": "windows-only"}, - {"package": "MediaNews", "reason": "windows-only"}, - {"package": "doBy", "reason": "hang"}, - {"package": "IDPmisc", "reason": "hang"}, - {"package": "frailtypack", "reason": "hang"}, - {"package": "afex", "reason": "hang"}, - {"package": "FrF2", "reason": "hang"}, - {"package": "DoE.base", "reason": "hang"}, - {"package": "agricolae", "reason": "hang"}, - {"package": "doFuture", "reason": "hang"}, - {"package": "fscaret", "reason": "hang"}, - {"package": "PHYLOGR", "reason": "hang"}, - {"package": "seewave", "reason": "hang"}, - {"package": "pls", "reason": "hang"}, - {"package": "relaimpo", "reason": "hang"}, - {"package": "geepack", "reason": "hang"}, - {"package": "gggenes", "reason": "hang"}, - {"package": "NPCirc", "reason": "hang"}, - {"package": "repmis", "reason": "hang"}, - {"package": "PNDSIBGE", "reason": "hang"}, - {"package": "lidR", "reason": "hang"}, - {"package": "poismf", "reason": "hang"}, - {"package": "neonstore", "reason": "hang"}, - {"package": "MachineShop", "reason": "hang"}, - {"package": "mvst", "reason": "hang"}, - {"package": "MacBehaviour", "reason": "hang"}, - {"package": "mcmcderive", "reason": "hang"}, - {"package": "RGIFT", "reason": "hang"}, - {"package": "KnowBR", "reason": "hang"}, - {"package": "netmeta", "reason": "hang"}, - {"package": "spdep", "reason": "hang"}, - {"package": "Rfast", "reason": "hang"}, - {"package": "compareGroups", "reason": "hang"}, - {"package": "ff", "reason": "hang"}, - {"package": "GsymPoint", "reason": "hang"}, - {"package": "RcppDynProg", "reason": "hang"}, - {"package": "comtradr", "reason": "hang"}, - {"package": "FD", "reason": "hang"}, - {"package": "PearsonDS", "reason": "hang"}, - {"package": "DCluster", "reason": "hang"}, - {"package": "gRc", "reason": "hang"}, - {"package": "mixlm", "reason": "hang"}, - {"package": "geospt", "reason": "hang"}, - {"package": "fdth", "reason": "hang"}, - {"package": "ffmanova", "reason": "hang"}, - {"package": "fiery", "reason": "hang"}, - {"package": "ffscrapr", "reason": "hang"}, - {"package": "cold", "reason": "hang"}, - {"package": "RcmdrPlugin.DoE", "reason": "hang"}, - {"package": "RcmdrPlugin.NMBU", "reason": "hang"}, - {"package": "RcmdrPlugin.RiskDemo", "reason": "hang"}, - {"package": "RcmdrPlugin.ROC", "reason": "hang"}, - {"package": "RcmdrPlugin.TeachStat", "reason": "hang"}, - {"package": "RcmdrPlugin.TeachingDemos", "reason": "hang"}, - {"package": "RcmdrPlugin.UCA", "reason": "hang"}, - {"package": "RcmdrPlugin.WorldFlora", "reason": "hang"}, - {"package": "RcmdrPlugin.aRnova", "reason": "hang"}, - {"package": "RcmdrPlugin.depthTools", "reason": "hang"}, - {"package": "RcmdrPlugin.orloca", "reason": "hang"}, - {"package": "RcmdrPlugin.sos", "reason": "hang"}, - {"package": "RcmdrPlugin.survival", "reason": "hang"}, - {"package": "RcmdrPlugin.temis", "reason": "hang"}, - {"package": "GWlasso", "reason": "hang"}, - {"package": "GWmodelVis", "reason": "hang"} + { "package": "RInno", "reason": "windows-only" }, + { "package": "KeyboardSimulator", "reason": "windows-only" }, + { "package": "R2PPT", "reason": "windows-only" }, + { "package": "RWinEdt", "reason": "windows-only" }, + { "package": "blatr", "reason": "windows-only" }, + { "package": "excel.link", "reason": "windows-only" }, + { "package": "spectrino", "reason": "windows-only" }, + { "package": "taskscheduleR", "reason": "windows-only" }, + { "package": "MDSGUI", "reason": "windows-only" }, + { "package": "BiplotGUI", "reason": "windows-only" }, + { "package": "R2wd", "reason": "windows-only" }, + { "package": "rFUSION", "reason": "windows-only" }, + { "package": "MediaNews", "reason": "windows-only" }, + { "package": "doBy", "reason": "hang" }, + { "package": "IDPmisc", "reason": "hang" }, + { "package": "frailtypack", "reason": "hang" }, + { "package": "afex", "reason": "hang" }, + { "package": "FrF2", "reason": "hang" }, + { "package": "DoE.base", "reason": "hang" }, + { "package": "agricolae", "reason": "hang" }, + { "package": "doFuture", "reason": "hang" }, + { "package": "fscaret", "reason": "hang" }, + { "package": "PHYLOGR", "reason": "hang" }, + { "package": "seewave", "reason": "hang" }, + { "package": "pls", "reason": "hang" }, + { "package": "relaimpo", "reason": "hang" }, + { "package": "geepack", "reason": "hang" }, + { "package": "gggenes", "reason": "hang" }, + { "package": "NPCirc", "reason": "hang" }, + { "package": "repmis", "reason": "hang" }, + { "package": "PNDSIBGE", "reason": "hang" }, + { "package": "lidR", "reason": "hang" }, + { "package": "poismf", "reason": "hang" }, + { "package": "neonstore", "reason": "hang" }, + { "package": "MachineShop", "reason": "hang" }, + { "package": "mvst", "reason": "hang" }, + { "package": "MacBehaviour", "reason": "hang" }, + { "package": "mcmcderive", "reason": "hang" }, + { "package": "RGIFT", "reason": "hang" }, + { "package": "KnowBR", "reason": "hang" }, + { "package": "netmeta", "reason": "hang" }, + { "package": "spdep", "reason": "hang" }, + { "package": "Rfast", "reason": "hang" }, + { "package": "compareGroups", "reason": "hang" }, + { "package": "ff", "reason": "hang" }, + { "package": "GsymPoint", "reason": "hang" }, + { "package": "RcppDynProg", "reason": "hang" }, + { "package": "comtradr", "reason": "hang" }, + { "package": "FD", "reason": "hang" }, + { "package": "PearsonDS", "reason": "hang" }, + { "package": "DCluster", "reason": "hang" }, + { "package": "gRc", "reason": "hang" }, + { "package": "mixlm", "reason": "hang" }, + { "package": "geospt", "reason": "hang" }, + { "package": "fdth", "reason": "hang" }, + { "package": "ffmanova", "reason": "hang" }, + { "package": "fiery", "reason": "hang" }, + { "package": "ffscrapr", "reason": "hang" }, + { "package": "cold", "reason": "hang" }, + { "package": "RcmdrPlugin.DoE", "reason": "hang" }, + { "package": "RcmdrPlugin.NMBU", "reason": "hang" }, + { "package": "RcmdrPlugin.RiskDemo", "reason": "hang" }, + { "package": "RcmdrPlugin.ROC", "reason": "hang" }, + { "package": "RcmdrPlugin.TeachStat", "reason": "hang" }, + { "package": "RcmdrPlugin.TeachingDemos", "reason": "hang" }, + { "package": "RcmdrPlugin.UCA", "reason": "hang" }, + { "package": "RcmdrPlugin.WorldFlora", "reason": "hang" }, + { "package": "RcmdrPlugin.aRnova", "reason": "hang" }, + { "package": "RcmdrPlugin.depthTools", "reason": "hang" }, + { "package": "RcmdrPlugin.orloca", "reason": "hang" }, + { "package": "RcmdrPlugin.sos", "reason": "hang" }, + { "package": "RcmdrPlugin.survival", "reason": "hang" }, + { "package": "RcmdrPlugin.temis", "reason": "hang" }, + { "package": "GWlasso", "reason": "hang" }, + { "package": "GWmodelVis", "reason": "hang" } ] diff --git a/local/fetch-rebuild-packages-from-issue.R b/local/fetch-rebuild-packages-from-issue.R index d944dd0..26e544d 100644 --- a/local/fetch-rebuild-packages-from-issue.R +++ b/local/fetch-rebuild-packages-from-issue.R @@ -1,15 +1,21 @@ library(httr2, quietly = TRUE) forgejo_base <- "https://git.devxy.io/api/v1" -repo <- "devxy/build-cran-binaries" -platform <- Sys.getenv("PLATFORM") -arch <- Sys.getenv("ARCH") -token <- Sys.getenv("FORGEJO_TOKEN") -output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") +repo <- "devxy/build-cran-binaries" +platform <- Sys.getenv("PLATFORM") +arch <- Sys.getenv("ARCH") +token <- Sys.getenv("FORGEJO_TOKEN") +output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") -if (nchar(platform) == 0) stop("PLATFORM env var is not set") -if (nchar(arch) == 0) stop("ARCH env var is not set") -if (nchar(token) == 0) stop("FORGEJO_TOKEN env var is not set") +if (nchar(platform) == 0) { + stop("PLATFORM env var is not set") +} +if (nchar(arch) == 0) { + stop("ARCH env var is not set") +} +if (nchar(token) == 0) { + stop("FORGEJO_TOKEN env var is not set") +} os_family <- if (grepl("^ubuntu", platform)) { "Ubuntu" @@ -21,12 +27,16 @@ os_family <- if (grepl("^ubuntu", platform)) { platform } -issue_title <- sprintf("Missing package binaries for latest version (%s)", os_family) +issue_title <- sprintf( + "Missing package binaries for latest version (%s)", + os_family +) cat(sprintf("Searching for issue: %s\n", issue_title)) search_url <- sprintf( "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", - forgejo_base, repo, + forgejo_base, + repo, utils::URLencode(issue_title, reserved = TRUE) ) search_resp <- request(search_url) |> @@ -34,7 +44,9 @@ search_resp <- request(search_url) |> req_perform() issues <- resp_body_json(search_resp, simplifyVector = FALSE) -match_idx <- which(vapply(issues, function(x) x$title, character(1)) == issue_title) +match_idx <- which( + vapply(issues, function(x) x$title, character(1)) == issue_title +) if (length(match_idx) == 0) { cat("No matching issue found - nothing to rebuild\n") @@ -55,7 +67,10 @@ lines <- strsplit(body, "\n", fixed = TRUE)[[1]] plat_header <- sprintf("## %s", platform) plat_idx <- which(lines == plat_header) if (length(plat_idx) == 0) { - cat(sprintf("No section found for platform %s - nothing to rebuild\n", platform)) + cat(sprintf( + "No section found for platform %s - nothing to rebuild\n", + platform + )) writeLines(character(0), output_file) q("no") } @@ -64,7 +79,11 @@ if (length(plat_idx) == 0) { arch_pattern <- sprintf("^### %s ", arch) arch_idx <- which(grepl(arch_pattern, lines) & seq_along(lines) > plat_idx[1]) if (length(arch_idx) == 0) { - cat(sprintf("No section found for arch %s under %s - nothing to rebuild\n", arch, platform)) + cat(sprintf( + "No section found for arch %s under %s - nothing to rebuild\n", + arch, + platform + )) writeLines(character(0), output_file) q("no") } @@ -82,6 +101,11 @@ pkg_lines <- section_lines[grepl("^- ", section_lines)] pkgs <- sub("^- ([^ ]+) \\(.*\\)$", "\\1", pkg_lines) pkgs <- pkgs[nchar(pkgs) > 0 & pkgs != "_None_"] -cat(sprintf("Found %d rebuildable packages for %s/%s\n", length(pkgs), platform, arch)) +cat(sprintf( + "Found %d rebuildable packages for %s/%s\n", + length(pkgs), + platform, + arch +)) writeLines(pkgs, output_file) cat(sprintf("Wrote package list to %s\n", output_file)) diff --git a/local/label-removed-cran-packages.R b/local/label-removed-cran-packages.R index 92cd940..12dff9d 100644 --- a/local/label-removed-cran-packages.R +++ b/local/label-removed-cran-packages.R @@ -6,9 +6,13 @@ archived <- quickcode::archivedPkg() |> pull(name) # Query all distinct pkgs in the DB -con <- DBI::dbConnect(RPostgres::Postgres(), - dbname = "build_metadata", host = "r-binaries.devxy.io", - port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "r_binaries", + password = Sys.getenv("PGPASS"), sslmode = "require" ) @@ -22,7 +26,9 @@ to_process <- pkgs_db[pkgs_db %in% archived] # for all matches, set 'removed = TRUE' sapply(to_process, function(.x) { - DBI::dbExecute(con, "UPDATE single_builds SET removed = 'TRUE' where name = $1", + DBI::dbExecute( + con, + "UPDATE single_builds SET removed = 'TRUE' where name = $1", params = list(.x) ) }) diff --git a/local/last-processed-by-platform.R b/local/last-processed-by-platform.R index f913610..398b6c1 100644 --- a/local/last-processed-by-platform.R +++ b/local/last-processed-by-platform.R @@ -7,4 +7,4 @@ query_metadata_table() |> group_by(platform, arch) |> arrange(desc(timestamp)) |> select(name, timestamp) |> - filter(row_number()==1) + filter(row_number() == 1) diff --git a/local/missing-cran-packages-db.R b/local/missing-cran-packages-db.R index 56b3d94..882bca4 100644 --- a/local/missing-cran-packages-db.R +++ b/local/missing-cran-packages-db.R @@ -3,22 +3,34 @@ library(bincraft) library(dplyr) library(DBI) -con <- DBI::dbConnect(RPostgres::Postgres(), - dbname = "build_metadata", host = "r-binaries.devxy.io", - port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "r_binaries", + password = Sys.getenv("PGPASS"), sslmode = "require" ) `%nin%` <- Negate(`%in%`) -new_packages <- get_new_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name -removed_pkgs <- get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name +new_packages <- get_new_cran_packages(lubridate::interval( + lubridate::today(), + lubridate::today() - 2 +))$name +removed_pkgs <- get_removed_cran_packages(lubridate::interval( + lubridate::today(), + lubridate::today() - 2 +))$name # filter windows packages # filter new packages from the last X days -cran_pkgs <- unique(tools::CRAN_package_db() |> - filter(`OS_type` != "windows" | is.na(`OS_type`)) |> - filter(`Package` %nin% new_packages) |> - pull(Package)) +cran_pkgs <- unique( + tools::CRAN_package_db() |> + filter(`OS_type` != "windows" | is.na(`OS_type`)) |> + filter(`Package` %nin% new_packages) |> + pull(Package) +) arch <- "arm64" platform <- "redhat-8" @@ -28,13 +40,24 @@ platform <- "alpine-321" platform <- "ubuntu-2204" platform <- "ubuntu-2404" -data <- dbGetQuery(con, "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", params = list(platform, arch)) -removed_pkgs = get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name +data <- dbGetQuery( + con, + "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", + params = list(platform, arch) +) +removed_pkgs = get_removed_cran_packages(lubridate::interval( + lubridate::today(), + lubridate::today() - 2 +))$name pkgs_db <- unique(data$name) pkgs_db = setdiff(pkgs_db, removed_pkgs) pkgs <- setdiff(cran_pkgs, pkgs_db) -formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)]))))) +formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)])))) +) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") cat(formatted_pkgs_one_line, "\n", sep = "") @@ -44,7 +67,6 @@ clipr::write_clip(formatted_pkgs_one_line) # sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs))) - s3fs::s3_file_system( aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"), diff --git a/local/missing-cran-packages.R b/local/missing-cran-packages.R index ab71b5b..ca5cad8 100644 --- a/local/missing-cran-packages.R +++ b/local/missing-cran-packages.R @@ -10,19 +10,39 @@ s3fs::s3_file_system( ) s3fs::s3_dir_ls("s3://devxy-r-package-binaries") -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib" +)) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib" +)) -files_b=basename(files) +files_b = basename(files) files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) cran_pkgs <- tools::CRAN_package_db() |> @@ -30,7 +50,11 @@ cran_pkgs <- tools::CRAN_package_db() |> pull(Package) pkgs = setdiff(cran_pkgs, files_b_split) -formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900]))))) +formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs[1:900])))) +) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") cat(formatted_pkgs_one_line, "\n", sep = "") diff --git a/local/missing-packages-in-index.R b/local/missing-packages-in-index.R index 810c89d..6453888 100644 --- a/local/missing-packages-in-index.R +++ b/local/missing-packages-in-index.R @@ -7,42 +7,109 @@ s3fs::s3_file_system( region_name = "hel1", ) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib" +)) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib" +)) -files_b=basename(files) +files_b = basename(files) files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) -files_b_split = setdiff(files_b_split, c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive")) +files_b_split = setdiff( + files_b_split, + c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive") +) # windows-only -files_b_split = setdiff(files_b_split, c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive", "RInno", "KeyboardSimulator", "R2PPT", "RWinEdt", "blatr", "excel.link", "spectrino", "taskscheduleR", "MDSGUI", "BiplotGUI", "R2wd")) -pkgs_index = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib")[, "Package"] -pkgs_index_b=basename(pkgs_index) +files_b_split = setdiff( + files_b_split, + c( + "PACKAGES", + "PACKAGES.gz", + "PACKAGES.rds", + "PACKAGES.db", + "Archive", + "RInno", + "KeyboardSimulator", + "R2PPT", + "RWinEdt", + "blatr", + "excel.link", + "spectrino", + "taskscheduleR", + "MDSGUI", + "BiplotGUI", + "R2wd" + ) +) +pkgs_index = available.packages( + "https://cran.devxy.io/amd64/jammy/latest/src/contrib" +)[, "Package"] +pkgs_index_b = basename(pkgs_index) if (length(files_b_split) != length(pkgs_index_b)) { pkgs_missing = setdiff(files_b_split, pkgs_index_b) message("Packages missing in index but present in S3:") pkgs_missing - formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs_missing[1:900]))))) + formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs_missing[1:900])))) + ) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") cat(formatted_pkgs_one_line, "\n", sep = "") } -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib" +)) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib" +)) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 91ee96e..7a3652d 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -17,7 +17,11 @@ suppressPackageStartupMessages(library(data.table)) arch = Sys.getenv("ARCH") # target: alpine-322, ubuntu-2404, redhat-9, etc. -platform = paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-") +platform = paste( + Sys.getenv("OS"), + gsub("[.]", "", Sys.getenv("OS_VERSION")), + sep = "-" +) # Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10") codename = bincraft::set_codename(NULL) @@ -34,7 +38,9 @@ con <- DBI::dbConnect( cran_archive = tools::CRAN_archive_db() cran_release = tools::CRAN_package_db() # Subset cran_archive to only those packages -cran_archive_in_release <- cran_archive[names(cran_archive) %in% cran_release$Package] +cran_archive_in_release <- cran_archive[ + names(cran_archive) %in% cran_release$Package +] archive_versions <- rbindlist( lapply(names(cran_archive), function(pkg) { @@ -126,11 +132,15 @@ query_error <- function(pkg, ver) { # Fetch all relevant columns from the database errored_pkgs <- DBI::dbGetQuery( con, - sprintf("SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", platform, arch) + sprintf( + "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", + platform, + arch + ) ) errored_pkgs <- as.data.table(errored_pkgs) setkey(pkgs_to_build, Package, Version) -setnames(errored_pkgs, c("Package","Version")) +setnames(errored_pkgs, c("Package", "Version")) setkey(errored_pkgs, Package, Version) ### Final subsetting @@ -140,3 +150,62 @@ pkgs <- pkgs_no_error[!s3_dt] # Deduplicate pkgs <- unique(pkgs) setorder(pkgs, Package, Version) + +### R-minor sensitivity (classify once per package, applied to all versions) +source(file.path("local", "r-minor-helpers.R")) +risky_deps <- bincraft::abi_risky_linking_deps() + +release_meta <- data.table( + Package = cran_release$Package, + NeedsCompilation = cran_release$NeedsCompilation, + LinkingTo = cran_release$LinkingTo +) + +meta <- release_meta[Package %in% unique(pkgs$Package)] +meta[, + triage := mapply( + classify_from_metadata, + NeedsCompilation, + LinkingTo, + MoreArgs = list(risky_deps = risky_deps) + ) +] + +# Only the "ambiguous" compiled packages need a source grep. +ambiguous <- meta[triage == "ambiguous", Package] +sensitive_ambiguous <- character() +if (length(ambiguous) > 0L) { + tmp_src <- file.path(tempdir(), "abi_src") + dir.create(tmp_src, showWarnings = FALSE, recursive = TRUE) + sens <- vapply( + ambiguous, + function(pkg) { + out <- tryCatch( + { + dl <- utils::download.packages( + pkg, + destdir = tmp_src, + repos = "https://cloud.r-project.org", + quiet = TRUE + ) + isTRUE(as.logical(bincraft::needs_per_minor_recompile(dl[1L, 2L]))) + }, + error = function(e) TRUE + ) # fail safe: treat as sensitive + out + }, + logical(1L) + ) + sensitive_ambiguous <- ambiguous[sens] +} + +sensitive_pkgs <- unique(c( + meta[triage == "sensitive", Package], + sensitive_ambiguous +)) +pkgs[, r_minor_sensitive := Package %in% sensitive_pkgs] +sprintf( + "R-minor-sensitive packages: %s of %s", + length(sensitive_pkgs), + uniqueN(pkgs$Package) +) diff --git a/local/packages-without-any-binary.R b/local/packages-without-any-binary.R index a6994ad..f42196f 100644 --- a/local/packages-without-any-binary.R +++ b/local/packages-without-any-binary.R @@ -1,17 +1,24 @@ ### Lists packages without any successful binaries, i.e. pkgs for which all builds errored (per platform & arch) library(DBI) library(dplyr) -con <- DBI::dbConnect(RPostgres::Postgres(), - dbname = "build_metadata", host = "r-binaries.devxy.io", - port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "r_binaries", + password = Sys.getenv("PGPASS"), sslmode = "require" ) -cran_pkgs <- tools::CRAN_package_db() |> - filter(`Date/Publication` <= "2024-12-02") |> +cran_pkgs <- tools::CRAN_package_db() |> + filter(`Date/Publication` <= "2024-12-02") |> pull(Package) -data <- dbGetQuery(con, "SELECT name,platform,arch,removed,error_occurred FROM single_builds;") +data <- dbGetQuery( + con, + "SELECT name,platform,arch,removed,error_occurred FROM single_builds;" +) pkgs <- data |> filter(platform == "alpine-320", arch == "arm64") |> @@ -22,7 +29,11 @@ pkgs <- data |> pkgs = setdiff(cran_pkgs, pkgs) # Format, remove line breaks, and print as a single line -formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900]))))) +formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs[1:900])))) +) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") diff --git a/local/r-minor-helpers.R b/local/r-minor-helpers.R new file mode 100644 index 0000000..65aafdf --- /dev/null +++ b/local/r-minor-helpers.R @@ -0,0 +1,33 @@ +# Metadata-only ABI triage so the full build avoids downloading every source. +# Mirrors bincraft::abi_classify rules 1-2; "ambiguous" packages still need a +# source grep via bincraft::needs_per_minor_recompile(). + +classify_from_metadata <- function(needs_compilation, linking_to, risky_deps) { + nc <- if (length(needs_compilation) == 0L || is.na(needs_compilation)) { + "" + } else { + tolower(trimws(needs_compilation)) + } + if (!identical(nc, "yes")) { + return("not-sensitive") + } + lt <- if (length(linking_to) == 0L || is.na(linking_to)) "" else linking_to + linked <- trimws(unlist(strsplit(lt, "[,\n]"))) + linked <- sub("\\s*\\(.*\\)$", "", linked) # strip "(>= x)" constraints + linked <- linked[nzchar(linked)] + if (any(linked %in% risky_deps)) { + return("sensitive") + } + "ambiguous" +} + +parse_build_args <- function(args) { + sensitive_only <- "--sensitive-only" %in% args + pos <- args[!startsWith(args, "--")] + list( + sensitive_only = sensitive_only, + split_into = as.integer(pos[1L]), + split_index = as.integer(pos[2L]), + ncpus = as.integer(pos[3L]) + ) +} diff --git a/local/tests/test-build-all-args.R b/local/tests/test-build-all-args.R new file mode 100644 index 0000000..3308fc3 --- /dev/null +++ b/local/tests/test-build-all-args.R @@ -0,0 +1,13 @@ +source(file.path("..", "r-minor-helpers.R")) + +test_that("parse_build_args splits flags from positionals", { + a <- parse_build_args(c("--sensitive-only", "4", "2", "8")) + expect_true(a$sensitive_only) + expect_identical(a$split_into, 4L) + expect_identical(a$split_index, 2L) + expect_identical(a$ncpus, 8L) + + b <- parse_build_args(c("4", "2", "8")) + expect_false(b$sensitive_only) + expect_identical(b$split_into, 4L) +}) diff --git a/local/tests/test-r-minor-helpers.R b/local/tests/test-r-minor-helpers.R new file mode 100644 index 0000000..ac7b0a6 --- /dev/null +++ b/local/tests/test-r-minor-helpers.R @@ -0,0 +1,28 @@ +source(file.path("..", "r-minor-helpers.R")) + +test_that("pure-r (NeedsCompilation != yes) is not sensitive", { + expect_identical(classify_from_metadata("no", NA, c("Rcpp")), "not-sensitive") + expect_identical( + classify_from_metadata("", "Rcpp", c("Rcpp")), + "not-sensitive" + ) +}) + +test_that("LinkingTo a risky dep is sensitive (version constraints stripped)", { + expect_identical( + classify_from_metadata("yes", "Rcpp (>= 1.0)", c("Rcpp")), + "sensitive" + ) + expect_identical( + classify_from_metadata("yes", "R6,\n cpp11", c("Rcpp", "cpp11")), + "sensitive" + ) +}) + +test_that("compiled but no risky LinkingTo is ambiguous (needs source)", { + expect_identical( + classify_from_metadata("yes", "R6", c("Rcpp", "cpp11")), + "ambiguous" + ) + expect_identical(classify_from_metadata("yes", NA, c("Rcpp")), "ambiguous") +}) diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index c6b67e5..6b32037 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -15,10 +15,14 @@ library(httr2, quietly = TRUE) # Environment / config # --------------------------------------------------------------------------- platform <- Sys.getenv("PLATFORM") -arch <- Sys.getenv("ARCH") +arch <- Sys.getenv("ARCH") -if (nchar(platform) == 0) stop("PLATFORM env var is not set") -if (nchar(arch) == 0) stop("ARCH env var is not set") +if (nchar(platform) == 0) { + stop("PLATFORM env var is not set") +} +if (nchar(arch) == 0) { + stop("ARCH env var is not set") +} # Map platform names to S3 codenames s3_codename <- gsub("-", "", platform) @@ -37,8 +41,13 @@ os_family <- if (grepl("^ubuntu", platform)) { platform } -cat(sprintf("Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n", - platform, arch, s3_codename, os_family)) +cat(sprintf( + "Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n", + platform, + arch, + s3_codename, + os_family +)) # --------------------------------------------------------------------------- # 1. Query PostgreSQL for known build failures (before s3fs init to avoid @@ -47,19 +56,20 @@ cat(sprintf("Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n", cat("Connecting to PostgreSQL...\n") con <- DBI::dbConnect( RPostgres::Postgres(), - dbname = "build_metadata", - host = "r-binaries.devxy.io", - port = 15432, - user = "rpkgs", + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", password = Sys.getenv("PGPASS"), - sslmode = "require" + sslmode = "require" ) errored_pkgs <- DBI::dbGetQuery( con, sprintf( "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE AND platform = '%s' AND arch = '%s'", - platform, arch + platform, + arch ) ) DBI::dbDisconnect(con) @@ -69,7 +79,12 @@ if (nrow(errored_dt) > 0) { setnames(errored_dt, c("Package", "Version")) setkey(errored_dt, Package, Version) } -cat(sprintf("Found %d known build failures for %s/%s\n", nrow(errored_dt), platform, arch)) +cat(sprintf( + "Found %d known build failures for %s/%s\n", + nrow(errored_dt), + platform, + arch +)) # --------------------------------------------------------------------------- # 2. CRAN release packages (uses curl internally - must come after PG) @@ -86,28 +101,36 @@ cran_dt <- data.table( # --------------------------------------------------------------------------- cat("Connecting to S3...\n") s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE ) -s3_path <- sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, s3_codename) +s3_path <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", + arch, + s3_codename +) cat(sprintf("Listing S3 path: %s\n", s3_path)) s3_pkgs <- tryCatch( s3fs::s3_dir_ls(s3_path, recurse = FALSE), error = function(e) { - cat(sprintf("WARNING: Could not list S3 path %s: %s\n", s3_path, conditionMessage(e))) + cat(sprintf( + "WARNING: Could not list S3 path %s: %s\n", + s3_path, + conditionMessage(e) + )) character(0) } ) file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) -parts <- parts[sapply(parts, length) == 3] +matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) +parts <- regmatches(file_names, matches) +parts <- parts[sapply(parts, length) == 3] if (length(parts) == 0) { s3_dt <- data.table(Package = character(0), Version = character(0)) } else { @@ -117,13 +140,18 @@ if (length(parts) == 0) { ) } -cat(sprintf("S3 contains %d tarballs for %s/%s\n", nrow(s3_dt), arch, s3_codename)) +cat(sprintf( + "S3 contains %d tarballs for %s/%s\n", + nrow(s3_dt), + arch, + s3_codename +)) # --------------------------------------------------------------------------- # 4. Find missing packages (CRAN release version not in S3) # --------------------------------------------------------------------------- setkey(cran_dt, Package, Version) -setkey(s3_dt, Package, Version) +setkey(s3_dt, Package, Version) missing_dt <- cran_dt[!s3_dt] cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) @@ -132,9 +160,15 @@ cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) # --------------------------------------------------------------------------- script_dir <- tryCatch( dirname(normalizePath( - if (exists("ofile", envir = sys.frame(1), inherits = FALSE)) sys.frame(1)$ofile - else commandArgs(trailingOnly = FALSE)[grepl("--file=", commandArgs(trailingOnly = FALSE))] |> - sub("--file=", "", x = _), + if (exists("ofile", envir = sys.frame(1), inherits = FALSE)) { + sys.frame(1)$ofile + } else { + commandArgs(trailingOnly = FALSE)[grepl( + "--file=", + commandArgs(trailingOnly = FALSE) + )] |> + sub("--file=", "", x = _) + }, mustWork = FALSE )), error = function(e) "local" @@ -143,13 +177,23 @@ excluded_path <- file.path(script_dir, "excluded-packages.json") excluded_pkgs <- data.table(package = character(0), reason = character(0)) if (file.exists(excluded_path)) { - excluded_raw <- jsonlite::fromJSON(excluded_path) + excluded_raw <- jsonlite::fromJSON(excluded_path) excluded_pkgs <- as.data.table(excluded_raw) - cat(sprintf("Loaded %d excluded packages from %s\n", nrow(excluded_pkgs), excluded_path)) + cat(sprintf( + "Loaded %d excluded packages from %s\n", + nrow(excluded_pkgs), + excluded_path + )) missing_dt <- missing_dt[!Package %in% excluded_pkgs$package] - cat(sprintf("%d packages remain after removing exclusions\n", nrow(missing_dt))) + cat(sprintf( + "%d packages remain after removing exclusions\n", + nrow(missing_dt) + )) } else { - cat(sprintf("No excluded-packages.json found at %s -- skipping exclusion step\n", excluded_path)) + cat(sprintf( + "No excluded-packages.json found at %s -- skipping exclusion step\n", + excluded_path + )) } # --------------------------------------------------------------------------- @@ -157,24 +201,34 @@ if (file.exists(excluded_path)) { # --------------------------------------------------------------------------- if (nrow(errored_dt) > 0) { known_failures_dt <- missing_dt[errored_dt, nomatch = 0] - rebuildable_dt <- missing_dt[!errored_dt] + rebuildable_dt <- missing_dt[!errored_dt] } else { known_failures_dt <- missing_dt[0] - rebuildable_dt <- missing_dt + rebuildable_dt <- missing_dt } -cat(sprintf("Rebuildable: %d | Known failures: %d\n", - nrow(rebuildable_dt), nrow(known_failures_dt))) +cat(sprintf( + "Rebuildable: %d | Known failures: %d\n", + nrow(rebuildable_dt), + nrow(known_failures_dt) +)) # --------------------------------------------------------------------------- # 7. Write rebuildable package names to cache RDS # --------------------------------------------------------------------------- -cache_dir <- "/mnt/cache/packages" -cache_file <- file.path(cache_dir, sprintf("weekly_rebuild_%s_%s.rds", platform, arch)) +cache_dir <- "/mnt/cache/packages" +cache_file <- file.path( + cache_dir, + sprintf("weekly_rebuild_%s_%s.rds", platform, arch) +) if (dir.exists(cache_dir)) { saveRDS(rebuildable_dt$Package, cache_file) - cat(sprintf("Wrote %d rebuildable packages to %s\n", nrow(rebuildable_dt), cache_file)) + cat(sprintf( + "Wrote %d rebuildable packages to %s\n", + nrow(rebuildable_dt), + cache_file + )) } else { cat(sprintf("Cache dir %s does not exist -- skipping RDS write\n", cache_dir)) } @@ -186,29 +240,47 @@ forgejo_token <- Sys.getenv("FORGEJO_TOKEN") if (nchar(forgejo_token) == 0) { cat("FORGEJO_TOKEN not set -- skipping issue update\n") } else { - issue_title <- sprintf("Missing package binaries for latest version (%s)", os_family) + issue_title <- sprintf( + "Missing package binaries for latest version (%s)", + os_family + ) - n_missing <- nrow(missing_dt) - n_rebuild <- nrow(rebuildable_dt) + n_missing <- nrow(missing_dt) + n_rebuild <- nrow(rebuildable_dt) - arch_lines <- sprintf("### %s (%d missing, %d to rebuild)", arch, n_missing, n_rebuild) + arch_lines <- sprintf( + "### %s (%d missing, %d to rebuild)", + arch, + n_missing, + n_rebuild + ) if (nrow(rebuildable_dt) > 0) { - arch_lines <- c(arch_lines, + arch_lines <- c( + arch_lines, paste0("- ", rebuildable_dt$Package, " (", rebuildable_dt$Version, ")") ) } else { arch_lines <- c(arch_lines, "_None_") } if (nrow(known_failures_dt) > 0) { - arch_lines <- c(arch_lines, + arch_lines <- c( + arch_lines, "", "#### Known build failures", - paste0("- ", known_failures_dt$Package, " (", known_failures_dt$Version, ")") + paste0( + "- ", + known_failures_dt$Package, + " (", + known_failures_dt$Version, + ")" + ) ) } build_excluded_footer <- function() { - if (nrow(excluded_pkgs) == 0) return(character(0)) + if (nrow(excluded_pkgs) == 0) { + return(character(0)) + } entries <- paste( paste0(excluded_pkgs$package, " (", excluded_pkgs$reason, ")"), collapse = ", " @@ -217,11 +289,12 @@ if (nchar(forgejo_token) == 0) { } forgejo_base <- "https://git.devxy.io/api/v1" - repo <- "devxy/build-cran-binaries" + repo <- "devxy/build-cran-binaries" search_url <- sprintf( "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", - forgejo_base, repo, + forgejo_base, + repo, utils::URLencode(issue_title, reserved = TRUE) ) search_resp <- httr2::request(search_url) |> @@ -230,15 +303,19 @@ if (nchar(forgejo_token) == 0) { existing_issues <- httr2::resp_body_json(search_resp, simplifyVector = FALSE) - match_idx <- which(sapply(existing_issues, function(x) x$title) == issue_title) + match_idx <- which( + sapply(existing_issues, function(x) x$title) == issue_title + ) today_str <- format(Sys.Date(), "%Y-%m-%d") if (length(match_idx) > 0) { # ---- Update existing issue ---- issue_number <- existing_issues[[match_idx[1]]]$number - old_body <- existing_issues[[match_idx[1]]]$body - if (is.null(old_body)) old_body <- "" + old_body <- existing_issues[[match_idx[1]]]$body + if (is.null(old_body)) { + old_body <- "" + } lines <- strsplit(old_body, "\n", fixed = TRUE)[[1]] @@ -258,7 +335,11 @@ if (nchar(forgejo_token) == 0) { if (length(plat_idx) == 0) { # Platform section missing -- insert before --- footer footer_idx <- which(lines == "---") - insert_at <- if (length(footer_idx) > 0) footer_idx[length(footer_idx)] else length(lines) + 1 + insert_at <- if (length(footer_idx) > 0) { + footer_idx[length(footer_idx)] + } else { + length(lines) + 1 + } new_plat_block <- c(sprintf("## %s", platform), "", arch_lines, "") lines <- c( @@ -271,10 +352,14 @@ if (nchar(forgejo_token) == 0) { # End of platform section: next ## or --- at a higher level, or EOF next_section <- which(grepl("^## |^---", lines) & seq_along(lines) > pi) - plat_end <- if (length(next_section) > 0) next_section[1] - 1 else length(lines) + plat_end <- if (length(next_section) > 0) { + next_section[1] - 1 + } else { + length(lines) + } - plat_lines <- lines[seq(pi, plat_end)] - arch_local_idx <- which(plat_lines == arch_header) + plat_lines <- lines[seq(pi, plat_end)] + arch_local_idx <- which(plat_lines == arch_header) if (length(arch_local_idx) == 0) { # Append arch subsection at end of platform block @@ -285,11 +370,13 @@ if (nchar(forgejo_token) == 0) { lines[seq(plat_end + 1, length(lines))] ) } else { - ai <- pi + arch_local_idx[1] - 1 # absolute line index + ai <- pi + arch_local_idx[1] - 1 # absolute line index # End of arch subsection - next_arch <- which(grepl("^### |^## |^---", lines) & seq_along(lines) > ai) - arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end + next_arch <- which( + grepl("^### |^## |^---", lines) & seq_along(lines) > ai + ) + arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end lines <- c( lines[seq_len(ai - 1)], @@ -302,8 +389,12 @@ if (nchar(forgejo_token) == 0) { # Rebuild excluded footer excl_hdr_idx <- which(lines == "## Excluded packages") if (length(excl_hdr_idx) > 0) { - pre_dash <- which(lines == "---" & seq_along(lines) < excl_hdr_idx[1]) - remove_from <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_hdr_idx[1] + pre_dash <- which(lines == "---" & seq_along(lines) < excl_hdr_idx[1]) + remove_from <- if (length(pre_dash) > 0) { + pre_dash[length(pre_dash)] + } else { + excl_hdr_idx[1] + } lines <- lines[seq_len(remove_from - 1)] } footer <- build_excluded_footer() @@ -313,10 +404,15 @@ if (nchar(forgejo_token) == 0) { new_body <- paste(lines, collapse = "\n") - patch_url <- sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, issue_number) + patch_url <- sprintf( + "%s/repos/%s/issues/%d", + forgejo_base, + repo, + issue_number + ) httr2::request(patch_url) |> httr2::req_headers( - Authorization = paste("token", forgejo_token), + Authorization = paste("token", forgejo_token), `Content-Type` = "application/json" ) |> httr2::req_body_json(list(body = new_body)) |> @@ -324,7 +420,6 @@ if (nchar(forgejo_token) == 0) { httr2::req_perform() cat(sprintf("Updated Forgejo issue #%d: %s\n", issue_number, issue_title)) - } else { # ---- Create new issue ---- body_lines <- c( @@ -342,17 +437,21 @@ if (nchar(forgejo_token) == 0) { post_url <- sprintf("%s/repos/%s/issues", forgejo_base, repo) create_resp <- httr2::request(post_url) |> httr2::req_headers( - Authorization = paste("token", forgejo_token), + Authorization = paste("token", forgejo_token), `Content-Type` = "application/json" ) |> httr2::req_body_json(list( title = issue_title, - body = paste(body_lines, collapse = "\n") + body = paste(body_lines, collapse = "\n") )) |> httr2::req_perform() new_issue <- httr2::resp_body_json(create_resp) - cat(sprintf("Created Forgejo issue #%d: %s\n", new_issue$number, issue_title)) + cat(sprintf( + "Created Forgejo issue #%d: %s\n", + new_issue$number, + issue_title + )) } } diff --git a/renovate.json b/renovate.json index 97ca6ad..53c7306 100644 --- a/renovate.json +++ b/renovate.json @@ -1,11 +1,7 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "local>devxy/renovate-config" - ], - "ignorePaths": [ - "docker/**" - ], + "extends": ["local>devxy/renovate-config"], + "ignorePaths": ["docker/**"], "customManagers": [ { "customType": "regex", From 1224bd74fe6ac1fa4c649f1d477b678ec8861d2e Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 09:03:45 +0000 Subject: [PATCH 015/125] feat(local): just rebuild recipe for targeted builds via remote buildx (#87) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds a local `just rebuild` recipe to (re)build specific versions of a single package on a given OS/arch, dispatching to a remote buildx builder (the build runs there, not locally). - `just rebuild ...` → `docker buildx build --builder --platform linux/ …` (amd64→artemis, arm64→gaia; names + `R_VERSION` env-overridable). - `docker/build-one.Dockerfile` runs `build-one.R` as a secret-mounted `RUN`, built `--no-cache --output type=cacheonly` (pure side-effect: the S3 upload; no image kept). - `local/build-one.R` auto-classifies each version via the ABI classifier (risky → per-minor slot `contrib//`, else generic), force-rebuilds + uploads + stores metadata, then refreshes the touched slot's `PACKAGES` index. ## Prerequisites - buildx builders named `artemis` (amd64) and `gaia` (arm64) registered (`docker buildx create --name artemis ssh://…`). - Exported secrets: `B2_S3_ACCESS_KEY`, `B2_S3_SECRET_KEY`, `PGPASS` (`GITHUB_PAT` optional). - bincraft `v4.2.0` tag must exist (the build installs `@v4.2.0` and uses its classifier + per-minor index API). Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/87 --- docker/build-one.Dockerfile | 32 +++++++++ justfile | 62 ++++++++++++++++++ local/build-one.R | 126 ++++++++++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 docker/build-one.Dockerfile create mode 100644 justfile create mode 100644 local/build-one.R diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile new file mode 100644 index 0000000..ddfd069 --- /dev/null +++ b/docker/build-one.Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1 +# Targeted (re)build of specific package versions, executed on a remote buildx +# builder. The build's effect is the S3 upload performed by build-one.R; no +# image is kept (the justfile uses --output type=cacheonly). CACHEBUST forces +# the RUN to re-execute on every invocation. +# +# Build context is `local/` (see the `rebuild` recipe in the justfile). +ARG OS +ARG OS_VERSION +FROM reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION} + +ARG R_VERSION=4.5.3 +ARG PACKAGE +ARG VERSIONS +ARG CACHEBUST + +WORKDIR /work +COPY build-one.R /work/build-one.R + +RUN --mount=type=secret,id=b2_access,required=true \ + --mount=type=secret,id=b2_secret,required=true \ + --mount=type=secret,id=pgpass,required=true \ + --mount=type=secret,id=github_pat,required=false \ + export B2_S3_ACCESS_KEY="$(cat /run/secrets/b2_access)" && \ + export B2_S3_SECRET_KEY="$(cat /run/secrets/b2_secret)" && \ + export PGPASS="$(cat /run/secrets/pgpass)" && \ + export GITHUB_PAT="$(cat /run/secrets/github_pat 2>/dev/null || true)" && \ + /opt/R/${R_VERSION}/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' && \ + XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); \ + XVFB_ARGS=""; \ + if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi; \ + $XVFB $XVFB_ARGS -- /opt/R/${R_VERSION}/bin/Rscript /work/build-one.R "${PACKAGE}" ${VERSIONS} diff --git a/justfile b/justfile new file mode 100644 index 0000000..a475505 --- /dev/null +++ b/justfile @@ -0,0 +1,62 @@ +# Local helpers for build-cran-binaries. +# +# `rebuild` (re)builds specific versions of a single package on a given OS/arch +# by dispatching to a remote buildx builder (the build runs there, not locally). +# Sensitivity is auto-detected: risky packages land in the per-minor slot +# (contrib//), everything else in the generic slot; the touched index is +# refreshed so the result is immediately servable. +# +# Prerequisites: +# - buildx builders named `artemis` (amd64) and `gaia` (arm64), e.g. +# docker buildx create --name artemis --node artemis ssh:// +# docker buildx create --name gaia --node gaia ssh:// +# - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional) +# +# Overridable (env or `just VAR=… rebuild …`): +# R_VERSION (default 4.5.3) — selects the R minor → the per-minor slot +# AMD64_BUILDER / ARM64_BUILDER — buildx builder names +# +# Examples: +# just rebuild alpine 3.23 amd64 rlang 1.1.4 1.1.3 +# just rebuild redhat 10 arm64 data.table 1.15.4 +# R_VERSION=4.4.3 just rebuild ubuntu noble amd64 Rcpp 1.0.12 + +r_version := env_var_or_default("R_VERSION", "4.5.3") +amd64_builder := env_var_or_default("AMD64_BUILDER", "artemis") +arm64_builder := env_var_or_default("ARM64_BUILDER", "gaia") + +# (re)build PACKAGE at one or more VERSIONS on OS/TAG/ARCH via a remote builder +rebuild os tag arch package *versions: + #!/usr/bin/env bash + set -euo pipefail + if [ -z "{{ versions }}" ]; then + echo "error: provide at least one version, e.g. just rebuild alpine 3.23 amd64 rlang 1.1.4" >&2 + exit 1 + fi + case "{{ arch }}" in + amd64) builder="{{ amd64_builder }}" ;; + arm64) builder="{{ arm64_builder }}" ;; + *) echo "error: arch must be 'amd64' or 'arm64'" >&2; exit 1 ;; + esac + : "${B2_S3_ACCESS_KEY:?set B2_S3_ACCESS_KEY in your environment}" + : "${B2_S3_SECRET_KEY:?set B2_S3_SECRET_KEY in your environment}" + : "${PGPASS:?set PGPASS in your environment}" + : "${GITHUB_PAT:=}" + echo "Dispatching build of {{ package }} ({{ versions }}) on {{ os }}:{{ tag }}/{{ arch }} (R {{ r_version }}) to builder '$builder'" + docker buildx build \ + --builder "$builder" \ + --platform "linux/{{ arch }}" \ + --no-cache \ + --output type=cacheonly \ + --secret id=b2_access,env=B2_S3_ACCESS_KEY \ + --secret id=b2_secret,env=B2_S3_SECRET_KEY \ + --secret id=pgpass,env=PGPASS \ + --secret id=github_pat,env=GITHUB_PAT \ + --build-arg OS="{{ os }}" \ + --build-arg OS_VERSION="{{ tag }}" \ + --build-arg R_VERSION="{{ r_version }}" \ + --build-arg PACKAGE="{{ package }}" \ + --build-arg VERSIONS="{{ versions }}" \ + --build-arg CACHEBUST="$(date +%s)" \ + -f docker/build-one.Dockerfile \ + local diff --git a/local/build-one.R b/local/build-one.R new file mode 100644 index 0000000..d67a206 --- /dev/null +++ b/local/build-one.R @@ -0,0 +1,126 @@ +# Targeted (re)build of specific versions of a single package. +# Invoked inside a build-env container (see docker/build-one.Dockerfile). +# Usage: build-one.R [ ...] +# Sensitivity is auto-detected per version via bincraft's ABI classifier: +# risky packages go to the per-minor slot, everything else to the generic slot. + +sink(stdout(), type = "message") +options(crayon.enabled = TRUE, future.globals.onReference = NULL) + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2L) { + stop("usage: build-one.R [ ...]", call. = FALSE) +} +package <- args[1L] +versions <- args[-1L] + +library(bincraft, quietly = TRUE) + +s3 <- list( + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") +) + +# Clone the CRAN source for a version and ask the ABI classifier whether it +# must be rebuilt per R minor. Fails safe to TRUE so a possibly-fragile binary +# is never served from the cross-minor generic slot by mistake. +classify <- function(pkg, ver) { + dest <- file.path(tempdir(), sprintf("classify_%s_%s", pkg, ver)) + on.exit(unlink(dest, recursive = TRUE, force = TRUE), add = TRUE) + tryCatch( + { + system2( + "git", + c( + "clone", + "-q", + "--branch", + ver, + sprintf("https://github.com/cran/%s", pkg), + dest + ) + ) + isTRUE(as.logical(bincraft::needs_per_minor_recompile(dest))) + }, + error = function(e) { + message(sprintf( + "classify failed for %s %s: %s; treating as r-minor-sensitive", + pkg, + ver, + conditionMessage(e) + )) + TRUE + } + ) +} + +minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) +touched_generic <- FALSE +touched_minor <- FALSE + +for (ver in versions) { + sensitive <- classify(package, ver) + cat(sprintf( + "Building %s %s (r_minor_sensitive=%s, R %s)\n", + package, + ver, + sensitive, + minor + )) + bincraft::build_binary_package( + package, + tag = ver, + is_r_minor_sensitive = sensitive, + force = TRUE, + upload = TRUE, + archive = TRUE, + store_build_metadata = TRUE, + s3_endpoint = s3$s3_endpoint, + s3_region = s3$s3_region, + s3_bucket = s3$s3_bucket, + s3_access_key_id = s3$s3_access_key_id, + s3_secret_access_key = s3$s3_secret_access_key, + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432 + ) + if (sensitive) touched_minor <- TRUE else touched_generic <- TRUE +} + +# Refresh the PACKAGES index for each slot we wrote to, so the (re)built binary +# is immediately resolvable by clients. +codename <- bincraft::set_codename(NULL) +if (touched_generic) { + cat("Refreshing generic index\n") + bincraft::upload_package_index( + codename = codename, + s3_endpoint = s3$s3_endpoint, + s3_region = s3$s3_region, + s3_bucket = s3$s3_bucket, + s3_access_key_id = s3$s3_access_key_id, + s3_secret_access_key = s3$s3_secret_access_key + ) +} +if (touched_minor) { + cat(sprintf("Refreshing per-minor index %s\n", minor)) + bincraft::upload_package_index( + codename = codename, + r_minor = minor, + s3_endpoint = s3$s3_endpoint, + s3_region = s3$s3_region, + s3_bucket = s3$s3_bucket, + s3_access_key_id = s3$s3_access_key_id, + s3_secret_access_key = s3$s3_secret_access_key + ) +} From 7f5674ffb3e9020440960c32058024c7ae225460 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 09:08:48 +0000 Subject: [PATCH 016/125] fix(local): accept TF_VAR_-prefixed B2 keys, optional GITHUB_PAT (#88) ## Summary Follow-up fix to the `just rebuild` recipe (#87): the local `.envrc` exports the B2 credentials as `TF_VAR_B2_S3_ACCESS_KEY` / `TF_VAR_B2_S3_SECRET_KEY` (and has no `GITHUB_PAT`), but the recipe required the plain `B2_S3_ACCESS_KEY` / `B2_S3_SECRET_KEY` names and always passed a `github_pat` secret. - Accept the `TF_VAR_`-prefixed B2 names (falling back to the plain names if set directly). - Pass the `github_pat` buildx secret only when `GITHUB_PAT` is set, so the build works without it. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/88 --- justfile | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/justfile b/justfile index a475505..06f2200 100644 --- a/justfile +++ b/justfile @@ -38,20 +38,28 @@ rebuild os tag arch package *versions: arm64) builder="{{ arm64_builder }}" ;; *) echo "error: arch must be 'amd64' or 'arm64'" >&2; exit 1 ;; esac - : "${B2_S3_ACCESS_KEY:?set B2_S3_ACCESS_KEY in your environment}" - : "${B2_S3_SECRET_KEY:?set B2_S3_SECRET_KEY in your environment}" - : "${PGPASS:?set PGPASS in your environment}" - : "${GITHUB_PAT:=}" + # Accept TF_VAR_-prefixed names (direnv) or plain names. + export B2_S3_ACCESS_KEY="${B2_S3_ACCESS_KEY:-${TF_VAR_B2_S3_ACCESS_KEY:-}}" + export B2_S3_SECRET_KEY="${B2_S3_SECRET_KEY:-${TF_VAR_B2_S3_SECRET_KEY:-}}" + : "${B2_S3_ACCESS_KEY:?set B2_S3_ACCESS_KEY or TF_VAR_B2_S3_ACCESS_KEY}" + : "${B2_S3_SECRET_KEY:?set B2_S3_SECRET_KEY or TF_VAR_B2_S3_SECRET_KEY}" + : "${PGPASS:?set PGPASS}" + secret_args=( + --secret id=b2_access,env=B2_S3_ACCESS_KEY + --secret id=b2_secret,env=B2_S3_SECRET_KEY + --secret id=pgpass,env=PGPASS + ) + # GITHUB_PAT is optional (raises the GitHub API rate limit); pass only if set. + if [ -n "${GITHUB_PAT:-}" ]; then + secret_args+=(--secret id=github_pat,env=GITHUB_PAT) + fi echo "Dispatching build of {{ package }} ({{ versions }}) on {{ os }}:{{ tag }}/{{ arch }} (R {{ r_version }}) to builder '$builder'" docker buildx build \ --builder "$builder" \ --platform "linux/{{ arch }}" \ --no-cache \ --output type=cacheonly \ - --secret id=b2_access,env=B2_S3_ACCESS_KEY \ - --secret id=b2_secret,env=B2_S3_SECRET_KEY \ - --secret id=pgpass,env=PGPASS \ - --secret id=github_pat,env=GITHUB_PAT \ + "${secret_args[@]}" \ --build-arg OS="{{ os }}" \ --build-arg OS_VERSION="{{ tag }}" \ --build-arg R_VERSION="{{ r_version }}" \ From f51617018dcd485536f2e6b86588ebc29079e5de Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 13:02:47 +0200 Subject: [PATCH 017/125] ci: bump bincraft to 4.2.1 and scope multi-R loops to versioned R dirs - Pin bincraft v4.2.0 -> v4.2.1 (per-minor archive.rds/index fix) across the process-updates, weekly-rebuild and archive workflows. - Restrict the per-R-minor build and index loops to /opt/R/[0-9]* so the `current` symlink dir is not treated as a distinct minor (avoids double- building a minor and writing a bogus contrib/current/ index slot). --- .crow/archive-missed-packages.yaml | 2 +- .crow/build-all-versions-amd64.yaml | 4 ++-- .crow/build-all-versions-arm64.yaml | 4 ++-- .crow/process-updates-alpine-322-amd64.yaml | 6 +++--- .crow/process-updates-alpine-322-arm64.yaml | 6 +++--- .crow/process-updates-alpine-323-amd64.yaml | 6 +++--- .crow/process-updates-alpine-323-arm64.yaml | 6 +++--- .crow/process-updates-redhat-10-amd64.yaml | 6 +++--- .crow/process-updates-redhat-10-arm64.yaml | 6 +++--- .crow/process-updates-redhat-8-amd64.yaml | 6 +++--- .crow/process-updates-redhat-8-arm64.yaml | 6 +++--- .crow/process-updates-redhat-9-amd64.yaml | 6 +++--- .crow/process-updates-redhat-9-arm64.yaml | 6 +++--- .crow/process-updates-ubuntu-2204-amd64.yaml | 6 +++--- .crow/process-updates-ubuntu-2204-arm64.yaml | 6 +++--- .crow/process-updates-ubuntu-2404-amd64.yaml | 6 +++--- .crow/process-updates-ubuntu-2404-arm64.yaml | 6 +++--- .crow/weekly-rebuild-missing-alpine-322-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-322-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-323-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-alpine-323-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-10-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-10-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-8-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-8-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-9-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-redhat-9-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml | 2 +- .crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml | 2 +- 31 files changed, 61 insertions(+), 61 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index e18a457..0350f52 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,7 +62,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), workers = 2L)' backend_options: diff --git a/.crow/build-all-versions-amd64.yaml b/.crow/build-all-versions-amd64.yaml index 0c74ef7..5d225b6 100644 --- a/.crow/build-all-versions-amd64.yaml +++ b/.crow/build-all-versions-amd64.yaml @@ -72,7 +72,7 @@ steps: - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -121,7 +121,7 @@ steps: - | CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e "cat(bincraft::set_codename(NULL))") /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/build-all-versions-arm64.yaml b/.crow/build-all-versions-arm64.yaml index aac08c3..6700157 100644 --- a/.crow/build-all-versions-arm64.yaml +++ b/.crow/build-all-versions-arm64.yaml @@ -85,7 +85,7 @@ steps: - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -134,7 +134,7 @@ steps: - | CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e "cat(bincraft::set_codename(NULL))") /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index d1f67c3..30732ce 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -52,14 +52,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -68,7 +68,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine322', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index 617ed42..45dd475 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -51,14 +51,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -67,7 +67,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine322', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index 86bfc10..119a825 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -52,14 +52,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -68,7 +68,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine323', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index d8f9260..c8169d4 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -51,14 +51,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -67,7 +67,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine323', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-redhat-10-amd64.yaml b/.crow/process-updates-redhat-10-amd64.yaml index 307c814..7ff3208 100644 --- a/.crow/process-updates-redhat-10-amd64.yaml +++ b/.crow/process-updates-redhat-10-amd64.yaml @@ -53,7 +53,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -61,7 +61,7 @@ steps: - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -70,7 +70,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel10', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-redhat-10-arm64.yaml b/.crow/process-updates-redhat-10-arm64.yaml index e41fd03..1604c9e 100644 --- a/.crow/process-updates-redhat-10-arm64.yaml +++ b/.crow/process-updates-redhat-10-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -59,7 +59,7 @@ steps: - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -68,7 +68,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel10', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml index b1ce39d..0381706 100644 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ b/.crow/process-updates-redhat-8-amd64.yaml @@ -52,14 +52,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -68,7 +68,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel8', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml index e5f8b18..cc10f00 100644 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ b/.crow/process-updates-redhat-8-arm64.yaml @@ -51,14 +51,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -67,7 +67,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel8', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml index b0525c3..128b5d1 100644 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ b/.crow/process-updates-redhat-9-amd64.yaml @@ -53,14 +53,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -69,7 +69,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel9', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml index 870705a..89db00b 100644 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ b/.crow/process-updates-redhat-9-arm64.yaml @@ -51,14 +51,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -67,7 +67,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel9', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml index a7aad25..13ec7fb 100644 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ b/.crow/process-updates-ubuntu-2204-amd64.yaml @@ -53,14 +53,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -69,7 +69,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'jammy', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml index 4126add..99bcdc2 100644 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ b/.crow/process-updates-ubuntu-2204-arm64.yaml @@ -51,14 +51,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -67,7 +67,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'jammy', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml index 954f0ab..a9f3b7e 100644 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ b/.crow/process-updates-ubuntu-2404-amd64.yaml @@ -52,14 +52,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -68,7 +68,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'noble', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml index 0ced70a..81f1244 100644 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ b/.crow/process-updates-ubuntu-2404-arm64.yaml @@ -51,14 +51,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RV=$(basename "$(dirname "$(dirname "$RBIN")")") RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue @@ -67,7 +67,7 @@ steps: done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | - for RBIN in /opt/R/*/bin/R; do + for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'noble', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done diff --git a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml index cb395d9..a49a52c 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml index 1651b4e..9b1b91d 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml index 94d15af..821a8c0 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml index 38bc778..ddc9021 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml index 87e21f0..cfe45c1 100644 --- a/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml index 2b2cf88..763d919 100644 --- a/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml index 4ef638e..2278fc0 100644 --- a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml index d02673b..18edc0b 100644 --- a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml index c857116..f463e37 100644 --- a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml index c7968ba..1f997f9 100644 --- a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml index 6a2d625..34493b5 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml index d14c278..4d006b0 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml index e9002f8..e308b0f 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml index 1f17c45..bcacaef 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' From e5398af5f5e9a0d54b10dab0d030a9f2cfb2e554 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 13:02:47 +0200 Subject: [PATCH 018/125] =?UTF-8?q?feat(local):=20just=20rebuild=20?= =?UTF-8?q?=E2=80=94=20TF=5FVAR=20secrets,=20live=20progress,=20builder=20?= =?UTF-8?q?docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Accept TF_VAR_-prefixed B2 keys (from direnv); make GITHUB_PAT optional. - Stream the remote build with --progress=plain. - Document creating the artemis/gaia buildx builders with the docker-container driver (the default remote driver does not work with an ssh:// docker host). --- justfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/justfile b/justfile index 06f2200..2d9388d 100644 --- a/justfile +++ b/justfile @@ -7,9 +7,11 @@ # refreshed so the result is immediately servable. # # Prerequisites: -# - buildx builders named `artemis` (amd64) and `gaia` (arm64), e.g. -# docker buildx create --name artemis --node artemis ssh:// -# docker buildx create --name gaia --node gaia ssh:// +# - buildx builders named `artemis` (amd64) and `gaia` (arm64), created with the +# docker-container driver (runs BuildKit on the remote host's docker daemon over +# SSH). The default `remote` driver does NOT work with an ssh:// docker host. +# docker buildx create --name artemis --driver docker-container ssh:// +# docker buildx create --name gaia --driver docker-container ssh:// # - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional) # # Overridable (env or `just VAR=… rebuild …`): @@ -58,6 +60,7 @@ rebuild os tag arch package *versions: --builder "$builder" \ --platform "linux/{{ arch }}" \ --no-cache \ + --progress=plain \ --output type=cacheonly \ "${secret_args[@]}" \ --build-arg OS="{{ os }}" \ From 3ae4672d66cf8b51292bf7a3fb631b1e97c8dede Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 13:02:48 +0200 Subject: [PATCH 019/125] feat(local): just rebuild builds across all installed R minors - Dockerfile: install bincraft into each R minor, run a primary pass plus a sensitive-only pass under every other /opt/R/[0-9]* minor (deduped by minor), probe/skip xvfb, set GIT_TERMINAL_PROMPT=0; extra-minor failures are non-fatal. - build-one.R: add --sensitive-only mode, log + shallow-clone the ABI classify step, and clear cranlike's stale ./PACKAGES.db before each index refresh (workaround for the "table packages already exists" bug; pending cranlike fix). --- docker/build-one.Dockerfile | 34 ++++++++++++++++++++++++++++++--- local/build-one.R | 38 +++++++++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index ddfd069..1884a44 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -25,8 +25,36 @@ RUN --mount=type=secret,id=b2_access,required=true \ export B2_S3_SECRET_KEY="$(cat /run/secrets/b2_secret)" && \ export PGPASS="$(cat /run/secrets/pgpass)" && \ export GITHUB_PAT="$(cat /run/secrets/github_pat 2>/dev/null || true)" && \ - /opt/R/${R_VERSION}/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' && \ - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); \ + export GIT_TERMINAL_PROMPT=0 && \ + export OTEL_SDK_DISABLED=true && \ + XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run || true); \ XVFB_ARGS=""; \ if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi; \ - $XVFB $XVFB_ARGS -- /opt/R/${R_VERSION}/bin/Rscript /work/build-one.R "${PACKAGE}" ${VERSIONS} + USE_XVFB=0; \ + if [ -n "$XVFB" ] && $XVFB -a $XVFB_ARGS -- true >/dev/null 2>&1; then \ + USE_XVFB=1; echo "Using virtual display via $XVFB"; \ + else \ + echo "No working virtual display; building without xvfb" >&2; \ + fi; \ + run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \ + ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")'; }; \ + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ + seen=" $PRIMARY_MINOR "; \ + prc=0; failed=""; \ + echo "=== primary pass under R $R_VERSION ==="; \ + ensure_bincraft /opt/R/${R_VERSION}/bin/R; \ + run_build /opt/R/${R_VERSION}/bin/Rscript /work/build-one.R "${PACKAGE}" ${VERSIONS} || prc=$?; \ + for RBIN in /opt/R/*/bin/Rscript; do \ + [ -x "$RBIN" ] || continue; \ + RV=$(basename "$(dirname "$(dirname "$RBIN")")"); \ + case "$RV" in [0-9]*) ;; *) continue ;; esac; \ + RMINOR=$(echo "$RV" | cut -d. -f1-2); \ + case "$seen" in *" $RMINOR "*) continue ;; esac; \ + seen="$seen$RMINOR "; \ + echo "=== sensitive-only pass under R $RV ==="; \ + ensure_bincraft "$(dirname "$RBIN")/R"; \ + run_build "$RBIN" /work/build-one.R --sensitive-only "${PACKAGE}" ${VERSIONS} || failed="$failed $RV"; \ + done; \ + if [ -n "$failed" ]; then echo "WARNING: extra-minor build(s) failed (non-fatal; e.g. a version too old to compile on a newer R):$failed" >&2; fi; \ + if [ "$prc" != 0 ]; then echo "primary pass under R $R_VERSION failed (exit $prc)" >&2; fi; \ + exit "$prc" diff --git a/local/build-one.R b/local/build-one.R index d67a206..5531124 100644 --- a/local/build-one.R +++ b/local/build-one.R @@ -1,15 +1,21 @@ # Targeted (re)build of specific versions of a single package. # Invoked inside a build-env container (see docker/build-one.Dockerfile). -# Usage: build-one.R [ ...] +# Usage: build-one.R [--sensitive-only] [ ...] # Sensitivity is auto-detected per version via bincraft's ABI classifier: # risky packages go to the per-minor slot, everything else to the generic slot. +# With --sensitive-only, non-risky versions are skipped (used for the extra +# per-minor passes under non-primary R versions). -sink(stdout(), type = "message") options(crayon.enabled = TRUE, future.globals.onReference = NULL) args <- commandArgs(trailingOnly = TRUE) +sensitive_only <- "--sensitive-only" %in% args +args <- args[args != "--sensitive-only"] if (length(args) < 2L) { - stop("usage: build-one.R [ ...]", call. = FALSE) + stop( + "usage: build-one.R [--sensitive-only] [ ...]", + call. = FALSE + ) } package <- args[1L] versions <- args[-1L] @@ -32,17 +38,24 @@ classify <- function(pkg, ver) { on.exit(unlink(dest, recursive = TRUE, force = TRUE), add = TRUE) tryCatch( { + message(sprintf( + "[classify] cloning %s@%s from github.com/cran ...", + pkg, + ver + )) system2( "git", c( "clone", - "-q", + "--depth", + "1", "--branch", ver, sprintf("https://github.com/cran/%s", pkg), dest ) ) + message(sprintf("[classify] running ABI classifier on %s ...", pkg)) isTRUE(as.logical(bincraft::needs_per_minor_recompile(dest))) }, error = function(e) { @@ -67,6 +80,15 @@ touched_minor <- FALSE for (ver in versions) { sensitive <- classify(package, ver) + if (sensitive_only && !sensitive) { + message(sprintf( + "Skipping %s %s under R %s (not r-minor-sensitive)", + package, + ver, + minor + )) + next + } cat(sprintf( "Building %s %s (r_minor_sensitive=%s, R %s)\n", package, @@ -101,8 +123,15 @@ for (ver in versions) { # Refresh the PACKAGES index for each slot we wrote to, so the (re)built binary # is immediately resolvable by clients. codename <- bincraft::set_codename(NULL) +# cranlike keeps a working PACKAGES.db in the CWD; clear any copy left by a +# previous pass so each per-slot index is built fresh. Otherwise the 2nd index +# update in the same container fails with "table packages already exists". +clean_index_workdir <- function() { + unlink(c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db")) +} if (touched_generic) { cat("Refreshing generic index\n") + clean_index_workdir() bincraft::upload_package_index( codename = codename, s3_endpoint = s3$s3_endpoint, @@ -114,6 +143,7 @@ if (touched_generic) { } if (touched_minor) { cat(sprintf("Refreshing per-minor index %s\n", minor)) + clean_index_workdir() bincraft::upload_package_index( codename = codename, r_minor = minor, From a9ad907f9c71051fe5a44a6f9efd15fabaccfad7 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 13:02:48 +0200 Subject: [PATCH 020/125] chore(local): add S3 migration and CRAN-source helper scripts - migrate-s3-hetzner-to-backblaze.sh: rclone-based bucket migration helper. - find-R-api-packages.sh: scan CRAN package sources for a C-API usage pattern. - query-pkgs-without-old-versions.R, test-package-loading.R: ad-hoc helpers. --- local/find-R-api-packages.sh | 34 +++ local/migrate-s3-hetzner-to-backblaze.sh | 90 ++++++ local/query-pkgs-without-old-versions.R | 39 +++ local/test-package-loading.R | 331 +++++++++++++++++++++++ 4 files changed, 494 insertions(+) create mode 100644 local/find-R-api-packages.sh create mode 100644 local/migrate-s3-hetzner-to-backblaze.sh create mode 100644 local/query-pkgs-without-old-versions.R create mode 100644 local/test-package-loading.R diff --git a/local/find-R-api-packages.sh b/local/find-R-api-packages.sh new file mode 100644 index 0000000..1f36961 --- /dev/null +++ b/local/find-R-api-packages.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Directory to clone repos into +WORKDIR="cran_repos" +mkdir -p "$WORKDIR" +cd "$WORKDIR" + +# GitHub API paginates results, so we loop through pages +PAGE=1 +PER_PAGE=100 +MATCHES=() + +while :; do + # Fetch a page of repos + REPOS=$(curl -s "https://api.github.com/orgs/cran/repos?per_page=$PER_PAGE&page=$PAGE" | jq -r '.[].clone_url') + [ -z "$REPOS" ] && break + + for REPO_URL in $REPOS; do + REPO_NAME=$(basename "$REPO_URL" .git) + # Skip if already cloned + [ -d "$REPO_NAME" ] && continue + git clone --depth 1 "$REPO_URL" "$REPO_NAME" >/dev/null 2>&1 + if [ -d "$REPO_NAME/src" ]; then + # Search for Rinternals.h in src/ + if grep -r -q 'R_VERSION < R_Version(' "$REPO_NAME/src"; then + echo "$REPO_NAME" + fi + fi + # Clean up to save space + rm -rf "$REPO_NAME" + done + + PAGE=$((PAGE + 1)) +done diff --git a/local/migrate-s3-hetzner-to-backblaze.sh b/local/migrate-s3-hetzner-to-backblaze.sh new file mode 100644 index 0000000..8cbd829 --- /dev/null +++ b/local/migrate-s3-hetzner-to-backblaze.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Migrate S3 buckets from Hetzner Object Storage to Backblaze B2 via rclone. +# +# Prerequisites: +# 1. Install rclone: https://rclone.org/install/ +# 2. Configure two rclone remotes: +# rclone config create hetzner s3 \ +# provider=Other \ +# env_auth=false \ +# access_key_id=YOUR_HETZNER_KEY \ +# secret_access_key=YOUR_HETZNER_SECRET \ +# endpoint=fsn1.your-objectstorage.com # adjust region +# +# rclone config create backblaze s3 \ +# provider=Other \ +# env_auth=false \ +# access_key_id=YOUR_B2_KEY \ +# secret_access_key=YOUR_B2_APP_KEY \ +# endpoint=s3.us-west-004.backblazeb2.com # adjust region +# +# Usage: +# ./migrate-s3-hetzner-to-backblaze.sh [src:dst] ... +# ./migrate-s3-hetzner-to-backblaze.sh hetzner-bucket:backblaze-bucket + +HETZNER_REMOTE="${HETZNER_REMOTE:-hetzner}" +BACKBLAZE_REMOTE="${BACKBLAZE_REMOTE:-backblaze}" +RCLONE_FLAGS="${RCLONE_FLAGS:---transfers=64 --checkers=64 --fast-list}" + +if [[ $# -eq 0 ]]; then + echo "Usage: $0 [src-bucket:dst-bucket...]" + echo "" + echo " Each argument is a source:destination bucket pair separated by a colon." + echo "" + echo "Environment variables:" + echo " HETZNER_REMOTE rclone remote name for Hetzner (default: hetzner)" + echo " BACKBLAZE_REMOTE rclone remote name for Backblaze (default: backblaze)" + echo " RCLONE_FLAGS extra rclone flags (default: --transfers=16 --checkers=16 --fast-list)" + echo " DRY_RUN=1 show what would be copied without copying" + exit 1 +fi + +for cmd in rclone; do + if ! command -v "$cmd" &>/dev/null; then + echo "Error: $cmd is not installed." >&2 + exit 1 + fi +done + +# Verify remotes exist +for remote in "$HETZNER_REMOTE" "$BACKBLAZE_REMOTE"; do + if ! rclone listremotes | grep -q "^${remote}:$"; then + echo "Error: rclone remote '${remote}' not found. Run 'rclone config' to set it up." >&2 + exit 1 + fi +done + +DRY_RUN_FLAG="" +if [[ "${DRY_RUN:-0}" == "1" ]]; then + DRY_RUN_FLAG="--dry-run" + echo "=== DRY RUN MODE ===" +fi + +for pair in "$@"; do + if [[ "$pair" != *:* ]]; then + echo "Error: '$pair' is not a valid src:dst pair. Use format 'hetzner-bucket:backblaze-bucket'." >&2 + exit 1 + fi + + src_bucket="${pair%%:*}" + dst_bucket="${pair#*:}" + src="${HETZNER_REMOTE}:${src_bucket}" + dst="${BACKBLAZE_REMOTE}:${dst_bucket}" + + echo "" + echo "--- Migrating: ${src} -> ${dst} ---" + + # shellcheck disable=SC2086 + rclone sync \ + ${RCLONE_FLAGS} \ + ${DRY_RUN_FLAG} \ + --progress \ + "$src" "$dst" + + echo "--- Done: ${src_bucket} -> ${dst_bucket} ---" +done + +echo "" +echo "Migration complete." diff --git a/local/query-pkgs-without-old-versions.R b/local/query-pkgs-without-old-versions.R new file mode 100644 index 0000000..29ba891 --- /dev/null +++ b/local/query-pkgs-without-old-versions.R @@ -0,0 +1,39 @@ +library(s3fs) + +# List all files under contrib// +all_files <- s3fs::s3_dir_ls( + "s3://devxy-r-package-binaries-hel1/arm64/alpine322/latest/src/contrib/", + recurse = TRUE, + type = "file" +) + +pattern <- ".*/src/contrib/([^/_]+)_.*" +matches <- regmatches(all_files, regexec(pattern, all_files)) +pkg_names <- unique( + vapply( + matches, + function(x) if (length(x) > 1) x[2] else NA_character_, + character(1) + ) +) +pkg_names <- pkg_names[!is.na(pkg_names)] + +# For each package, check if Archive// contains any files +no_archive_files <- character(0) +for (pkg in pkg_names) { + archive_dir1 <- sprintf( + "s3://devxy-r-package-binaries-hel1/arm64/alpine322/latest/src/contrib/Archive/%s", + pkg + ) + archive_files <- unique(c( + tryCatch( + s3fs::s3_dir_ls(archive_dir1, recurse = TRUE), + error = function(e) character(0) + ) + )) + if (length(archive_files) == 0) { + no_archive_files <- c(no_archive_files, pkg) + } +} + +print(no_archive_files) diff --git a/local/test-package-loading.R b/local/test-package-loading.R new file mode 100644 index 0000000..26a2e82 --- /dev/null +++ b/local/test-package-loading.R @@ -0,0 +1,331 @@ +install.packages( + "pak", + repos = sprintf( + "https://r-lib.github.io/p/pak/stable/%s/%s/%s", + .Platform$pkgType, + R.Version()$os, + R.Version()$arch + ) +) + +Sys.setenv(PKG_SYSREQS = TRUE) +all_pkgs <- rownames(available.packages()) + +to_skip = c("ABRSQOL", "ACA", "ACE.CoCo") +all_pkgs = setdiff(all_pkgs, to_skip) + +for (i in all_pkgs) { + message(sprintf("\nInstalling %s", i)) + pak::pkg_install(i) + library(i, character.only = TRUE) +} + + +# Example data +all_pkgs <- rownames(available.packages()) +to_skip <- c( + "ABRSQOL", + "ACA", + "ACE.CoCo", + "ACEsimFit", + "ACNE", + "absorber", + "adapt4pv", + "adaptMCMC", + "addhaz", + "ADDT", + "ahaz", + "arm", + "arules", + "arulesCBA", + "aster2", + "BayesFactor", + "bc3net", + "bgsmtr", + "biglasso", + "BinNonNor", + "BinNor", + "bioassayR", + "birankr", + "BiRewire", + "bolasso", + "Boptbd", + "Brobdingnag", + "BSW", + "BTLLasso", + "bvartools", + "cAIC4", + "Category", + "celda", + "centiserve", + "cjoint", + "clinical", + "clipper", + "CodataGS", + "conos", + "CopulaInference", + "covEB", + "cplm", + "CRTgeeDR", + "cthreshER", + "ctmcmove", + "curephEM", + "CVST", + "dcGSA", + "dclone", + "dcsvm", + "DelayedArray", + "dglars", + "dhglm", + "disordR", + "distrom", + "dmm", + "DNABarcodes", + "DoubleCone", + "DRR", + "DTRlearn2", + "DWDLargeR", + "eds", + "EMCluster", + "EMMREML", + "evalITR", + "EventPointer", + "evola", + "excursions", + "expm", + "fanc", + "FAS", + "fastadi", + "fastPLS", + "fastRG", + "fdaPDE", + "flare", + "FoReco", + "frailtyHL", + "freebird", + "FSTpackage", + "gamlr", + "gamlss.lasso", + "gamm4", + "gbmt", + "gdim", + "gdistance", + "GeDS", + "geeM", + "genlasso", + "GenOrd", + "GenoScan", + "geomorph", + "geostatsp", + "GhostKnockoff", + "GIGSEA", + "GlarmaVarSel", + "glmm", + "glmmrBase", + "glmmrOptim", + "glmnet", + "glober", + "GPvam", + "graphpcor", + "gremlin", + "growthrate", + "grpCox", + "GSD", + "HelpersMG", + "hglm", + "hglm.data", + "hibayes", + "hierSDR", + "HMTL", + "hsem", + "ibmdbR", + "inca", + "INLAspacetime", + "INLAtools", + "invertiforms", + "irlba", + "islasso", + "ISLET", + "isotonic.pen", + "jordan", + "kinship2", + "KnockoffScreen", + "lcpm", + "leidenAlg", + "lfe", + "lingmatch", + "LKT", + "lme4", + "lme4breeding", + "lme4GS", + "logcondiscr", + "LPmerge", + "LRMF3", + "MAP", + "marcox", + "markovchain", + "MatrixExtra", + "matter", + "MBC", + "mcen", + "mclogit", + "MCMCglmm", + "mdhglm", + "MDPtoolbox", + "mediation", + "mefa4", + "metafor", + "mgwrsar", + "mi", + "midasml", + "mind", + "monocle", + "msda", + "MuData", + "MultiGlarmaVarSel", + "MultiOrd", + "MultiVarSel", + "mvglmmRank", + "N2R", + "nadiv", + "NBtsVarSel", + "NegBinBetaBinreg", + "NetworkRiskMeasures", + "neuroim2", + "NOISeq", + "numbat", + "optbdmaeAT", + "optimbase", + "OptimModel", + "optrcdmaeAT", + "OrdNor", + "pagoda2", + "PCovR", + "pedgene", + "pedigree", + "pedigreemm", + "pense", + "PERMANOVA", + "phateR", + "PhylogeneticEM", + "pleio", + "POINT", + "PoisBinNonNor", + "PoisBinOrd", + "PoisBinOrdNonNor", + "PoisBinOrdNor", + "PoisNonNor", + "PoisNor", + "PRISMA", + "ProbitSpatial", + "prodest", + "psqn", + "qlcMatrix", + "qpcR", + "QRM", + "quadrupen", + "QZ", + "ramps", + "randnet", + "randPedPCA", + "rBMF", + "RCBR", + "RealVAMS", + "REBayes", + "recommenderlab", + "Rediscover", + "reglogit", + "RESET", + "RGE", + "RGENERATEPREC", + "riemtan", + "RNewsflow", + "robustlmm", + "rsparse", + "rSPDE", + "rwc", + "S4Arrays", + "saeMSPE", + "sbw", + "scITD", + "scoup", + "sdwd", + "SEAGLE", + "sensory", + "serrsBayes", + "sglasso", + "sharpPen", + "SiPSiC", + "SKAT", + "snpReady", + "snpStats", + "softImpute", + "sommer", + "soptdmaeA", + "SOR", + "SparseArray", + "SparseChol", + "sparseLRMatrix", + "sparsenet", + "sparsenetgls", + "sparsestep", + "spatialprobit", + "spatialreg", + "spatstat.sparse", + "speedglm", + "sRDA", + "sSDR", + "ssfa", + "stcos", + "StratifiedSampling", + "sureLDA", + "survey", + "surveyvoi", + "svydiags", + "systemfit", + "TargetScore", + "text2map", + "textir", + "textmineR", + "textTinyR", + "tmvtnorm", + "TPEA", + "triversity", + "tsapp", + "tvReg", + "uwot", + "vagam", + "VAM", + "WaveSampling", + "WGScan", + "wordspace", + "workflowsets", + "ACSSpack", + "ADDT", + "AER" +) + +# Find the position of the last package in to_skip within all_pkgs +last_skip <- tail(to_skip, 1) + +# Find its position in all_pkgs (returns NA if not found) +start_pos <- match(last_skip, all_pkgs) + +# If not found, start from the beginning; else, start after last_skip +if (is.na(start_pos)) { + to_process <- all_pkgs +} else { + to_process <- all_pkgs[(start_pos + 1):length(all_pkgs)] +} + +if (length(to_process) == 0) { + message("All packages have been processed.") +} else { + for (i in to_process) { + message(sprintf("\nInstalling %s", i)) + pak::pkg_install(i) + library(i, character.only = TRUE) + } + # Update to_skip to include all up to the last processed + to_skip <- all_pkgs[1:(last_skip_pos + length(to_process))] +} From 95bcf6b37777f9fc00fec730e5a484f221f3e140 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 14:11:22 +0200 Subject: [PATCH 021/125] chore: build alpine 3.24 --- .crow/build-all-versions-amd64.yaml | 4 +--- .crow/build-all-versions-arm64.yaml | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.crow/build-all-versions-amd64.yaml b/.crow/build-all-versions-amd64.yaml index 5d225b6..dd33dc2 100644 --- a/.crow/build-all-versions-amd64.yaml +++ b/.crow/build-all-versions-amd64.yaml @@ -3,9 +3,7 @@ # OS (alpine) # OS_VERSION (3.22) # R_VERSION (4.5.3) -# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.23 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 -# crow pipeline create --var ARCH=arm64 --var OS=redhat --var OS_VERSION=10 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 -# crow pipeline create --var ARCH=amd64 --var OS=redhat --var OS_VERSION=10 --var R_VERSION=4.5.3 --var task=build-all-amd64 --branch=main --log-level=info 5 +# crow pipeline create --var ARCH=amd64 --var OS=alpine --var OS_VERSION=3.24 --var R_VERSION=4.5.3 --var task=build-all-amd64 --branch=main --log-level=info 5 when: - event: manual evaluate: 'task == "build-all-amd64"' diff --git a/.crow/build-all-versions-arm64.yaml b/.crow/build-all-versions-arm64.yaml index 6700157..26e5797 100644 --- a/.crow/build-all-versions-arm64.yaml +++ b/.crow/build-all-versions-arm64.yaml @@ -4,9 +4,8 @@ # OS_VERSION (3.22) # R_VERSION (4.5.3) # Skip list lives in local/excluded-packages.json (read by local/build-all.R). -# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.22 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --branch=main --log-level=info 7 -# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.23 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --branch=main --log-level=info 5 -# crow pipeline create --var ARCH=arm64 --var OS=redhat --var OS_VERSION=10 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 +# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.24 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 +# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.24 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 when: - event: manual evaluate: 'task == "build-all-arm64"' From 7e01f481c085dbc3729e49374f7bba2c19b5094c Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 14:50:40 +0200 Subject: [PATCH 022/125] perf(local): skip per-minor passes for non-r-minor-sensitive packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A package's ABI sensitivity is the same across R minors, so for a non-sensitive package the per-minor loop only ran `ensure_bincraft` (a ~14s bincraft/cranlike install per minor) before build-one.R classified and skipped it. The primary pass now writes a .r_minor_sensitive sentinel when it builds a sensitive package, and the wrapper gates the whole per-minor loop (installs included) on that file — non-sensitive rebuilds no longer touch other minors' libraries. --- docker/build-one.Dockerfile | 27 ++++++++++++++++----------- local/build-one.R | 7 +++++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 1884a44..2cd4d51 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -41,20 +41,25 @@ RUN --mount=type=secret,id=b2_access,required=true \ PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ seen=" $PRIMARY_MINOR "; \ prc=0; failed=""; \ + rm -f .r_minor_sensitive; \ echo "=== primary pass under R $R_VERSION ==="; \ ensure_bincraft /opt/R/${R_VERSION}/bin/R; \ run_build /opt/R/${R_VERSION}/bin/Rscript /work/build-one.R "${PACKAGE}" ${VERSIONS} || prc=$?; \ - for RBIN in /opt/R/*/bin/Rscript; do \ - [ -x "$RBIN" ] || continue; \ - RV=$(basename "$(dirname "$(dirname "$RBIN")")"); \ - case "$RV" in [0-9]*) ;; *) continue ;; esac; \ - RMINOR=$(echo "$RV" | cut -d. -f1-2); \ - case "$seen" in *" $RMINOR "*) continue ;; esac; \ - seen="$seen$RMINOR "; \ - echo "=== sensitive-only pass under R $RV ==="; \ - ensure_bincraft "$(dirname "$RBIN")/R"; \ - run_build "$RBIN" /work/build-one.R --sensitive-only "${PACKAGE}" ${VERSIONS} || failed="$failed $RV"; \ - done; \ + if [ -f .r_minor_sensitive ]; then \ + for RBIN in /opt/R/*/bin/Rscript; do \ + [ -x "$RBIN" ] || continue; \ + RV=$(basename "$(dirname "$(dirname "$RBIN")")"); \ + case "$RV" in [0-9]*) ;; *) continue ;; esac; \ + RMINOR=$(echo "$RV" | cut -d. -f1-2); \ + case "$seen" in *" $RMINOR "*) continue ;; esac; \ + seen="$seen$RMINOR "; \ + echo "=== sensitive-only pass under R $RV ==="; \ + ensure_bincraft "$(dirname "$RBIN")/R"; \ + run_build "$RBIN" /work/build-one.R --sensitive-only "${PACKAGE}" ${VERSIONS} || failed="$failed $RV"; \ + done; \ + else \ + echo "Package not r-minor-sensitive; skipping per-minor passes."; \ + fi; \ if [ -n "$failed" ]; then echo "WARNING: extra-minor build(s) failed (non-fatal; e.g. a version too old to compile on a newer R):$failed" >&2; fi; \ if [ "$prc" != 0 ]; then echo "primary pass under R $R_VERSION failed (exit $prc)" >&2; fi; \ exit "$prc" diff --git a/local/build-one.R b/local/build-one.R index 5531124..f2d6ebd 100644 --- a/local/build-one.R +++ b/local/build-one.R @@ -120,6 +120,13 @@ for (ver in versions) { if (sensitive) touched_minor <- TRUE else touched_generic <- TRUE } +# Signal the container wrapper whether per-minor passes are warranted (a single +# package's sensitivity is the same across R minors, so a non-sensitive package +# need not touch any other minor's library). +if (touched_minor) { + file.create(".r_minor_sensitive") +} + # Refresh the PACKAGES index for each slot we wrote to, so the (re)built binary # is immediately resolvable by clients. codename <- bincraft::set_codename(NULL) From 61a9ac560986b1f901aad8626fbaea04e74133f5 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 22:18:19 +0200 Subject: [PATCH 023/125] fix(ci): run per-minor index under the primary R in process-updates The per-minor upload_package_index() loop invoked each minor's own R, which loads bincraft from the shared /mnt/cache/R-pkgs cache (compiled under the primary R). Under a newer minor (e.g. 4.6) that fails to load ("undefined symbol: SETLENGTH") and the index step dies. Indexing is pure S3 work and needs no specific R, so run it under /opt/R/$R_VERSION like the generic index and build-all-versions already do. --- .crow/process-updates-alpine-322-amd64.yaml | 2 +- .crow/process-updates-alpine-322-arm64.yaml | 2 +- .crow/process-updates-alpine-323-amd64.yaml | 2 +- .crow/process-updates-alpine-323-arm64.yaml | 2 +- .crow/process-updates-redhat-10-amd64.yaml | 2 +- .crow/process-updates-redhat-10-arm64.yaml | 2 +- .crow/process-updates-redhat-8-amd64.yaml | 2 +- .crow/process-updates-redhat-8-arm64.yaml | 2 +- .crow/process-updates-redhat-9-amd64.yaml | 2 +- .crow/process-updates-redhat-9-arm64.yaml | 2 +- .crow/process-updates-ubuntu-2204-amd64.yaml | 2 +- .crow/process-updates-ubuntu-2204-arm64.yaml | 2 +- .crow/process-updates-ubuntu-2404-amd64.yaml | 2 +- .crow/process-updates-ubuntu-2404-arm64.yaml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index 30732ce..128436a 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -70,7 +70,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine322', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'alpine322', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done backend_options: kubernetes: diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index 45dd475..f6c3377 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -69,7 +69,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine322', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'alpine322', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done - name: Purge CDN cache diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index 119a825..8707b3e 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -70,7 +70,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine323', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'alpine323', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done backend_options: kubernetes: diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index c8169d4..6e59fb5 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -69,7 +69,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'alpine323', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'alpine323', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done - name: Purge CDN cache diff --git a/.crow/process-updates-redhat-10-amd64.yaml b/.crow/process-updates-redhat-10-amd64.yaml index 7ff3208..f4f9d51 100644 --- a/.crow/process-updates-redhat-10-amd64.yaml +++ b/.crow/process-updates-redhat-10-amd64.yaml @@ -72,7 +72,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel10', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel10', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done backend_options: kubernetes: diff --git a/.crow/process-updates-redhat-10-arm64.yaml b/.crow/process-updates-redhat-10-arm64.yaml index 1604c9e..e1746c3 100644 --- a/.crow/process-updates-redhat-10-arm64.yaml +++ b/.crow/process-updates-redhat-10-arm64.yaml @@ -70,7 +70,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel10', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel10', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done - name: Purge CDN cache diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml index 0381706..be63b2a 100644 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ b/.crow/process-updates-redhat-8-amd64.yaml @@ -70,7 +70,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel8', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel8', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done backend_options: kubernetes: diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml index cc10f00..ca21107 100644 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ b/.crow/process-updates-redhat-8-arm64.yaml @@ -69,7 +69,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel8', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel8', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done - name: Purge CDN cache diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml index 128b5d1..85e4e34 100644 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ b/.crow/process-updates-redhat-9-amd64.yaml @@ -71,7 +71,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel9', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel9', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done backend_options: kubernetes: diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml index 89db00b..793b467 100644 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ b/.crow/process-updates-redhat-9-arm64.yaml @@ -69,7 +69,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'rhel9', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel9', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done - name: Purge CDN cache diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml index 13ec7fb..213d100 100644 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ b/.crow/process-updates-ubuntu-2204-amd64.yaml @@ -71,7 +71,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'jammy', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'jammy', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done backend_options: kubernetes: diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml index 99bcdc2..144e67b 100644 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ b/.crow/process-updates-ubuntu-2204-arm64.yaml @@ -69,7 +69,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'jammy', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'jammy', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done - name: Purge CDN cache diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml index a9f3b7e..d328cb0 100644 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ b/.crow/process-updates-ubuntu-2404-amd64.yaml @@ -70,7 +70,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'noble', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'noble', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done backend_options: kubernetes: diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml index 81f1244..799d30b 100644 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ b/.crow/process-updates-ubuntu-2404-arm64.yaml @@ -69,7 +69,7 @@ steps: - | for RBIN in /opt/R/[0-9]*/bin/R; do RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - "$(dirname "$RBIN")/R" -q -e "library(bincraft); upload_package_index(codename = 'noble', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'noble', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done - name: Purge CDN cache From 8cf8e33bf5cd28e310f806dedd4b3e24b2bc40aa Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 22:22:56 +0200 Subject: [PATCH 024/125] fix(ci): build per-minor binaries with a per-minor R library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sensitive-only multi-R build passes ran under each minor's R but loaded bincraft from the shared R_LIBS_USER=/mnt/cache/R-pkgs cache, which is compiled under the primary R and fails to load under a newer minor (e.g. 4.6: "undefined symbol: SETLENGTH") — so those minor builds silently no-op'd. Give each non-primary minor its own R_LIBS_USER=/mnt/cache/R-pkgs- and install bincraft 4.2.1 there (built for that minor); the rest of bincraft's deps load from the per-version /opt/R//lib. Applied to the build-all-versions and process-updates sensitive-only passes. --- .crow/build-all-versions-amd64.yaml | 5 ++++- .crow/build-all-versions-arm64.yaml | 5 ++++- .crow/process-updates-alpine-322-amd64.yaml | 5 ++++- .crow/process-updates-alpine-322-arm64.yaml | 5 ++++- .crow/process-updates-alpine-323-amd64.yaml | 5 ++++- .crow/process-updates-alpine-323-arm64.yaml | 5 ++++- .crow/process-updates-redhat-10-amd64.yaml | 5 ++++- .crow/process-updates-redhat-10-arm64.yaml | 5 ++++- .crow/process-updates-redhat-8-amd64.yaml | 5 ++++- .crow/process-updates-redhat-8-arm64.yaml | 5 ++++- .crow/process-updates-redhat-9-amd64.yaml | 5 ++++- .crow/process-updates-redhat-9-arm64.yaml | 5 ++++- .crow/process-updates-ubuntu-2204-amd64.yaml | 5 ++++- .crow/process-updates-ubuntu-2204-arm64.yaml | 5 ++++- .crow/process-updates-ubuntu-2404-amd64.yaml | 5 ++++- .crow/process-updates-ubuntu-2404-arm64.yaml | 5 ++++- 16 files changed, 64 insertions(+), 16 deletions(-) diff --git a/.crow/build-all-versions-amd64.yaml b/.crow/build-all-versions-amd64.yaml index dd33dc2..1ec7a35 100644 --- a/.crow/build-all-versions-amd64.yaml +++ b/.crow/build-all-versions-amd64.yaml @@ -75,7 +75,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive pass under R $RV ===" - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" diff --git a/.crow/build-all-versions-arm64.yaml b/.crow/build-all-versions-arm64.yaml index 26e5797..92ea95e 100644 --- a/.crow/build-all-versions-arm64.yaml +++ b/.crow/build-all-versions-arm64.yaml @@ -89,7 +89,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive pass under R $RV ===" - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index 128436a..33cec6d 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -64,7 +64,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index f6c3377..f96add4 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -63,7 +63,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index 8707b3e..4b7e541 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -64,7 +64,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index 6e59fb5..9f3a095 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -63,7 +63,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-redhat-10-amd64.yaml b/.crow/process-updates-redhat-10-amd64.yaml index f4f9d51..4f49dfc 100644 --- a/.crow/process-updates-redhat-10-amd64.yaml +++ b/.crow/process-updates-redhat-10-amd64.yaml @@ -66,7 +66,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-redhat-10-arm64.yaml b/.crow/process-updates-redhat-10-arm64.yaml index e1746c3..4bec280 100644 --- a/.crow/process-updates-redhat-10-arm64.yaml +++ b/.crow/process-updates-redhat-10-arm64.yaml @@ -64,7 +64,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml index be63b2a..a96b456 100644 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ b/.crow/process-updates-redhat-8-amd64.yaml @@ -64,7 +64,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml index ca21107..f1914ef 100644 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ b/.crow/process-updates-redhat-8-arm64.yaml @@ -63,7 +63,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml index 85e4e34..f10347b 100644 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ b/.crow/process-updates-redhat-9-amd64.yaml @@ -65,7 +65,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml index 793b467..9103266 100644 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ b/.crow/process-updates-redhat-9-arm64.yaml @@ -63,7 +63,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml index 213d100..c4dd40d 100644 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ b/.crow/process-updates-ubuntu-2204-amd64.yaml @@ -65,7 +65,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml index 144e67b..300ecd5 100644 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ b/.crow/process-updates-ubuntu-2204-arm64.yaml @@ -63,7 +63,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml index d328cb0..6202a41 100644 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ b/.crow/process-updates-ubuntu-2404-amd64.yaml @@ -64,7 +64,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml index 799d30b..48aca05 100644 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ b/.crow/process-updates-ubuntu-2404-arm64.yaml @@ -63,7 +63,10 @@ steps: RMINOR=$(echo "$RV" | cut -d. -f1-2) [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue echo "=== R-minor-sensitive update pass under R $RV ===" - xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | From 6c97f34cf50077d8b85c4c7df9f827dba3f6deb9 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Mon, 15 Jun 2026 00:31:42 +0000 Subject: [PATCH 025/125] chore(deps): update pre-commit hook posit-dev/air-pre-commit to v0.9.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7edc32..6784ad2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit - rev: 0.8.2 + rev: 0.9.0 hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker From 2dddc5fe434fdac1faa9eecab7d689c42e5f489b Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 16 Jun 2026 00:31:56 +0000 Subject: [PATCH 026/125] chore(deps): update terraform bunnynet to v0.14.3 --- .terraform.lock.hcl | 70 ++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 1f15eb6..2ae039e 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.14.2" + version = "0.14.3" constraints = "~> 0.14" hashes = [ - "h1:/83HKJVEe5PlXxETNi3QYL0Tp9CkKZ6WS7IUhGG/OYU=", - "h1:3DUcGLigdB+8Bu8hh82yoLV+5m2zjrr5HC/QwG56aAY=", - "h1:7q5hf9ODMChDRsDKGJwv8EIlr5vzh3YlZjr3eEzkqN4=", - "h1:APYf6Y7TVdn8lU9C0aR2f1aUnLe652B4HRS9NOMMous=", - "h1:DmE7sOE8wrkYUsGDhUwhonG1LHJZS73IfkzNyACCJJM=", - "h1:K1OeDvRN2GuLjtwCIYuJXpXQYNaaPPknI165CTFyy9c=", - "h1:WPM2ucudI9tm75UoGc0rH9ZM0eKlhTGz5sgO3k4iN7E=", - "h1:WSOCqiEhlK450HYNKwMlUeI5O9inuCaNX5teWZUV/vw=", - "h1:XqaxnjsTBPtiqvHdTjEa0IVgMO+FZhcBxcqE1ALFNzI=", - "h1:YYoBZijn79PiwrIBwv/kb3PP0HzLm3SUxBIzwiuOots=", - "h1:enK6/t/ETDf4iNMrHygg2w9W//K/L0E4lML/bK+kLVc=", - "h1:gOEm/6/hbrg9BWOrsdHSB/GgZcma5zfytEbEDchDj/k=", - "h1:pQ9zZXoKrZRFcUj0OLRFnJK0P/PgY5S+4TpiqbRR5hA=", - "h1:qXl1v7FweyP8/DuzrQIh7LxDbWsWUo+u/wzqB0A93EU=", - "h1:vTuPTvNy3Y/SpmQqnNE1xSEjEoaJYc0b33+tl7q19Gg=", - "h1:wlS2vpTqH0PG85MPk/N1Q9CmtH+jL4pRjXLAkubSOK4=", - "h1:yuwpqTkuqKNwhwEDIOeJdzISZZxECxMiboDUqNHby6Y=", - "zh:019a5c9d068133d0414ab5cf043814a592b1b80f1dcc4afac5ee35d2c43e2409", - "zh:07ba98c56711b584f28dfe7eaea4bbb803ce0e0d7b793b06bb9c6e8ac8358a25", - "zh:0c0e2c343d047221dd0c3f99df6e24ff23e5c982507b634c8e5ed0bf654be2a0", - "zh:0efbabcec9d2bd802bc2cb043e4e7ca7be35716fcec890893b6847fdd7afd05b", - "zh:26a17d31655d45cefab9e1fa76cfb9189eb88d65ce8b3d9922a81f5d2345117e", - "zh:29b3c2447bf202ca33866f724dbe4da28be1e0c988531cd79428cba3ef42b838", - "zh:2bfb3f49610874cb44b2e4122e56a5048b634ef9461c1d02baff81e3c3f9dd91", - "zh:4a6f1db14163c0005eb04eafe1c395d662030a04856e1d08de68623f67c61d59", - "zh:5030883e81c4a2b4a1091cb40d18457dd4de72ce95907c2508184c8df289337f", - "zh:561c6104fd529ccc1da106ed4a8c54ab953f00f5cd0bfec737561c0756639734", - "zh:75eb1fc005852002dd42d3369672db93b423306ccf7b271c4d4c5c372b4924d0", - "zh:7775c4c1663af6f07efe9c0bbabeb66117bb734c1d21fb7e1358f29a18482baa", + "h1:2VmbbvV/3fVb/dHZ5m3CxSPoptpn/PuhvSgV/RA9Ae8=", + "h1:4MANuttWPyJGZlfbx5P4i2Jpbmvda1tfWgd3olK2nhY=", + "h1:AsSfDvm6flYQgjFnFVaRPv7v1/P5pj36dQ4N103Nw+A=", + "h1:D2AK5psWQmZqrOHsbeV70OP/zcG92i++AmGKBz/HVMA=", + "h1:DpJWi/bhHoZa/ccy2YB/c7HJjTlR6CuZBxGFCxU2uJk=", + "h1:Iz47FCzHBOZC1RrTxKwJwKsH6bdEvdYtO0+RBpbmz8I=", + "h1:RJQm4qEfljAmcJaFf45uXwUhtbRTwfDuiUWZ6Q7X+Cw=", + "h1:YQBi0MRW+dOfkxQYbjKkzDm7UIcc2FeFm8Cck+W3kgs=", + "h1:bWq/zJ2Chr5TNuz9y3eaGmzrmXsX30pl5HkGpxE1kVM=", + "h1:hwzEskDnFHaJFvKNAGv/wIxRT2SHcXLfwmBb8XNspq4=", + "h1:jGggry0lZmG0S0inM7Odae3275na9Gp3O97ZlvBiA2s=", + "h1:mJdpVc9qOpqXcPpptWfAET8qloo6qFMb+NzHesazybk=", + "h1:nbpkaopSGlmxkKLtNe9sKx9IqojoicD1+hNRdaYd5ok=", + "h1:pwuoUqikuVOut3RtPTnu6pNoDnyeXlEX7Rfb2Hr1Ch0=", + "h1:u0naFiWGrBVhI3AsUb/ZfS5N/GeiRk0ZkxjHtA10lF0=", + "h1:uozoSPzZ7MJIHj5O62W8n86NIfH4JtF7GvMNI6rKlrM=", + "h1:vDS308OeXVVgjojMCAAEQ1VshmOLjwra7obliKB5fv8=", + "zh:23200ad919df4c44a720c667c536d3a0ae0c235e1f7b53b9e56f88005bf01891", + "zh:2fe5445f578c6a244995538fde6dd7ef0dd5f3969686cba3d91fcf28ea31278d", + "zh:3a5a8139ed177ccbee74feb4d19c881e18723e4bf3a64b6d35931420402a2a1a", + "zh:5189bd6c03ec3c428185992f519f3bf520d4e9eba8771c240814adbecdf60140", + "zh:532ac8eb94a84ccf046d4cb4eded825269effdbb7597f392f848fbfad5dcae1d", + "zh:5f100279d73ce998c7bf4ffa2bbcd617ee307bf51c4a94146f210aeec268c4d0", + "zh:80cb58abd2e431386ab1b6ad186a6e6e5347448993bacc831c0251b9016445b9", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:c22bf8a5bb54c67531f944a15383ef6821ef3073e3036e05a2a8253c3e030443", - "zh:d1be91e623365a81203dbafedaf908d2cffd70ddedda3e2da6258fd227176086", - "zh:f7c9920523f9e82c1fee0c3d6ec2872eb322ddadf8af4ffff41d8fca397922e7", - "zh:f95f455cfdda5ad7f972606a6eaaa2a2a4efb1c3e4c0f7fc55631928266311c4", - "zh:fbfab962729139dcd6ac9529f395be296febcb2aba9017733f58cd1e9c186802", + "zh:9ac734c6aae19ea0832269f1f1dc3c3e100855c09b6168eecd8693958515b228", + "zh:a0e3b75176c71e5f41a6552cef57ad67d11778cb59fd2eb792e69f662b67ae01", + "zh:b4d0e1dd9a80d1ac9a3f1beaefb58d4b944cb0a30bdc672b9037a69b4dc20ec3", + "zh:b8309337e966528991cc2dd61ff0247864e4c3c7bef7dcf225f066edb6171050", + "zh:c34ce5fcdd481214697505bceb616a7dfcda7702f1afefa7dc5459965b8e01ca", + "zh:cb3f662ad06bb86aa763a5644ba318bc3db0ed30b04eb810c478e81f4b012d55", + "zh:cf0f0a459e2b6b1e014af9f53f32b5f555a249b61a0dcaccd3e4fe89178d8aa5", + "zh:d4dfb653a0295bfbbebcb4bb022b490f2c24ca31287ebf77fe9dcaee2e5ea658", + "zh:e49ba31aa092a30ea8a1091f7b26dbe41c3998f924f5b0d82810f27fc2f63501", + "zh:ea4d7d5688caf2f9cda3a1721779b61a051db6b43ca7267ae2712bd1262cfcfd", ] } From 1cd86b65e1cb4a6775ab0cf1b4fc66a660db8202 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 16 Jun 2026 07:27:20 +0000 Subject: [PATCH 027/125] feat(build): skip already-built package versions on workflow restart (#91) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary When a `build-all-*` workflow is restarted, the build job re-reads the static `pkgs_to_build.rds` that the install-deps step produced once, so it cycles over every package an interrupted run already built. This adds a DB-based skip filter so a restart only processes what is genuinely left. - At job start, `build-all.R` queries the `single_builds` metadata table for `(name, tag)` already built successfully (`error_occurred = FALSE`) on this `platform`/`arch`, and drops those pairs from the chunk before the build loop. It logs how many it skipped. - One indexed query, one round trip, run before the pak forks — no extra S3 listing and no new Python/s3fs memory pressure (`RPostgres`/`DBI` are already used in the container). - Errored versions are intentionally **not** skipped, so transient failures still get retried on restart. ## Dependency Correctness depends on a `error_occurred = FALSE` row meaning the binary is actually published. That guarantee is added in rpkgs/bincraft#56 (success row written only after a confirmed S3 upload). This PR should land together with / after a bincraft release including that fix. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/91 --- .crow/archive-missed-packages.yaml | 2 +- .crow/build-all-versions-amd64.yaml | 2 +- .crow/build-all-versions-arm64.yaml | 2 +- .crow/process-updates-alpine-322-amd64.yaml | 4 +-- .crow/process-updates-alpine-322-arm64.yaml | 4 +-- .crow/process-updates-alpine-323-amd64.yaml | 4 +-- .crow/process-updates-alpine-323-arm64.yaml | 4 +-- .crow/process-updates-redhat-10-amd64.yaml | 4 +-- .crow/process-updates-redhat-10-arm64.yaml | 4 +-- .crow/process-updates-redhat-8-amd64.yaml | 4 +-- .crow/process-updates-redhat-8-arm64.yaml | 4 +-- .crow/process-updates-redhat-9-amd64.yaml | 4 +-- .crow/process-updates-redhat-9-arm64.yaml | 4 +-- .crow/process-updates-ubuntu-2204-amd64.yaml | 4 +-- .crow/process-updates-ubuntu-2204-arm64.yaml | 4 +-- .crow/process-updates-ubuntu-2404-amd64.yaml | 4 +-- .crow/process-updates-ubuntu-2404-arm64.yaml | 4 +-- ...ekly-rebuild-missing-alpine-322-amd64.yaml | 2 +- ...ekly-rebuild-missing-alpine-322-arm64.yaml | 2 +- ...ekly-rebuild-missing-alpine-323-amd64.yaml | 2 +- ...ekly-rebuild-missing-alpine-323-arm64.yaml | 2 +- ...eekly-rebuild-missing-redhat-10-amd64.yaml | 2 +- ...eekly-rebuild-missing-redhat-10-arm64.yaml | 2 +- ...weekly-rebuild-missing-redhat-8-amd64.yaml | 2 +- ...weekly-rebuild-missing-redhat-8-arm64.yaml | 2 +- ...weekly-rebuild-missing-redhat-9-amd64.yaml | 2 +- ...weekly-rebuild-missing-redhat-9-arm64.yaml | 2 +- ...kly-rebuild-missing-ubuntu-2204-amd64.yaml | 2 +- ...kly-rebuild-missing-ubuntu-2204-arm64.yaml | 2 +- ...kly-rebuild-missing-ubuntu-2404-amd64.yaml | 2 +- ...kly-rebuild-missing-ubuntu-2404-arm64.yaml | 2 +- local/build-all.R | 25 +++++++++++++++++++ 32 files changed, 70 insertions(+), 45 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 0350f52..f1fe920 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,7 +62,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), workers = 2L)' backend_options: diff --git a/.crow/build-all-versions-amd64.yaml b/.crow/build-all-versions-amd64.yaml index 1ec7a35..be356fb 100644 --- a/.crow/build-all-versions-amd64.yaml +++ b/.crow/build-all-versions-amd64.yaml @@ -77,7 +77,7 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages diff --git a/.crow/build-all-versions-arm64.yaml b/.crow/build-all-versions-arm64.yaml index 92ea95e..be45659 100644 --- a/.crow/build-all-versions-arm64.yaml +++ b/.crow/build-all-versions-arm64.yaml @@ -91,7 +91,7 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index 33cec6d..eb45c72 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -66,7 +66,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index f96add4..13eae6f 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -65,7 +65,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index 4b7e541..b1149bd 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -66,7 +66,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index 9f3a095..7357d8c 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -65,7 +65,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-redhat-10-amd64.yaml b/.crow/process-updates-redhat-10-amd64.yaml index 4f49dfc..c52b1a7 100644 --- a/.crow/process-updates-redhat-10-amd64.yaml +++ b/.crow/process-updates-redhat-10-amd64.yaml @@ -53,7 +53,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -68,7 +68,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-redhat-10-arm64.yaml b/.crow/process-updates-redhat-10-arm64.yaml index 4bec280..9f4cc71 100644 --- a/.crow/process-updates-redhat-10-arm64.yaml +++ b/.crow/process-updates-redhat-10-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -66,7 +66,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml index a96b456..873d9bd 100644 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ b/.crow/process-updates-redhat-8-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -66,7 +66,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml index f1914ef..410a2b5 100644 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ b/.crow/process-updates-redhat-8-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -65,7 +65,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml index f10347b..1ca1332 100644 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ b/.crow/process-updates-redhat-9-amd64.yaml @@ -53,7 +53,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -67,7 +67,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml index 9103266..2a4e284 100644 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ b/.crow/process-updates-redhat-9-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -65,7 +65,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml index c4dd40d..ffc4005 100644 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ b/.crow/process-updates-ubuntu-2204-amd64.yaml @@ -53,7 +53,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -67,7 +67,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml index 300ecd5..c9a549b 100644 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ b/.crow/process-updates-ubuntu-2204-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -65,7 +65,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml index 6202a41..471019c 100644 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ b/.crow/process-updates-ubuntu-2404-amd64.yaml @@ -52,7 +52,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -66,7 +66,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml index 48aca05..4c6f7cf 100644 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ b/.crow/process-updates-ubuntu-2404-arm64.yaml @@ -51,7 +51,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -65,7 +65,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml index a49a52c..e1559ea 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml index 9b1b91d..dcfc103 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml index 821a8c0..4cbe013 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml index ddc9021..9886707 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml index cfe45c1..a10d3c1 100644 --- a/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml index 763d919..3a93bec 100644 --- a/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml index 2278fc0..1150679 100644 --- a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml index 18edc0b..abda7a9 100644 --- a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml index f463e37..88138ab 100644 --- a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml index 1f997f9..7c8ea49 100644 --- a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml index 34493b5..6a4c0da 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml index 4d006b0..88b95c2 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml index e308b0f..482efa7 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml index bcacaef..371c287 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml @@ -39,7 +39,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/local/build-all.R b/local/build-all.R index 4e7b44e..8ce1cd5 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -39,6 +39,31 @@ sprintf("# of package versions for this job: %s", nrow(chunk)) exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] chunk <- chunk[!chunk$Package %in% exclude, ] +# Skip package versions already built in a previous run. +# pkgs_to_build.rds is a static snapshot from the install-deps step, so on a +# restart it still lists everything an interrupted run already produced. The +# metadata DB reflects that progress, so we re-derive the remaining set here. +platform <- paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-") +arch <- Sys.getenv("ARCH") +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +built <- DBI::dbGetQuery( + con, + "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2 AND error_occurred = FALSE", + params = list(platform, arch) +) +DBI::dbDisconnect(con) +before <- nrow(chunk) +chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] +sprintf("Skipped %d already-built package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) + # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, # saving significant memory for pak subprocess forks From 4406a8735b6a690ec125291dd7f81ea9ee85916f Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 16 Jun 2026 09:33:06 +0000 Subject: [PATCH 028/125] refactor(ci): consolidate per-platform crow workflows into 3 matrix files (#92) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Collapse **44** per-platform crow pipeline files into **3** matrix-driven files (one per family), using `matrix.include` + the crow #1165 declarative manual `variables:` block: | Family | Before | After | |---|---|---| | process-updates | 14 | `process-updates.yaml` | | weekly-audit-missing | 16 | `weekly-audit-missing.yaml` | | weekly-rebuild-missing | 14 | `weekly-rebuild-missing.yaml` | - **One file per family** (not split by arch). Arch placement is via the agent label as a matrix var (`artemis`=amd64, `gaia`=arm64) for process-updates and weekly-rebuild; weekly-audit keeps its arch-`nodeSelector` placement (no agent label). - **Matrix axis is named `OS`** (e.g. `redhat-9`) per request, with per-row `R_VERSION`, image tag, codename, and `process_new` (FALSE for alpine). - **Runs preserved 1:1:** - *Cron*: each existing `--` cron still fires only its matching matrix row via `CI_PIPELINE_CRON`. Server-side cron entries unchanged. - *Manual*: a #1165 dropdown variable (`process_cran_updates` / `weekly_audit_missing` / `weekly_rebuild_missing`) selects a single `-` or `all`. weekly-rebuild defaults to `all`, matching its former bare `event: manual` trigger. ## Behavior deltas (intentional, flagged) To fit one file per family the backend was unified: - The amd64-only `node.kubernetes.io/instance-type: AX42` nodeSelector pin is **dropped**; placement now relies on the `artemis`/`gaia` agents. - **process-updates arm64** jobs now share the same `backend_options` as amd64 (resource requests/limits + tolerations) — they previously had none, so they gain an 18Gi memory limit. Tell me if arm64 should stay uncapped. ## Validation required before merge Relies on crow interpolating matrix variables inside `when.evaluate`, `labels.agent`, and `commands`. Interpolation in image/env/commands is standard crow; the novel bit is `when.evaluate`. The manual dropdown routing exercises the same interpolation as the cron routing, so on this branch: 1. Trigger manually with the dropdown = `redhat-9-amd64` -> confirm only that one job runs (image `build-env-redhat:9`, R 4.4.3, agent artemis). 2. Trigger with `all` -> confirm all os/arch jobs schedule. Crons only fire on the default branch, so no collision while unmerged. Fallback if `${...}` doesn't interpolate in `when`: a runtime `case "$CI_PIPELINE_CRON" in ...` guard, same files otherwise. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/92 --- .crow/process-updates-alpine-322-amd64.yaml | 108 --------- .crow/process-updates-alpine-322-arm64.yaml | 90 ------- .crow/process-updates-alpine-323-amd64.yaml | 108 --------- .crow/process-updates-alpine-323-arm64.yaml | 90 ------- .crow/process-updates-redhat-10-amd64.yaml | 110 --------- .crow/process-updates-redhat-10-arm64.yaml | 91 ------- .crow/process-updates-redhat-8-amd64.yaml | 108 --------- .crow/process-updates-redhat-8-arm64.yaml | 90 ------- .crow/process-updates-redhat-9-amd64.yaml | 109 --------- .crow/process-updates-redhat-9-arm64.yaml | 90 ------- .crow/process-updates-ubuntu-2204-amd64.yaml | 109 --------- .crow/process-updates-ubuntu-2204-arm64.yaml | 90 ------- .crow/process-updates-ubuntu-2404-amd64.yaml | 108 --------- .crow/process-updates-ubuntu-2404-arm64.yaml | 90 ------- .crow/process-updates.yaml | 226 ++++++++++++++++++ ...weekly-audit-missing-alpine-321-amd64.yaml | 52 ---- ...weekly-audit-missing-alpine-321-arm64.yaml | 52 ---- ...weekly-audit-missing-alpine-322-amd64.yaml | 51 ---- ...weekly-audit-missing-alpine-322-arm64.yaml | 50 ---- ...weekly-audit-missing-alpine-323-amd64.yaml | 51 ---- ...weekly-audit-missing-alpine-323-arm64.yaml | 51 ---- .../weekly-audit-missing-redhat-10-amd64.yaml | 51 ---- .../weekly-audit-missing-redhat-10-arm64.yaml | 51 ---- .../weekly-audit-missing-redhat-8-amd64.yaml | 52 ---- .../weekly-audit-missing-redhat-8-arm64.yaml | 52 ---- .../weekly-audit-missing-redhat-9-amd64.yaml | 52 ---- .../weekly-audit-missing-redhat-9-arm64.yaml | 52 ---- ...eekly-audit-missing-ubuntu-2204-amd64.yaml | 52 ---- ...eekly-audit-missing-ubuntu-2204-arm64.yaml | 52 ---- ...eekly-audit-missing-ubuntu-2404-amd64.yaml | 52 ---- ...eekly-audit-missing-ubuntu-2404-arm64.yaml | 52 ---- .crow/weekly-audit-missing.yaml | 161 +++++++++++++ ...ekly-rebuild-missing-alpine-322-amd64.yaml | 64 ----- ...ekly-rebuild-missing-alpine-323-amd64.yaml | 64 ----- ...ekly-rebuild-missing-alpine-323-arm64.yaml | 63 ----- ...eekly-rebuild-missing-redhat-10-amd64.yaml | 64 ----- ...eekly-rebuild-missing-redhat-10-arm64.yaml | 63 ----- ...weekly-rebuild-missing-redhat-8-amd64.yaml | 64 ----- ...weekly-rebuild-missing-redhat-8-arm64.yaml | 63 ----- ...weekly-rebuild-missing-redhat-9-amd64.yaml | 64 ----- ...weekly-rebuild-missing-redhat-9-arm64.yaml | 63 ----- ...kly-rebuild-missing-ubuntu-2204-amd64.yaml | 64 ----- ...kly-rebuild-missing-ubuntu-2204-arm64.yaml | 63 ----- ...kly-rebuild-missing-ubuntu-2404-amd64.yaml | 64 ----- ...kly-rebuild-missing-ubuntu-2404-arm64.yaml | 63 ----- ...arm64.yaml => weekly-rebuild-missing.yaml} | 125 ++++++++-- 46 files changed, 498 insertions(+), 3056 deletions(-) delete mode 100644 .crow/process-updates-alpine-322-amd64.yaml delete mode 100644 .crow/process-updates-alpine-322-arm64.yaml delete mode 100644 .crow/process-updates-alpine-323-amd64.yaml delete mode 100644 .crow/process-updates-alpine-323-arm64.yaml delete mode 100644 .crow/process-updates-redhat-10-amd64.yaml delete mode 100644 .crow/process-updates-redhat-10-arm64.yaml delete mode 100644 .crow/process-updates-redhat-8-amd64.yaml delete mode 100644 .crow/process-updates-redhat-8-arm64.yaml delete mode 100644 .crow/process-updates-redhat-9-amd64.yaml delete mode 100644 .crow/process-updates-redhat-9-arm64.yaml delete mode 100644 .crow/process-updates-ubuntu-2204-amd64.yaml delete mode 100644 .crow/process-updates-ubuntu-2204-arm64.yaml delete mode 100644 .crow/process-updates-ubuntu-2404-amd64.yaml delete mode 100644 .crow/process-updates-ubuntu-2404-arm64.yaml create mode 100644 .crow/process-updates.yaml delete mode 100644 .crow/weekly-audit-missing-alpine-321-amd64.yaml delete mode 100644 .crow/weekly-audit-missing-alpine-321-arm64.yaml delete mode 100644 .crow/weekly-audit-missing-alpine-322-amd64.yaml delete mode 100644 .crow/weekly-audit-missing-alpine-322-arm64.yaml delete mode 100644 .crow/weekly-audit-missing-alpine-323-amd64.yaml delete mode 100644 .crow/weekly-audit-missing-alpine-323-arm64.yaml delete mode 100644 .crow/weekly-audit-missing-redhat-10-amd64.yaml delete mode 100644 .crow/weekly-audit-missing-redhat-10-arm64.yaml delete mode 100644 .crow/weekly-audit-missing-redhat-8-amd64.yaml delete mode 100644 .crow/weekly-audit-missing-redhat-8-arm64.yaml delete mode 100644 .crow/weekly-audit-missing-redhat-9-amd64.yaml delete mode 100644 .crow/weekly-audit-missing-redhat-9-arm64.yaml delete mode 100644 .crow/weekly-audit-missing-ubuntu-2204-amd64.yaml delete mode 100644 .crow/weekly-audit-missing-ubuntu-2204-arm64.yaml delete mode 100644 .crow/weekly-audit-missing-ubuntu-2404-amd64.yaml delete mode 100644 .crow/weekly-audit-missing-ubuntu-2404-arm64.yaml create mode 100644 .crow/weekly-audit-missing.yaml delete mode 100644 .crow/weekly-rebuild-missing-alpine-322-amd64.yaml delete mode 100644 .crow/weekly-rebuild-missing-alpine-323-amd64.yaml delete mode 100644 .crow/weekly-rebuild-missing-alpine-323-arm64.yaml delete mode 100644 .crow/weekly-rebuild-missing-redhat-10-amd64.yaml delete mode 100644 .crow/weekly-rebuild-missing-redhat-10-arm64.yaml delete mode 100644 .crow/weekly-rebuild-missing-redhat-8-amd64.yaml delete mode 100644 .crow/weekly-rebuild-missing-redhat-8-arm64.yaml delete mode 100644 .crow/weekly-rebuild-missing-redhat-9-amd64.yaml delete mode 100644 .crow/weekly-rebuild-missing-redhat-9-arm64.yaml delete mode 100644 .crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml delete mode 100644 .crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml delete mode 100644 .crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml delete mode 100644 .crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml rename .crow/{weekly-rebuild-missing-alpine-322-arm64.yaml => weekly-rebuild-missing.yaml} (51%) diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml deleted file mode 100644 index eb45c72..0000000 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ /dev/null @@ -1,108 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-322-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-322-amd64"' - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'alpine322', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: alpine322 - ARCH: amd64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml deleted file mode 100644 index 13eae6f..0000000 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ /dev/null @@ -1,90 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-322-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-322-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-alpine-321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'alpine322', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: alpine322 - ARCH: arm64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml deleted file mode 100644 index b1149bd..0000000 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ /dev/null @@ -1,108 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-323-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-323-amd64"' - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-323 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-alpine-323:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'alpine323', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: alpine323 - ARCH: amd64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml deleted file mode 100644 index 7357d8c..0000000 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ /dev/null @@ -1,90 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-323-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-323-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-323 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-alpine-321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'alpine323', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: alpine323 - ARCH: arm64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-redhat-10-amd64.yaml b/.crow/process-updates-redhat-10-amd64.yaml deleted file mode 100644 index c52b1a7..0000000 --- a/.crow/process-updates-redhat-10-amd64.yaml +++ /dev/null @@ -1,110 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-redhat-10-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "redhat-10-amd64"' - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-redhat:10 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-10 - ARCH: amd64 - NCPUS: 2 - MEMORY: 18Gi - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-redhat-10:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel10', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: rhel10 - ARCH: amd64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-redhat-10-arm64.yaml b/.crow/process-updates-redhat-10-arm64.yaml deleted file mode 100644 index 9f4cc71..0000000 --- a/.crow/process-updates-redhat-10-arm64.yaml +++ /dev/null @@ -1,91 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-redhat-10-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "redhat-10-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-redhat:10 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-10 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-redhat-10:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel10', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: rhel10 - ARCH: arm64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml deleted file mode 100644 index 873d9bd..0000000 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ /dev/null @@ -1,108 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-redhat-8-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "redhat-8-amd64"' - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-redhat:8 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-redhat-8:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel8', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: rhel8 - ARCH: amd64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml deleted file mode 100644 index 410a2b5..0000000 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ /dev/null @@ -1,90 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-redhat-8-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "redhat-8-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-redhat:8 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-redhat-8:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel8', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: rhel8 - ARCH: arm64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml deleted file mode 100644 index 1ca1332..0000000 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ /dev/null @@ -1,109 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-redhat-9-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "redhat-9-amd64"' - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-redhat:9 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: amd64 - NCPUS: 2 - MEMORY: 18Gi - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel9', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: rhel9 - ARCH: amd64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml deleted file mode 100644 index 2a4e284..0000000 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ /dev/null @@ -1,90 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-redhat-9-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "redhat-9-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-redhat:9 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-redhat-9:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'rhel9', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: rhel9 - ARCH: arm64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml deleted file mode 100644 index ffc4005..0000000 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ /dev/null @@ -1,109 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-ubuntu-2204-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "ubuntu-2204-amd64"' - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: amd64 - NCPUS: 2 - MEMORY: 18Gi - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'jammy', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: jammy - ARCH: amd64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml deleted file mode 100644 index c9a549b..0000000 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ /dev/null @@ -1,90 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-ubuntu-2204-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "ubuntu-2204-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'jammy', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: jammy - ARCH: arm64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml deleted file mode 100644 index 471019c..0000000 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ /dev/null @@ -1,108 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-ubuntu-2404-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "ubuntu-2404-amd64"' - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-ubuntu:noble - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - ARCH: amd64 - NCPUS: 1 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'noble', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: noble - ARCH: amd64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml deleted file mode 100644 index 4c6f7cf..0000000 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ /dev/null @@ -1,90 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-ubuntu-2404-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "ubuntu-2404-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-ubuntu:noble - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive update pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - done - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = 'noble', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - SUBDOMAIN3: 'cran.rpkgs.com' - OS_ID: noble - ARCH: arm64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml new file mode 100644 index 0000000..3c8141a --- /dev/null +++ b/.crow/process-updates.yaml @@ -0,0 +1,226 @@ +# Consolidated process-updates pipeline (all platforms, both arches). +# One matrix row per OS/arch replaces the former per-platform files. +# Routing is preserved 1:1: +# - cron: each existing `process-cran-updates--` cron fires only +# its matching matrix row (via CI_PIPELINE_CRON). +# - manual: pick a target from the `process_cran_updates` dropdown +# ("all" = every os/arch). +# Arch placement is handled by the agent label (artemis=amd64, gaia=arm64). +variables: + process_cran_updates: + description: "Manual run target: a specific -, or 'all' for every os/arch." + options: + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + default: all + +when: + - event: cron + evaluate: 'CI_PIPELINE_CRON == "process-cran-updates-${OS}-${ARCH}"' + - event: manual + evaluate: 'process_cran_updates == "all" || process_cran_updates == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + agent: ${AGENT} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.5.3 + IMG: alpine:3.24 + OS_ID: alpine322 + PROCESS_NEW: FALSE + - OS: alpine-322 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.5.3 + IMG: alpine:3.24 + OS_ID: alpine322 + PROCESS_NEW: FALSE + - OS: alpine-323 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.5.3 + IMG: alpine:3.24 + OS_ID: alpine323 + PROCESS_NEW: FALSE + - OS: alpine-323 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.5.3 + IMG: alpine:3.24 + OS_ID: alpine323 + PROCESS_NEW: FALSE + - OS: redhat-8 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: redhat:8 + OS_ID: rhel8 + PROCESS_NEW: TRUE + - OS: redhat-8 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: redhat:8 + OS_ID: rhel8 + PROCESS_NEW: TRUE + - OS: redhat-9 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: redhat:9 + OS_ID: rhel9 + PROCESS_NEW: TRUE + - OS: redhat-9 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: redhat:9 + OS_ID: rhel9 + PROCESS_NEW: TRUE + - OS: redhat-10 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.5.3 + IMG: redhat:10 + OS_ID: rhel10 + PROCESS_NEW: TRUE + - OS: redhat-10 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.5.3 + IMG: redhat:10 + OS_ID: rhel10 + PROCESS_NEW: TRUE + - OS: ubuntu-2204 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + OS_ID: jammy + PROCESS_NEW: TRUE + - OS: ubuntu-2204 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + OS_ID: jammy + PROCESS_NEW: TRUE + - OS: ubuntu-2404 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: ubuntu:noble + OS_ID: noble + PROCESS_NEW: TRUE + - OS: ubuntu-2404 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: ubuntu:noble + OS_ID: noble + PROCESS_NEW: TRUE + +steps: + - name: 'Processing Updates' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ${OS} + OS_ID: ${OS_ID} + ARCH: ${ARCH} + NCPUS: 2 + INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/[0-9]*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true + R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = '${OS_ID}', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 3000m + limits: + memory: 18Gi + cpu: 3000m + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + SUBDOMAIN1: 'cran.devxy.io' + SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' + SUBDOMAIN3: 'cran.rpkgs.com' + OS_ID: ${OS_ID} + ARCH: ${ARCH} + commands: + - apk add --no-cache -q bash curl + - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/weekly-audit-missing-alpine-321-amd64.yaml b/.crow/weekly-audit-missing-alpine-321-amd64.yaml deleted file mode 100644 index 906ab7b..0000000 --- a/.crow/weekly-audit-missing-alpine-321-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-321-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-alpine-321-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: alpine-321 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-alpine-321-arm64.yaml b/.crow/weekly-audit-missing-alpine-321-arm64.yaml deleted file mode 100644 index 41ad809..0000000 --- a/.crow/weekly-audit-missing-alpine-321-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-321-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-alpine-321-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: alpine-321 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-alpine-322-amd64.yaml b/.crow/weekly-audit-missing-alpine-322-amd64.yaml deleted file mode 100644 index 2ed0d81..0000000 --- a/.crow/weekly-audit-missing-alpine-322-amd64.yaml +++ /dev/null @@ -1,51 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-322-amd64 - - event: manual - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: alpine-322 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-alpine-322-arm64.yaml b/.crow/weekly-audit-missing-alpine-322-arm64.yaml deleted file mode 100644 index ef065e3..0000000 --- a/.crow/weekly-audit-missing-alpine-322-arm64.yaml +++ /dev/null @@ -1,50 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-322-arm64 - - event: manual - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: alpine-322 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-alpine-323-amd64.yaml b/.crow/weekly-audit-missing-alpine-323-amd64.yaml deleted file mode 100644 index 9e613f6..0000000 --- a/.crow/weekly-audit-missing-alpine-323-amd64.yaml +++ /dev/null @@ -1,51 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-323-amd64 - - event: manual - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: alpine-323 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-alpine-323-arm64.yaml b/.crow/weekly-audit-missing-alpine-323-arm64.yaml deleted file mode 100644 index e0130c4..0000000 --- a/.crow/weekly-audit-missing-alpine-323-arm64.yaml +++ /dev/null @@ -1,51 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-323-arm64 - - event: manual - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: alpine-323 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-10-amd64.yaml b/.crow/weekly-audit-missing-redhat-10-amd64.yaml deleted file mode 100644 index d1c53cb..0000000 --- a/.crow/weekly-audit-missing-redhat-10-amd64.yaml +++ /dev/null @@ -1,51 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-redhat-10-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-redhat-10-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:10 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: redhat-10 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-10-arm64.yaml b/.crow/weekly-audit-missing-redhat-10-arm64.yaml deleted file mode 100644 index b040da0..0000000 --- a/.crow/weekly-audit-missing-redhat-10-arm64.yaml +++ /dev/null @@ -1,51 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-redhat-10-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-redhat-10-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:10 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: redhat-10 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-8-amd64.yaml b/.crow/weekly-audit-missing-redhat-8-amd64.yaml deleted file mode 100644 index 81fb7d0..0000000 --- a/.crow/weekly-audit-missing-redhat-8-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-redhat-8-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-redhat-8-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:8 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: redhat-8 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-8-arm64.yaml b/.crow/weekly-audit-missing-redhat-8-arm64.yaml deleted file mode 100644 index 0a624f8..0000000 --- a/.crow/weekly-audit-missing-redhat-8-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-redhat-8-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-redhat-8-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:8 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: redhat-8 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-9-amd64.yaml b/.crow/weekly-audit-missing-redhat-9-amd64.yaml deleted file mode 100644 index 0d4b043..0000000 --- a/.crow/weekly-audit-missing-redhat-9-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-redhat-9-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-redhat-9-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: redhat-9 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-9-arm64.yaml b/.crow/weekly-audit-missing-redhat-9-arm64.yaml deleted file mode 100644 index adda69f..0000000 --- a/.crow/weekly-audit-missing-redhat-9-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-redhat-9-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-redhat-9-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: redhat-9 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml b/.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml deleted file mode 100644 index 17b759d..0000000 --- a/.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-ubuntu-2204-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-ubuntu-2204-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: ubuntu-2204 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml b/.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml deleted file mode 100644 index 4a8591b..0000000 --- a/.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-ubuntu-2204-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-ubuntu-2204-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: ubuntu-2204 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml b/.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml deleted file mode 100644 index 2b94959..0000000 --- a/.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-ubuntu-2404-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-ubuntu-2404-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:noble - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: ubuntu-2404 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml b/.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml deleted file mode 100644 index b9a16dc..0000000 --- a/.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-ubuntu-2404-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-ubuntu-2404-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:noble - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - PLATFORM: ubuntu-2404 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml new file mode 100644 index 0000000..7a44f2b --- /dev/null +++ b/.crow/weekly-audit-missing.yaml @@ -0,0 +1,161 @@ +# Consolidated weekly-audit-missing pipeline (all platforms, both arches). +# One matrix row per OS/arch replaces the former per-platform files. +# Routing is preserved 1:1: +# - cron: each existing `weekly-audit-missing--` cron fires only +# its matching matrix row (via CI_PIPELINE_CRON). +# - manual: pick a target from the `weekly_audit_missing` dropdown +# ("all" = every os/arch). +# Arch placement is via the agent label (artemis=amd64, gaia=arm64). +variables: + weekly_audit_missing: + description: "Manual run target: a specific -, or 'all' for every os/arch." + options: + - all + - alpine-321-amd64 + - alpine-321-arm64 + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + default: all + +when: + - event: cron + evaluate: 'CI_PIPELINE_CRON == "weekly-audit-missing-${OS}-${ARCH}"' + - event: manual + evaluate: 'weekly_audit_missing == "all" || weekly_audit_missing == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + agent: ${AGENT} + +matrix: + include: + - OS: alpine-321 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-321 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-322 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-322 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-323 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-323 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: ubuntu:noble + +steps: + - name: 'Audit missing binaries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + PLATFORM: ${OS} + ARCH: ${ARCH} + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' + - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' + backend_options: + docker: + resources: + requests: + memory: 2Gi + cpu: 2000m + limits: + memory: 4Gi + cpu: 2000m diff --git a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml deleted file mode 100644 index e1559ea..0000000 --- a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml +++ /dev/null @@ -1,64 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-322-amd64 - - event: manual - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - ARCH: amd64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml deleted file mode 100644 index 4cbe013..0000000 --- a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml +++ /dev/null @@ -1,64 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-323-amd64 - - event: manual - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-323 - ARCH: amd64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml deleted file mode 100644 index 9886707..0000000 --- a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml +++ /dev/null @@ -1,63 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-323-arm64 - - event: manual - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-323 - ARCH: arm64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml deleted file mode 100644 index a10d3c1..0000000 --- a/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml +++ /dev/null @@ -1,64 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-redhat-10-amd64 - - event: manual - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:10 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-10 - ARCH: amd64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml deleted file mode 100644 index 3a93bec..0000000 --- a/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml +++ /dev/null @@ -1,63 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-redhat-10-arm64 - - event: manual - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:10 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-10 - ARCH: arm64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml deleted file mode 100644 index 1150679..0000000 --- a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml +++ /dev/null @@ -1,64 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-redhat-8-amd64 - - event: manual - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:8 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - ARCH: amd64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml deleted file mode 100644 index abda7a9..0000000 --- a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml +++ /dev/null @@ -1,63 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-redhat-8-arm64 - - event: manual - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:8 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - ARCH: arm64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml deleted file mode 100644 index 88138ab..0000000 --- a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml +++ /dev/null @@ -1,64 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-redhat-9-amd64 - - event: manual - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: amd64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml deleted file mode 100644 index 7c8ea49..0000000 --- a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml +++ /dev/null @@ -1,63 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-redhat-9-arm64 - - event: manual - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: arm64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml deleted file mode 100644 index 6a4c0da..0000000 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml +++ /dev/null @@ -1,64 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-ubuntu-2204-amd64 - - event: manual - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: amd64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml deleted file mode 100644 index 88b95c2..0000000 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml +++ /dev/null @@ -1,63 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-ubuntu-2204-arm64 - - event: manual - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: arm64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml deleted file mode 100644 index 482efa7..0000000 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml +++ /dev/null @@ -1,64 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-ubuntu-2404-amd64 - - event: manual - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:noble - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - ARCH: amd64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml deleted file mode 100644 index 371c287..0000000 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml +++ /dev/null @@ -1,63 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-ubuntu-2404-arm64 - - event: manual - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:noble - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.4.3 - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - ARCH: arm64 - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml b/.crow/weekly-rebuild-missing.yaml similarity index 51% rename from .crow/weekly-rebuild-missing-alpine-322-arm64.yaml rename to .crow/weekly-rebuild-missing.yaml index dcfc103..e06edf0 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -1,16 +1,120 @@ +# Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). +# One matrix row per OS/arch replaces the former per-platform files. +# Routing is preserved 1:1: +# - cron: each existing `weekly-rebuild-missing--` cron fires only +# its matching matrix row (via CI_PIPELINE_CRON). +# - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the +# previous bare manual trigger that ran every os/arch); pick a +# single - to run just one. +# Arch placement is handled by the agent label (artemis=amd64, gaia=arm64). +variables: + weekly_rebuild_missing: + description: "Manual run target: a specific -, or 'all' for every os/arch." + options: + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + default: all + when: - event: cron - cron: weekly-rebuild-missing-alpine-322-arm64 + evaluate: 'CI_PIPELINE_CRON == "weekly-rebuild-missing-${OS}-${ARCH}"' - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' skip_clone: true labels: - agent: gaia + agent: ${AGENT} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-322 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-323 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-323 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + AGENT: artemis + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + AGENT: gaia + R_VERSION: 4.4.3 + IMG: ubuntu:noble steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + image: reg.devxy.io/rpkgs/build-env-${IMG} pull: true environment: RED_HAT_DEV_PW: @@ -30,10 +134,10 @@ steps: GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: 4.5.3 + R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - ARCH: arm64 + PLATFORM: ${OS} + ARCH: ${ARCH} NCPUS: 2 commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . @@ -46,7 +150,7 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: - kubernetes: + docker: resources: requests: memory: 5Gi @@ -54,10 +158,3 @@ steps: limits: memory: 18Gi cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' From b297f0fbc021b0235cf720e39508a4a405105646 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 16 Jun 2026 09:33:13 +0000 Subject: [PATCH 029/125] refactor(ci): consolidate build-all-versions workflows into 2 files (#93) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Consolidate the four `build-all-versions*` files into two, mirroring the matrix approach used for the process-updates / weekly families: | Before | After | |---|---| | `build-all-versions-amd64.yaml` + `build-all-versions-arm64.yaml` | `build-all-versions.yaml` | | `build-all-versions-install-deps-amd64.yaml` + `-arm64.yaml` | `build-all-versions-install-deps.yaml` | `ARCH` becomes a matrix axis carrying the per-arch differences: `agent` (artemis/gaia), `BACKEND` (kubernetes/docker), and `R_PKG_CACHE_DIR`. Build-all keeps its `SPLIT_INDEX` parallelism, so its matrix is ARCH x SPLIT = 8 rows; only the rows matching the selected arch run. **Invocation unchanged.** Routing uses the existing task selector `task == "build-all-${ARCH}"`, so `--var task=build-all-amd64 ...` still runs exactly the amd64 rows. `OS` / `OS_VERSION` / `R_VERSION` / `K8S_INSTANCE_TYPE` are still passed via `--var`. (You can drop `--var ARCH=...` now — arch comes from the matrix/task.) **No cron.** These pipelines are manual-only, so there are no scheduled runs to affect. ## Validation before merge Same matrix-interpolation-in-`when`/`labels` mechanism validated for #92, plus it now relies on per-row `agent`/`BACKEND` selecting the right execution backend (kubernetes for amd64, docker for arm64). Before merge, on this branch trigger a small `task=build-all-arm64` run and confirm it lands on the gaia/docker backend (and amd64 on artemis/kubernetes). Originals are in git history for rollback. Note: `archive-missed-packages.yaml` was already a single consolidated matrix file (cron + manual) and is left untouched. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/93 --- .crow/build-all-versions-arm64.yaml | 167 ------------------ ...build-all-versions-install-deps-arm64.yaml | 56 ------ ...l => build-all-versions-install-deps.yaml} | 48 +++-- ...ons-amd64.yaml => build-all-versions.yaml} | 124 +++++++------ 4 files changed, 100 insertions(+), 295 deletions(-) delete mode 100644 .crow/build-all-versions-arm64.yaml delete mode 100644 .crow/build-all-versions-install-deps-arm64.yaml rename .crow/{build-all-versions-install-deps-amd64.yaml => build-all-versions-install-deps.yaml} (66%) rename .crow/{build-all-versions-amd64.yaml => build-all-versions.yaml} (69%) diff --git a/.crow/build-all-versions-arm64.yaml b/.crow/build-all-versions-arm64.yaml deleted file mode 100644 index be45659..0000000 --- a/.crow/build-all-versions-arm64.yaml +++ /dev/null @@ -1,167 +0,0 @@ -### Required extra vars -# ARCH (amd64) -# OS (alpine) -# OS_VERSION (3.22) -# R_VERSION (4.5.3) -# Skip list lives in local/excluded-packages.json (read by local/build-all.R). -# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.24 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 -# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.24 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 -when: - - event: manual - evaluate: 'task == "build-all-arm64"' - -skip_clone: true - -labels: - agent: gaia - -matrix: - include: - # - SPLIT_INTO: 5 - # SPLIT_INDEX: 1 - # BACKEND: kubernetes - # - SPLIT_INTO: 5 - # SPLIT_INDEX: 2 - # BACKEND: kubernetes - # - SPLIT_INTO: 5 - # SPLIT_INDEX: 3 - # BACKEND: kubernetes - # - SPLIT_INTO: 5 - # SPLIT_INDEX: 4 - # BACKEND: kubernetes - # - SPLIT_INTO: 5 - # SPLIT_INDEX: 5 - # BACKEND: kubernetes - - SPLIT_INTO: 4 - SPLIT_INDEX: 1 - BACKEND: docker - - SPLIT_INTO: 4 - SPLIT_INDEX: 2 - BACKEND: docker - - SPLIT_INTO: 4 - SPLIT_INDEX: 3 - BACKEND: docker - - SPLIT_INTO: 4 - SPLIT_INDEX: 4 - BACKEND: docker - # - SPLIT_INTO: 5 - # SPLIT_INDEX: 5 - # BACKEND: docker - -depends_on: - - build-all-versions-install-deps-arm64 - -steps: - - name: 'Build binaries' - image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - NCPUS: 2 - volumes: - - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - - | - PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$RBIN")")") - RMINOR=$(echo "$RV" | cut -d. -f1-2) - [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue - echo "=== R-minor-sensitive pass under R $RV ===" - LIB="/mnt/cache/R-pkgs-$RMINOR" - mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true - done - # archive missed packages - - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" - backend_options: - docker: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - name: 'Upload package indexes' - image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache - commands: - - | - CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e "cat(bincraft::set_codename(NULL))") - /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - backend_options: - docker: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-all-versions-install-deps-arm64.yaml b/.crow/build-all-versions-install-deps-arm64.yaml deleted file mode 100644 index 98decea..0000000 --- a/.crow/build-all-versions-install-deps-arm64.yaml +++ /dev/null @@ -1,56 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-all-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Install deps and bincraft' - image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' - pull: true - environment: - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - volumes: - - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache - commands: - # one-time full wipe to fix corrupted .so files from previous failed builds - # - rm -rf /mnt/cache/R-pkgs - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" - backend_options: - docker: - resources: - requests: - memory: 5Gi - cpu: 1000m - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-all-versions-install-deps-amd64.yaml b/.crow/build-all-versions-install-deps.yaml similarity index 66% rename from .crow/build-all-versions-install-deps-amd64.yaml rename to .crow/build-all-versions-install-deps.yaml index 285a4e2..11738ce 100644 --- a/.crow/build-all-versions-install-deps-amd64.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -1,12 +1,43 @@ +# Consolidated install-deps for build-all (both arches in one file). +# Triggered together with build-all-versions; `target_arch` routes the matrix. +# Variables are declared so the manual-run form exposes them (crow #1165); +# they are merged with build-all-versions' identical declarations. +variables: + target_arch: + description: "Architecture to build." + options: + - amd64 + - arm64 + default: amd64 + OS: + description: "Base OS image name (e.g. alpine, redhat, ubuntu)." + default: alpine + OS_VERSION: + description: "OS version / image tag (e.g. 3.24, 9, jammy, noble)." + default: "3.24" + R_VERSION: + description: "Primary R version under /opt/R." + options: + - 4.5.3 + - 4.4.3 + default: 4.5.3 + when: - event: manual - evaluate: 'task == "build-all-amd64"' + evaluate: 'target_arch == "${ARCH}"' skip_clone: true labels: platform: linux/${ARCH} - agent: artemis + agent: ${AGENT} + +matrix: + include: + - ARCH: amd64 + AGENT: artemis + - ARCH: arm64 + AGENT: gaia steps: - name: 'Install deps and bincraft' @@ -42,16 +73,3 @@ steps: requests: memory: 5Gi cpu: 1000m - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-all-versions-amd64.yaml b/.crow/build-all-versions.yaml similarity index 69% rename from .crow/build-all-versions-amd64.yaml rename to .crow/build-all-versions.yaml index be356fb..479c8d6 100644 --- a/.crow/build-all-versions-amd64.yaml +++ b/.crow/build-all-versions.yaml @@ -1,39 +1,83 @@ -### Required extra vars -# ARCH (amd64) -# OS (alpine) -# OS_VERSION (3.22) -# R_VERSION (4.5.3) -# crow pipeline create --var ARCH=amd64 --var OS=alpine --var OS_VERSION=3.24 --var R_VERSION=4.5.3 --var task=build-all-amd64 --branch=main --log-level=info 5 +### Manual full rebuild of all CRAN binaries for one OS/arch. +# Set the variables below in the manual-run form (or via --var on the CLI). +# `target_arch` routes the matrix; OS/OS_VERSION/R_VERSION select the platform +# image and cache volume. Placement is via the agent label (artemis/gaia). +# Skip list lives in local/excluded-packages.json (read by local/build-all.R). +variables: + target_arch: + description: "Architecture to build." + options: + - amd64 + - arm64 + default: amd64 + OS: + description: "Base OS image name (e.g. alpine, redhat, ubuntu)." + default: alpine + OS_VERSION: + description: "OS version / image tag (e.g. 3.24, 9, jammy, noble)." + default: "3.24" + R_VERSION: + description: "Primary R version under /opt/R." + options: + - 4.5.3 + - 4.4.3 + default: 4.5.3 + when: - event: manual - evaluate: 'task == "build-all-amd64"' + evaluate: 'target_arch == "${ARCH}"' skip_clone: true labels: - platform: linux/amd64 - agent: artemis + platform: linux/${ARCH} + agent: ${AGENT} matrix: include: - - SPLIT_INTO: 4 + - ARCH: amd64 + AGENT: artemis + R_PKG_CACHE_DIR: '' + SPLIT_INTO: 4 SPLIT_INDEX: 1 - BACKEND: kubernetes - - SPLIT_INTO: 4 + - ARCH: amd64 + AGENT: artemis + R_PKG_CACHE_DIR: '' + SPLIT_INTO: 4 SPLIT_INDEX: 2 - BACKEND: kubernetes - - SPLIT_INTO: 4 + - ARCH: amd64 + AGENT: artemis + R_PKG_CACHE_DIR: '' + SPLIT_INTO: 4 SPLIT_INDEX: 3 - BACKEND: kubernetes - - SPLIT_INTO: 4 + - ARCH: amd64 + AGENT: artemis + R_PKG_CACHE_DIR: '' + SPLIT_INTO: 4 + SPLIT_INDEX: 4 + - ARCH: arm64 + AGENT: gaia + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + SPLIT_INTO: 4 + SPLIT_INDEX: 1 + - ARCH: arm64 + AGENT: gaia + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + SPLIT_INTO: 4 + SPLIT_INDEX: 2 + - ARCH: arm64 + AGENT: gaia + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + SPLIT_INTO: 4 + SPLIT_INDEX: 3 + - ARCH: arm64 + AGENT: gaia + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + SPLIT_INTO: 4 SPLIT_INDEX: 4 - BACKEND: kubernetes - # - SPLIT_INTO: 4 - # SPLIT_INDEX: 5 - # BACKEND: kubernetes depends_on: - - build-all-versions-install-deps-amd64 + - build-all-versions-install-deps steps: - name: 'Build binaries' @@ -55,7 +99,7 @@ steps: # normal env vars GIT_USER: pat-s # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: '' + R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR} R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache NCPUS: 2 @@ -64,8 +108,6 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' - # pkgs = readRDS('/mnt/cache/pkgs_amd64.rds'); - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -91,22 +133,6 @@ steps: limits: memory: 20Gi cpu: 2000m - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - name: 'Upload package indexes' image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true @@ -134,20 +160,4 @@ steps: cpu: 1000m limits: memory: 20Gi - cpu: 2000m - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' + cpu: 2000m \ No newline at end of file From dc92bd6a2c9339a26d37441c341a1e05d4c17fd7 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 17 Jun 2026 11:03:12 +0200 Subject: [PATCH 030/125] fix(ci): restore cron triggers via per-row cron name filter The matrix consolidation routed cron events through `evaluate: 'CI_PIPELINE_CRON == "..."'`, but CI_PIPELINE_CRON is not exposed to the when.evaluate constraint context, so no matrix row ever matched and the scheduled process-updates / weekly runs stopped firing. Switch back to the first-class `cron:` name filter (still parameterized per matrix row) which the server matches directly against the fired cron. --- .crow/process-updates.yaml | 4 ++-- .crow/weekly-audit-missing.yaml | 4 ++-- .crow/weekly-rebuild-missing.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 3c8141a..74a7b0f 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -2,7 +2,7 @@ # One matrix row per OS/arch replaces the former per-platform files. # Routing is preserved 1:1: # - cron: each existing `process-cran-updates--` cron fires only -# its matching matrix row (via CI_PIPELINE_CRON). +# its matching matrix row (via the per-row `cron:` name filter). # - manual: pick a target from the `process_cran_updates` dropdown # ("all" = every os/arch). # Arch placement is handled by the agent label (artemis=amd64, gaia=arm64). @@ -29,7 +29,7 @@ variables: when: - event: cron - evaluate: 'CI_PIPELINE_CRON == "process-cran-updates-${OS}-${ARCH}"' + cron: process-cran-updates-${OS}-${ARCH} - event: manual evaluate: 'process_cran_updates == "all" || process_cran_updates == "${OS}-${ARCH}"' diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 7a44f2b..b10c108 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -2,7 +2,7 @@ # One matrix row per OS/arch replaces the former per-platform files. # Routing is preserved 1:1: # - cron: each existing `weekly-audit-missing--` cron fires only -# its matching matrix row (via CI_PIPELINE_CRON). +# its matching matrix row (via the per-row `cron:` name filter). # - manual: pick a target from the `weekly_audit_missing` dropdown # ("all" = every os/arch). # Arch placement is via the agent label (artemis=amd64, gaia=arm64). @@ -31,7 +31,7 @@ variables: when: - event: cron - evaluate: 'CI_PIPELINE_CRON == "weekly-audit-missing-${OS}-${ARCH}"' + cron: weekly-audit-missing-${OS}-${ARCH} - event: manual evaluate: 'weekly_audit_missing == "all" || weekly_audit_missing == "${OS}-${ARCH}"' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index e06edf0..f6bf9e5 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -2,7 +2,7 @@ # One matrix row per OS/arch replaces the former per-platform files. # Routing is preserved 1:1: # - cron: each existing `weekly-rebuild-missing--` cron fires only -# its matching matrix row (via CI_PIPELINE_CRON). +# its matching matrix row (via the per-row `cron:` name filter). # - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the # previous bare manual trigger that ran every os/arch); pick a # single - to run just one. @@ -30,7 +30,7 @@ variables: when: - event: cron - evaluate: 'CI_PIPELINE_CRON == "weekly-rebuild-missing-${OS}-${ARCH}"' + cron: weekly-rebuild-missing-${OS}-${ARCH} - event: manual evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' From 76f677ba5bc66030e4eeb20aab13a70fd2e2f14f Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 17 Jun 2026 11:35:44 +0200 Subject: [PATCH 031/125] fix(ci): quote matrix PROCESS_NEW so R receives uppercase FALSE/TRUE Unquoted `PROCESS_NEW: FALSE` is parsed as a YAML boolean and interpolated into the R command as lowercase `false`, causing `process_new = false` and `Error: object 'false' not found`. Quoting keeps the literal FALSE/TRUE that R expects. --- .crow/process-updates.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 74a7b0f..46ebfe0 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -46,98 +46,98 @@ matrix: R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine322 - PROCESS_NEW: FALSE + PROCESS_NEW: "FALSE" - OS: alpine-322 ARCH: arm64 AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine322 - PROCESS_NEW: FALSE + PROCESS_NEW: "FALSE" - OS: alpine-323 ARCH: amd64 AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine323 - PROCESS_NEW: FALSE + PROCESS_NEW: "FALSE" - OS: alpine-323 ARCH: arm64 AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine323 - PROCESS_NEW: FALSE + PROCESS_NEW: "FALSE" - OS: redhat-8 ARCH: amd64 AGENT: artemis R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: TRUE + PROCESS_NEW: "TRUE" - OS: redhat-8 ARCH: arm64 AGENT: gaia R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: TRUE + PROCESS_NEW: "TRUE" - OS: redhat-9 ARCH: amd64 AGENT: artemis R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: TRUE + PROCESS_NEW: "TRUE" - OS: redhat-9 ARCH: arm64 AGENT: gaia R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: TRUE + PROCESS_NEW: "TRUE" - OS: redhat-10 ARCH: amd64 AGENT: artemis R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: TRUE + PROCESS_NEW: "TRUE" - OS: redhat-10 ARCH: arm64 AGENT: gaia R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: TRUE + PROCESS_NEW: "TRUE" - OS: ubuntu-2204 ARCH: amd64 AGENT: artemis R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: TRUE + PROCESS_NEW: "TRUE" - OS: ubuntu-2204 ARCH: arm64 AGENT: gaia R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: TRUE + PROCESS_NEW: "TRUE" - OS: ubuntu-2404 ARCH: amd64 AGENT: artemis R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: TRUE + PROCESS_NEW: "TRUE" - OS: ubuntu-2404 ARCH: arm64 AGENT: gaia R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: TRUE + PROCESS_NEW: "TRUE" steps: - name: 'Processing Updates' From 38756122376bd7bba8a78bbb79f6ccc65b049b13 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 17 Jun 2026 12:47:04 +0200 Subject: [PATCH 032/125] fix(ci): restore xwfb-run/xvfb-run detection for rhel-10 process-updates The matrix consolidation hardcoded `xvfb-run`, which is absent on the rhel-10 image (it ships Xwayland's xwfb-run), causing `xvfb-run: command not found`. Re-add the per-platform launcher detection the old rhel-10 files used: prefer xwfb-run + weston, fall back to xvfb-run. --- .crow/process-updates.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 46ebfe0..ee5b7cc 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -181,8 +181,10 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) for RBIN in /opt/R/[0-9]*/bin/R; do @@ -193,7 +195,7 @@ steps: LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true - R_LIBS_USER="$LIB" xvfb-run "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | From 4c67703f5290b55a707f5186a91c5eac1edd7574 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 17 Jun 2026 16:26:04 +0000 Subject: [PATCH 033/125] refactor(ci): route workflows by agent group label instead of named agent (#94) ## Summary - Switch crow workflow placement from a per-agent label (`agent: ${AGENT}`) to a group label (`group: rpkgs-${ARCH}`), so jobs target the `rpkgs-amd64` / `rpkgs-arm64` agent pools instead of a single named agent (`artemis`/`gaia`). - Remove the now-unused `AGENT` matrix variable from all build/process/weekly workflows. - Update header comments to reflect group-based placement. Agents opt into a pool via `CROW_AGENT_LABELS="group=rpkgs-"`. The group name derives from each matrix row's `ARCH`, so adding or replacing agents no longer requires touching the workflow files. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/94 --- .crow/build-all-versions-install-deps.yaml | 4 +--- .crow/build-all-versions.yaml | 12 ++---------- .crow/process-updates.yaml | 18 ++---------------- .crow/weekly-audit-missing.yaml | 20 ++------------------ .crow/weekly-rebuild-missing.yaml | 18 ++---------------- 5 files changed, 9 insertions(+), 63 deletions(-) diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 11738ce..ad51300 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -30,14 +30,12 @@ skip_clone: true labels: platform: linux/${ARCH} - agent: ${AGENT} + group: rpkgs-${ARCH} matrix: include: - ARCH: amd64 - AGENT: artemis - ARCH: arm64 - AGENT: gaia steps: - name: 'Install deps and bincraft' diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 479c8d6..639ce80 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -1,7 +1,7 @@ ### Manual full rebuild of all CRAN binaries for one OS/arch. # Set the variables below in the manual-run form (or via --var on the CLI). # `target_arch` routes the matrix; OS/OS_VERSION/R_VERSION select the platform -# image and cache volume. Placement is via the agent label (artemis/gaia). +# image and cache volume. Placement is via the group label (rpkgs-amd64/rpkgs-arm64). # Skip list lives in local/excluded-packages.json (read by local/build-all.R). variables: target_arch: @@ -31,47 +31,39 @@ skip_clone: true labels: platform: linux/${ARCH} - agent: ${AGENT} + group: rpkgs-${ARCH} matrix: include: - ARCH: amd64 - AGENT: artemis R_PKG_CACHE_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: amd64 - AGENT: artemis R_PKG_CACHE_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: amd64 - AGENT: artemis R_PKG_CACHE_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: amd64 - AGENT: artemis R_PKG_CACHE_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 4 - ARCH: arm64 - AGENT: gaia R_PKG_CACHE_DIR: /mnt/cache/pkgcache SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: arm64 - AGENT: gaia R_PKG_CACHE_DIR: /mnt/cache/pkgcache SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: arm64 - AGENT: gaia R_PKG_CACHE_DIR: /mnt/cache/pkgcache SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: arm64 - AGENT: gaia R_PKG_CACHE_DIR: /mnt/cache/pkgcache SPLIT_INTO: 4 SPLIT_INDEX: 4 diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index ee5b7cc..9d40c67 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -5,7 +5,7 @@ # its matching matrix row (via the per-row `cron:` name filter). # - manual: pick a target from the `process_cran_updates` dropdown # ("all" = every os/arch). -# Arch placement is handled by the agent label (artemis=amd64, gaia=arm64). +# Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: process_cran_updates: description: "Manual run target: a specific -, or 'all' for every os/arch." @@ -36,104 +36,90 @@ when: skip_clone: true labels: - agent: ${AGENT} + group: rpkgs-${ARCH} matrix: include: - OS: alpine-322 ARCH: amd64 - AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine322 PROCESS_NEW: "FALSE" - OS: alpine-322 ARCH: arm64 - AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine322 PROCESS_NEW: "FALSE" - OS: alpine-323 ARCH: amd64 - AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine323 PROCESS_NEW: "FALSE" - OS: alpine-323 ARCH: arm64 - AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine323 PROCESS_NEW: "FALSE" - OS: redhat-8 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 PROCESS_NEW: "TRUE" - OS: redhat-8 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 PROCESS_NEW: "TRUE" - OS: redhat-9 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 PROCESS_NEW: "TRUE" - OS: redhat-9 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 PROCESS_NEW: "TRUE" - OS: redhat-10 ARCH: amd64 - AGENT: artemis R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 PROCESS_NEW: "TRUE" - OS: redhat-10 ARCH: arm64 - AGENT: gaia R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 PROCESS_NEW: "TRUE" - OS: ubuntu-2204 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy PROCESS_NEW: "TRUE" - OS: ubuntu-2204 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy PROCESS_NEW: "TRUE" - OS: ubuntu-2404 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble PROCESS_NEW: "TRUE" - OS: ubuntu-2404 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index b10c108..6ff71e3 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -5,7 +5,7 @@ # its matching matrix row (via the per-row `cron:` name filter). # - manual: pick a target from the `weekly_audit_missing` dropdown # ("all" = every os/arch). -# Arch placement is via the agent label (artemis=amd64, gaia=arm64). +# Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64). variables: weekly_audit_missing: description: "Manual run target: a specific -, or 'all' for every os/arch." @@ -38,88 +38,72 @@ when: skip_clone: true labels: - agent: ${AGENT} + group: rpkgs-${ARCH} matrix: include: - OS: alpine-321 ARCH: amd64 - AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-321 ARCH: arm64 - AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-322 ARCH: amd64 - AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-322 ARCH: arm64 - AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-323 ARCH: amd64 - AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-323 ARCH: arm64 - AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: redhat-8 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: redhat:8 - OS: redhat-8 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: redhat:8 - OS: redhat-9 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: redhat:9 - OS: redhat-9 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: redhat:9 - OS: redhat-10 ARCH: amd64 - AGENT: artemis R_VERSION: 4.5.3 IMG: redhat:10 - OS: redhat-10 ARCH: arm64 - AGENT: gaia R_VERSION: 4.5.3 IMG: redhat:10 - OS: ubuntu-2204 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: ubuntu:jammy - OS: ubuntu-2204 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: ubuntu:jammy - OS: ubuntu-2404 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: ubuntu:noble - OS: ubuntu-2404 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: ubuntu:noble diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index f6bf9e5..843429e 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -6,7 +6,7 @@ # - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the # previous bare manual trigger that ran every os/arch); pick a # single - to run just one. -# Arch placement is handled by the agent label (artemis=amd64, gaia=arm64). +# Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: weekly_rebuild_missing: description: "Manual run target: a specific -, or 'all' for every os/arch." @@ -37,78 +37,64 @@ when: skip_clone: true labels: - agent: ${AGENT} + group: rpkgs-${ARCH} matrix: include: - OS: alpine-322 ARCH: amd64 - AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-322 ARCH: arm64 - AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-323 ARCH: amd64 - AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-323 ARCH: arm64 - AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: redhat-8 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: redhat:8 - OS: redhat-8 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: redhat:8 - OS: redhat-9 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: redhat:9 - OS: redhat-9 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: redhat:9 - OS: redhat-10 ARCH: amd64 - AGENT: artemis R_VERSION: 4.5.3 IMG: redhat:10 - OS: redhat-10 ARCH: arm64 - AGENT: gaia R_VERSION: 4.5.3 IMG: redhat:10 - OS: ubuntu-2204 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: ubuntu:jammy - OS: ubuntu-2204 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: ubuntu:jammy - OS: ubuntu-2404 ARCH: amd64 - AGENT: artemis R_VERSION: 4.4.3 IMG: ubuntu:noble - OS: ubuntu-2404 ARCH: arm64 - AGENT: gaia R_VERSION: 4.4.3 IMG: ubuntu:noble From fa955c10bf7450b1a2188620e799ac7dba6383bf Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 17 Jun 2026 17:49:04 +0000 Subject: [PATCH 034/125] fix: recompute package snapshot when missing from cache (#95) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `build-all.R` reads three snapshot files from `/mnt/cache/packages/` that the `build-all-versions-install-deps` step precomputes: `pkgs_to_build.rds`, `r_minor_sensitive_pkgs.rds`, and `s3_cache.rds`. That cache volume is **per-agent**, so a build job scheduled on a different (fresh) agent than the one that ran install-deps finds the snapshot absent and dies at `readRDS` (`cannot open compressed file '/mnt/cache/packages/pkgs_to_build.rds'`). This adds a conditional guard at the top of `build-all.R`: when any of the three files is missing, it sources `local/packages-to-build.R` (which has all needed creds via `PGPASS` / `B2_S3_*` env, already present in the build step) and writes the derived `.rds` files — exactly mirroring the install-deps command. - The first build job on a fresh agent repopulates the shared cache, so subsequent jobs on that agent reuse it. - Concurrent jobs that also miss simply redo the work (accepted tradeoff vs. slow shared storage like NFS). - Saves use a temp-file + atomic `file.rename`, so a concurrent reader never sees a half-written `.rds`. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/95 --- local/build-all.R | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/local/build-all.R b/local/build-all.R index 8ce1cd5..f8e8efd 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -15,6 +15,31 @@ library(bincraft, quietly = TRUE) library(future) plan("sequential") +# The install-deps step precomputes the package snapshot into /mnt/cache, but +# that volume is per-agent: a job landing on a fresh agent (or racing +# install-deps) finds it empty. Recompute the snapshot here when any part is +# missing, so the first job on an agent repopulates the cache for the jobs that +# follow; concurrent jobs that also miss simply redo the work. Write via a +# temp file + atomic rename so a concurrent reader never sees a half-written rds. +package_cache_files <- c( + "/mnt/cache/packages/pkgs_to_build.rds", + "/mnt/cache/packages/r_minor_sensitive_pkgs.rds", + "/mnt/cache/packages/s3_cache.rds" +) +if (!all(file.exists(package_cache_files))) { + message("Package snapshot missing from cache; recomputing via packages-to-build.R") + dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) + save_rds_atomic <- function(obj, path) { + tmp <- paste0(path, ".tmp.", Sys.getpid()) + saveRDS(obj, tmp) + file.rename(tmp, path) + } + source(file.path("local", "packages-to-build.R")) + save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds") + save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds") + message("Package snapshot recomputed.") +} + pkgs <- if (sensitive_only) { readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds") } else { From f9bc72d5ab69c63ea6b474e404360ffbaca3597f Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 18 Jun 2026 09:24:53 +0200 Subject: [PATCH 035/125] chore: bump bincraft to 4.2.3 --- .crow/archive-missed-packages.yaml | 2 +- .crow/build-all-versions.yaml | 4 ++-- .crow/process-updates.yaml | 4 ++-- .crow/weekly-rebuild-missing.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index f1fe920..8ea940b 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,7 +62,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), workers = 2L)' backend_options: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 639ce80..381f975 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -111,7 +111,7 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages @@ -152,4 +152,4 @@ steps: cpu: 1000m limits: memory: 20Gi - cpu: 2000m \ No newline at end of file + cpu: 2000m diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 9d40c67..f15b811 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -164,7 +164,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run @@ -180,7 +180,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 843429e..34651ba 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -129,7 +129,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' From 0f4330c05ca2580489e8903a9e7cc1f1d0afb818 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 18 Jun 2026 11:27:02 +0000 Subject: [PATCH 036/125] fix(ci): make build-all self-sufficient on agents without install-deps (#96) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes the recurring `build-all-*` failure on agents where `build-all-versions-install-deps` did **not** run. Both symptoms in the logs — `Parameter 3 does not have length 1` (repeated, in the metadata DB queries) and `argument is of length zero` (system-dependency install) — were the same bug: inside bincraft, `platform` was zero-length. `local/build-all.R` calls `build_binary_package()` without passing `platform`, so bincraft resolves it from the container codename. The primary build step never (re)installed/pinned bincraft and relied on whatever sat in the **per-agent** cache volume; `depends_on` only orders steps, it does not co-locate them on the same agent, so a job landing where install-deps never ran got a stale bincraft that left `platform` empty. ## Changes - **`build-all-versions.yaml`**: pin bincraft `@v4.2.3` in the primary build step (mirroring the R-minor pass and `process-updates.yaml`), so every agent uses a known-good bincraft regardless of where install-deps ran. - Align the R-minor pass `v4.2.2 → v4.2.3`. - Export `OS`/`OS_VERSION`/`ARCH` as runtime env vars — previously only available for `${...}` interpolation, so `build-all.R`'s already-built dedup query matched platform `"-"` and skipped nothing. - Fix the unarchive call: pass the codename via `bincraft::set_codename(NULL)` instead of the malformed `paste(OS, OS_VERSION)` (`"alpine 3.24"`), matching `archive-missed-packages.yaml`. - **`build-all-versions-install-deps.yaml`**: pin install-deps to `@v4.2.3` (was installing HEAD), so the precomputed snapshot and per-agent library stay consistent pipeline-wide. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/96 --- .crow/build-all-versions-install-deps.yaml | 5 +++-- .crow/build-all-versions.yaml | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index ad51300..97610df 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -62,8 +62,9 @@ steps: # - rm -rf /mnt/cache/R-pkgs - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + # Pin the same bincraft version the build steps use, so the precomputed + # snapshot and the per-agent library stay consistent across the pipeline. + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 381f975..b2ede89 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -90,6 +90,13 @@ steps: from_secret: GITHUB_PAT # normal env vars GIT_USER: pat-s + # Export the platform selectors as runtime env vars. They are otherwise + # only available for ${...} interpolation (image/volume), so build-all.R's + # `Sys.getenv("OS")/("OS_VERSION")/("ARCH")` would be empty and its + # already-built dedup query would match platform "-" and skip nothing. + OS: ${OS} + OS_VERSION: ${OS_VERSION} + ARCH: ${ARCH} # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR} R_LIBS_USER: /mnt/cache/R-pkgs @@ -100,6 +107,14 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # The primary pass must not rely on build-all-versions-install-deps having + # run on *this* agent: depends_on only orders the steps, but the cache + # volume is per-agent, so a job landing on an agent where install-deps did + # not run would otherwise use a stale bincraft (which resolves `platform` + # to a zero-length value and breaks every metadata query and the sysdeps + # install). Pin bincraft here, exactly like the R-minor pass below. + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -114,8 +129,9 @@ steps: R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done - # archive missed packages - - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" + # archive missed packages; first arg is the codename (e.g. "alpine324"), + # derived via bincraft like the upload step, not paste(OS, OS_VERSION). + - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(bincraft::set_codename(NULL), Sys.getenv('ARCH'), workers = $NCPUS)" backend_options: docker: resources: From 83c761b73670461772615543776d145db5e34978 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 18 Jun 2026 11:39:11 +0000 Subject: [PATCH 037/125] feat(ci): OS/OS_VERSION manual dropdowns + restore lost crow fix (#97) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Two changes: 1. **`OS`/`OS_VERSION` manual-run dropdowns** — give these form variables explicit `options:` lists (like `target_arch` and `R_VERSION`), so the manual-run form shows dropdowns instead of free-text, in both `build-all-versions.yaml` and `build-all-versions-install-deps.yaml`. Crow form variables are independent (no cascading), so the operator still has to pick a coherent `OS` + `OS_VERSION` combination (e.g. `redhat` + `9`, not `alpine` + `jammy`). 2. **Repairs `main`** — the crow fix from PR #96 (`bc2f6f1`) was lost when that PR was squashed (only the first commit was captured). As a result `main` currently carries the `OS: ${OS}` env vars that break Crow parsing (`unable to parse variable name`) and the unfixed `build-all.R`. This PR re-applies that fix: drop the env additions and derive `platform`/`arch` inside `build-all.R` from the container (bincraft codename → platform mapping + `Sys.info()` arch). ## Notes - `OS_VERSION` options are quoted strings so tags like `8`/`9`/`10` aren't parsed as integers. - Validated: both YAMLs parse, `build-all.R` parses. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/97 --- .crow/build-all-versions-install-deps.yaml | 15 ++++++++++-- .crow/build-all-versions.yaml | 22 ++++++++++-------- local/build-all.R | 27 ++++++++++++++++++++-- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 97610df..1c2df64 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -10,10 +10,21 @@ variables: - arm64 default: amd64 OS: - description: "Base OS image name (e.g. alpine, redhat, ubuntu)." + description: "Base OS image name." + options: + - alpine + - redhat + - ubuntu default: alpine OS_VERSION: - description: "OS version / image tag (e.g. 3.24, 9, jammy, noble)." + description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + options: + - "3.24" + - "8" + - "9" + - "10" + - "jammy" + - "noble" default: "3.24" R_VERSION: description: "Primary R version under /opt/R." diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index b2ede89..ed392fd 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -11,10 +11,21 @@ variables: - arm64 default: amd64 OS: - description: "Base OS image name (e.g. alpine, redhat, ubuntu)." + description: "Base OS image name." + options: + - alpine + - redhat + - ubuntu default: alpine OS_VERSION: - description: "OS version / image tag (e.g. 3.24, 9, jammy, noble)." + description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + options: + - "3.24" + - "8" + - "9" + - "10" + - "jammy" + - "noble" default: "3.24" R_VERSION: description: "Primary R version under /opt/R." @@ -90,13 +101,6 @@ steps: from_secret: GITHUB_PAT # normal env vars GIT_USER: pat-s - # Export the platform selectors as runtime env vars. They are otherwise - # only available for ${...} interpolation (image/volume), so build-all.R's - # `Sys.getenv("OS")/("OS_VERSION")/("ARCH")` would be empty and its - # already-built dedup query would match platform "-" and skip nothing. - OS: ${OS} - OS_VERSION: ${OS_VERSION} - ARCH: ${ARCH} # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR} R_LIBS_USER: /mnt/cache/R-pkgs diff --git a/local/build-all.R b/local/build-all.R index f8e8efd..6c3725c 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -68,8 +68,31 @@ chunk <- chunk[!chunk$Package %in% exclude, ] # pkgs_to_build.rds is a static snapshot from the install-deps step, so on a # restart it still lists everything an interrupted run already produced. The # metadata DB reflects that progress, so we re-derive the remaining set here. -platform <- paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-") -arch <- Sys.getenv("ARCH") +# Derive platform + arch from the running container, mirroring the codename -> +# platform mapping bincraft uses internally. The OS/OS_VERSION selectors are +# workflow-level CI variables that are not injected into the container +# environment, so Sys.getenv() would return "" and this pre-filter would query +# platform "-" and skip nothing. +codename <- bincraft::set_codename(NULL) +platform <- switch( + codename, + jammy = "ubuntu-2204", + noble = "ubuntu-2404", + resolute = "ubuntu-2604", + rhel10 = "redhat-10", + rhel9 = "redhat-9", + rhel8 = "redhat-8", + alpine320 = "alpine-320", + alpine321 = "alpine-321", + alpine322 = "alpine-322", + alpine323 = "alpine-323", + alpine324 = "alpine-324", + alpine325 = "alpine-325", + alpine326 = "alpine-326", + NA_character_ +) +local_machine <- Sys.info()[["machine"]] +arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" con <- DBI::dbConnect( RPostgres::Postgres(), dbname = "build_metadata", From 3420523849c29fbc75a4968fabdd70922dce4c7c Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 18 Jun 2026 13:40:13 +0200 Subject: [PATCH 038/125] chore: add all options for OS_VERSION --- .crow/build-all-versions.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index ed392fd..494641c 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -20,12 +20,15 @@ variables: OS_VERSION: description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." options: + - "3.22" + - "3.23" - "3.24" - "8" - "9" - "10" - "jammy" - "noble" + - "resolute" default: "3.24" R_VERSION: description: "Primary R version under /opt/R." From a6788f9044956f74eb83d15d8c49db90b5ab1789 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sat, 20 Jun 2026 00:31:34 +0000 Subject: [PATCH 039/125] chore(deps): update pre-commit hook posit-dev/air-pre-commit to v0.10.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6784ad2..b7953ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit - rev: 0.9.0 + rev: 0.10.0 hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker From b46472b59eec75e4b56aee364b30113e4a8d21b7 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 21 Jun 2026 10:54:33 +0200 Subject: [PATCH 040/125] fix(ci): pass Backblaze B2 credentials to process_unarchived_pkgs The function defaults to Hetzner S3 with HETZNER_S3_*_K3S env vars that are not set in these workflows, so paws failed with "No compatible credentials provided". Pass the B2 endpoint/region/bucket and B2_S3_* secrets explicitly, matching the other S3 calls. Also use set_codename(NULL) for the build-all-versions codename so it matches the S3 repo path. --- .crow/archive-missed-packages.yaml | 2 +- .crow/build-all-versions.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 8ea940b..673415c 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -64,7 +64,7 @@ steps: commands: - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), workers = 2L)' + - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' backend_options: kubernetes: resources: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 494641c..f495550 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -138,7 +138,7 @@ steps: done # archive missed packages; first arg is the codename (e.g. "alpine324"), # derived via bincraft like the upload step, not paste(OS, OS_VERSION). - - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(bincraft::set_codename(NULL), Sys.getenv('ARCH'), workers = $NCPUS)" + - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(bincraft::set_codename(NULL), Sys.getenv('ARCH'), s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), workers = $NCPUS)" backend_options: docker: resources: From a1d7064fb8fd15cc560c5f3690ce7d10d994dc8a Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 21 Jun 2026 11:04:24 +0200 Subject: [PATCH 041/125] feat(ci): add process-updates rows for alpine-324 and ubuntu resolute Add matrix rows and manual dropdown options for alpine 3.24 (alpine-324, PROCESS_NEW=FALSE) and ubuntu resolute / 26.04 (ubuntu-2604, OS_ID resolute, R 4.5.3, PROCESS_NEW=TRUE). --- .crow/process-updates.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index f15b811..e4c09ae 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -15,6 +15,8 @@ variables: - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -25,6 +27,8 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 default: all when: @@ -64,6 +68,18 @@ matrix: IMG: alpine:3.24 OS_ID: alpine323 PROCESS_NEW: "FALSE" + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + OS_ID: alpine324 + PROCESS_NEW: "FALSE" + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + OS_ID: alpine324 + PROCESS_NEW: "FALSE" - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 @@ -124,6 +140,18 @@ matrix: IMG: ubuntu:noble OS_ID: noble PROCESS_NEW: "TRUE" + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + OS_ID: resolute + PROCESS_NEW: "TRUE" + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + OS_ID: resolute + PROCESS_NEW: "TRUE" steps: - name: 'Processing Updates' From 81ca78edb63bdc5e43643fc46acd9ce7959434aa Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 23 Jun 2026 00:31:42 +0000 Subject: [PATCH 042/125] chore(deps): update terraform bunnynet to ~> 0.15 --- .terraform.lock.hcl | 72 ++++++++++++++++++++++----------------------- provider.tf | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 2ae039e..6ef10fa 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.14.3" - constraints = "~> 0.14" + version = "0.15.1" + constraints = "~> 0.15" hashes = [ - "h1:2VmbbvV/3fVb/dHZ5m3CxSPoptpn/PuhvSgV/RA9Ae8=", - "h1:4MANuttWPyJGZlfbx5P4i2Jpbmvda1tfWgd3olK2nhY=", - "h1:AsSfDvm6flYQgjFnFVaRPv7v1/P5pj36dQ4N103Nw+A=", - "h1:D2AK5psWQmZqrOHsbeV70OP/zcG92i++AmGKBz/HVMA=", - "h1:DpJWi/bhHoZa/ccy2YB/c7HJjTlR6CuZBxGFCxU2uJk=", - "h1:Iz47FCzHBOZC1RrTxKwJwKsH6bdEvdYtO0+RBpbmz8I=", - "h1:RJQm4qEfljAmcJaFf45uXwUhtbRTwfDuiUWZ6Q7X+Cw=", - "h1:YQBi0MRW+dOfkxQYbjKkzDm7UIcc2FeFm8Cck+W3kgs=", - "h1:bWq/zJ2Chr5TNuz9y3eaGmzrmXsX30pl5HkGpxE1kVM=", - "h1:hwzEskDnFHaJFvKNAGv/wIxRT2SHcXLfwmBb8XNspq4=", - "h1:jGggry0lZmG0S0inM7Odae3275na9Gp3O97ZlvBiA2s=", - "h1:mJdpVc9qOpqXcPpptWfAET8qloo6qFMb+NzHesazybk=", - "h1:nbpkaopSGlmxkKLtNe9sKx9IqojoicD1+hNRdaYd5ok=", - "h1:pwuoUqikuVOut3RtPTnu6pNoDnyeXlEX7Rfb2Hr1Ch0=", - "h1:u0naFiWGrBVhI3AsUb/ZfS5N/GeiRk0ZkxjHtA10lF0=", - "h1:uozoSPzZ7MJIHj5O62W8n86NIfH4JtF7GvMNI6rKlrM=", - "h1:vDS308OeXVVgjojMCAAEQ1VshmOLjwra7obliKB5fv8=", - "zh:23200ad919df4c44a720c667c536d3a0ae0c235e1f7b53b9e56f88005bf01891", - "zh:2fe5445f578c6a244995538fde6dd7ef0dd5f3969686cba3d91fcf28ea31278d", - "zh:3a5a8139ed177ccbee74feb4d19c881e18723e4bf3a64b6d35931420402a2a1a", - "zh:5189bd6c03ec3c428185992f519f3bf520d4e9eba8771c240814adbecdf60140", - "zh:532ac8eb94a84ccf046d4cb4eded825269effdbb7597f392f848fbfad5dcae1d", - "zh:5f100279d73ce998c7bf4ffa2bbcd617ee307bf51c4a94146f210aeec268c4d0", - "zh:80cb58abd2e431386ab1b6ad186a6e6e5347448993bacc831c0251b9016445b9", + "h1:/2NUpbtjkc+w6n5V3kGP0rSzGjN0K2Wdfe2K+CZdmhU=", + "h1:1TOrpmCR0aT5xX1sjt70zqlYkMJnVe6r0nx2B0DS/mE=", + "h1:4uC8r8ILr+vZJ230uGqIUaWQ7uORCrIzoEuYrAq2JuE=", + "h1:6b+2osJUfwcaYZ6mathLPf/58sr/4XkLXQrcSufnkMk=", + "h1:CPldfjf79QS8mIP+GWoS0FVhrFlyjvN2+39zfyP76Ik=", + "h1:EvXICHyGIKpoYlDeKHOPzfmpvdRdhgsOMcR2nb6+tKY=", + "h1:GZLq+nDxS1CyBH0ELGTSQj9X7ozemJ1jpPA4KwbtR+c=", + "h1:ISNFqL745IQgZ6yMLy8ofV8ixbYqZYa9JKdi2W3pmNk=", + "h1:IrNrEuvFd0nYDGQefwmT8d1CSJb9e8LN5w9vw1ODp7E=", + "h1:Ms79slY9bZ94+n4cwIHvI9+/cvbucwo6S1+z5KAiznw=", + "h1:NWA9XSEBcpSkgwwIvl6tHrxGQY3uYhqNS4Vnb9RLyLQ=", + "h1:UjvxxxggicLtiE3yTe1Gx0oLUTeZpWmgIfXuHzWHn1c=", + "h1:VgJjo14DGkU4Jwo4D3GT4/5sq1tdjiZscKS5l3cb890=", + "h1:dBu3AW5YNLIvbBIMNk3wUHKw4TW+BDbj34a+mCqYhWE=", + "h1:i5oGD06nQ3JRsBIa2u4wCej+ETgp970CFl75dOKkHno=", + "h1:mTqR+vD1AWPx+mu7S0/pzBy71z7WOrjH7arOP77PXh4=", + "h1:nmTM61G8vYjpofeEqspMORpsNvTGCNZySGfjdXardL0=", + "zh:0f9bf5aaa47164a4d6ae4433d5e285a9456a5053401b2bad4ed68622f574ddee", + "zh:3039bee421fb8855a919f449fc731d145254371f5f0c39cc4660f3aeed6a8b10", + "zh:36664b08186e0c194747b18dee24ed97327c6e704133d4cf0df27abef1652f86", + "zh:3c7eae99d8c5ff65dfb99c8b9c1980147282d68971e1ef1ff0f126a6bff59d8f", + "zh:5293cc21abf54f4e5745437ca2d40d206aae323b2e1d41cf45dcc63a8868cbb3", + "zh:5994e5145e616e7e881010717e4c7def2945eb6d933f62db4ec3167732ccac84", + "zh:7994db9ed3fdb6cbf21f2154ea962cf82f04e425988bedc6657de2497d6cb6c3", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9ac734c6aae19ea0832269f1f1dc3c3e100855c09b6168eecd8693958515b228", - "zh:a0e3b75176c71e5f41a6552cef57ad67d11778cb59fd2eb792e69f662b67ae01", - "zh:b4d0e1dd9a80d1ac9a3f1beaefb58d4b944cb0a30bdc672b9037a69b4dc20ec3", - "zh:b8309337e966528991cc2dd61ff0247864e4c3c7bef7dcf225f066edb6171050", - "zh:c34ce5fcdd481214697505bceb616a7dfcda7702f1afefa7dc5459965b8e01ca", - "zh:cb3f662ad06bb86aa763a5644ba318bc3db0ed30b04eb810c478e81f4b012d55", - "zh:cf0f0a459e2b6b1e014af9f53f32b5f555a249b61a0dcaccd3e4fe89178d8aa5", - "zh:d4dfb653a0295bfbbebcb4bb022b490f2c24ca31287ebf77fe9dcaee2e5ea658", - "zh:e49ba31aa092a30ea8a1091f7b26dbe41c3998f924f5b0d82810f27fc2f63501", - "zh:ea4d7d5688caf2f9cda3a1721779b61a051db6b43ca7267ae2712bd1262cfcfd", + "zh:93aa863e536ba9376ccf9e614e9edc9b214a2ce8c4316d416a8e249b436f52d2", + "zh:ab5cb4baeda57559686a0ccf0e09158aa64624ee6ba0ef32b769f13b11a43068", + "zh:ae9388b62eede8fd9272407bf75f8241a965bd489d45ec9dd3f9fac696d500e1", + "zh:caa5befd16960e2f69c7ec483e228e5ff43ab0979c17f1b874c9ffaa1c7c0e43", + "zh:cddd3e1067defa06a4e4ad5cb3940c7943e29c42417de57236aa7d3e2aeaae13", + "zh:cdfa44d591d0805116159556947904d70f534c6816188c45cf3a7544d2722ac9", + "zh:d607e9f1f3e09f13404f219e1893e3b3c77aece4afb54e999f934c021f41f576", + "zh:d8a397aca95125c6a0c0c78d2ded5843b9204effa9f5cf7419f017b500ad9228", + "zh:f5499eaff0d221725ad209d27d87c5b46d5c554caad7dc42947c760377abe3b0", + "zh:fc5f5cf433abc83e5169fa222992ec521c0c802075970251e3dd2c1d50c4f5c1", ] } diff --git a/provider.tf b/provider.tf index 0a9ba15..b267fcc 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.14" + version = "~> 0.15" } } } From 3ffa25ddac5a961df05dc675233abf9543287fa8 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sun, 28 Jun 2026 00:32:21 +0000 Subject: [PATCH 043/125] chore(deps): update pre-commit hook rbubley/mirrors-prettier to v3.8.5 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7953ae..44d02f0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.8.4 + rev: v3.8.5 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit From 597f759bb24000d92068ad765ebbba9ec5b859ab Mon Sep 17 00:00:00 2001 From: automation-bot Date: Mon, 29 Jun 2026 00:31:27 +0000 Subject: [PATCH 044/125] chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker to v3.8.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 44d02f0..c2ae531 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.7.0 + rev: v3.8.0 hooks: - id: editorconfig-checker - repo: https://github.com/adrienverge/yamllint.git From 1e910bc703b1f929281a8b8b6437fc23960c572c Mon Sep 17 00:00:00 2001 From: automation-bot Date: Mon, 29 Jun 2026 00:31:40 +0000 Subject: [PATCH 045/125] chore(deps): update pre-commit hook rbubley/mirrors-prettier to v3.9.1 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2ae531..05b89fe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.8.5 + rev: v3.9.1 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit From 55a18fd87d5abcea507b6ce2b4bf1a404b1fbdbb Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 30 Jun 2026 08:50:19 +0000 Subject: [PATCH 046/125] feat: patch registry + wiring for per-package patching (#103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds the curated **patch registry** and wiring that drives bincraft's new package-patching mechanism (see bincraft PR `feat/package-patching`). Lets specific packages be patched (env/configure/Makevars overrides or source diffs) before pak installs them — including as transitive dependencies — so compiler-/OS-specific failures like RcppParallel's bundled TBB stop cascading. ## What's included - `local/patches/registry.json` — initial entry: RcppParallel with `RCPP_PARALLEL_USE_TBB=0` for alpine / ubuntu-2604, plus `local/patches/README.md` schema docs. - `local/validate-patches.R` — validates schema, referenced patch files, and ambiguous overlaps; clean failure + exit 1 (no stacktrace). - `.pre-commit-config.yaml` — a `validate-patches` hook (re-runs when the registry or the validator changes). - `local/build-one.R` / `local/build-all.R` — pass `patches = "local/patches"` to `bincraft::build_binary_package()`. - `specs/2026-06-30-package-patching-design.md` and `plans/2026-06-30-package-patching-implementation.md`. ## ⚠️ Merge ordering (blocker) This PR adds a `patches = ...` argument to `build_binary_package()` calls. The `.crow/*.yaml` workflows currently pin bincraft **v4.2.3**, which does not accept that argument — CI will error with `unused argument (patches=...)` until: 1. bincraft **v4.3.0** is released (PR `feat/package-patching`), and 2. the pin is bumped in `.crow/build-all-versions-install-deps.yaml`, `.crow/build-all-versions.yaml`, and `.crow/process-updates.yaml`. The `.crow` pin bump will be added to this PR once bincraft v4.3.0 is tagged. Do not merge before then. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/103 --- .crow/archive-missed-packages.yaml | 2 +- .crow/build-all-versions-install-deps.yaml | 2 +- .crow/build-all-versions.yaml | 4 +- .crow/process-updates.yaml | 8 +- .crow/weekly-rebuild-missing.yaml | 4 +- .gitignore | 3 + .markdownlint-cli2.yaml | 3 + .pre-commit-config.yaml | 6 + README.md | 22 + local/build-all.R | 1 + local/build-one.R | 1 + local/patches/README.md | 43 + local/patches/registry.json | 12 + local/validate-patches.R | 56 + ...6-06-30-package-patching-implementation.md | 1477 +++++++++++++++++ specs/2026-06-30-package-patching-design.md | 157 ++ 16 files changed, 1791 insertions(+), 10 deletions(-) create mode 100644 local/patches/README.md create mode 100644 local/patches/registry.json create mode 100644 local/validate-patches.R create mode 100644 plans/2026-06-30-package-patching-implementation.md create mode 100644 specs/2026-06-30-package-patching-design.md diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 673415c..3feb157 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,7 +62,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' backend_options: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 1c2df64..3a7d92c 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -75,7 +75,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index f495550..076e482 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -121,7 +121,7 @@ steps: # to a zero-length value and breaks every metadata query and the sysdeps # install). Pin bincraft here, exactly like the R-minor pass below. - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -133,7 +133,7 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages; first arg is the codename (e.g. "alpine324"), diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index e4c09ae..b943b2d 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -192,13 +192,13 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, r_minor_detection = 'classifier', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) for RBIN in /opt/R/[0-9]*/bin/R; do @@ -208,8 +208,8 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' || true - R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, r_minor_detection = 'classifier', r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' || true + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 34651ba..5d88dec 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -129,12 +129,12 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.3") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.3")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: docker: resources: diff --git a/.gitignore b/.gitignore index 94e2c2f..f4124a2 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,6 @@ docs/ local/test.R .DS_Store docs/ + +# Superpowers SDD scratch (briefs, reports, ledger) +.superpowers/ diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml index 66e7cc8..032c0ea 100644 --- a/.markdownlint-cli2.yaml +++ b/.markdownlint-cli2.yaml @@ -2,3 +2,6 @@ ignores: - LICENSE.md - docs/superpowers/** + # Internal design docs (specs/plans) are not user-facing reference material. + - specs/** + - plans/** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 05b89fe..7448d6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,3 +43,9 @@ repos: entry: YAML filenames must have .yaml extension. language: fail files: .yml$ + - id: validate-patches + name: validate patch registry + entry: Rscript local/validate-patches.R + language: system + files: ^local/(patches/|validate-patches\.R$) + pass_filenames: false diff --git a/README.md b/README.md index 278a475..d030948 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,28 @@ For every package+tag combination: 1. Archive old package versions and keep the latest one in the root 1. Delete local binaries after successful upload +## Patching packages + +Some CRAN packages fail to compile on specific platforms due to compiler- or OS-specific issues unrelated to the package itself. +The canonical example is `RcppParallel`, whose bundled TBB sources fail on musl (Alpine) and newer compiler/OS combinations. +Because such packages are often transitive dependencies of many others, a single failure cascades: all dependents fail even though nothing is wrong with the dependent itself. + +To address this, frequently-failing packages can be "patched" before they are installed — whether as a direct build target or a transitive dependency pulled in by `pak`. + +The patch registry lives in `local/patches/registry.json`. +Each entry specifies a package and the platforms/versions it applies to, along with either lightweight build-time overrides (environment variables, configure arguments, Makevars) or a source diff (for deeper fixes). +See `local/patches/README.md` for the complete schema. + +Patching uses a two-tier approach: + +1. **Lightweight overrides:** environment variables, configure arguments, or Makevars settings applied during build — typically version-independent and fast. +2. **Source diffs:** unified diff patches applied to the unpacked source before building — more powerful but version-pinned. + +The system is implemented in `bincraft`: when a package needs patching, `bincraft` pre-builds it with the patch and serves the patched binary to `pak`, ensuring transitive dependents receive the fixed package. +This way, the fix cascades to all packages that depend on it. + +For the design rationale and architecture, see `specs/2026-06-30-package-patching-design.md`. + ## Build Environment Binaries are built on a mixed-architecture Kubernetes cluster using CI. diff --git a/local/build-all.R b/local/build-all.R index 6c3725c..c0fe7f7 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -140,6 +140,7 @@ mapply( metadata_db_sslmode = "require", metadata_db_port = 15432, archive = TRUE, + patches = "local/patches", upload = TRUE, store_build_metadata = TRUE ) diff --git a/local/build-one.R b/local/build-one.R index f2d6ebd..07c0b6a 100644 --- a/local/build-one.R +++ b/local/build-one.R @@ -103,6 +103,7 @@ for (ver in versions) { force = TRUE, upload = TRUE, archive = TRUE, + patches = "local/patches", store_build_metadata = TRUE, s3_endpoint = s3$s3_endpoint, s3_region = s3$s3_region, diff --git a/local/patches/README.md b/local/patches/README.md new file mode 100644 index 0000000..af24b2b --- /dev/null +++ b/local/patches/README.md @@ -0,0 +1,43 @@ +# Patch Registry + +This directory contains the curated registry of per-package build-time patches consumed by bincraft's `patches` argument. + +## Schema + +The registry is defined in `registry.json` as an array of patch entries. Each entry specifies lightweight build-time overrides (environment variables, configure arguments, Makevars) and optionally a source diff to apply before building. + +### Field semantics + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `package` | string | yes | CRAN package name. | +| `versions` | string | yes | `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. | +| `platforms` | array of strings | yes | Matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). An entry matches if any listed token matches any build token. `["*"]` matches all platforms. | +| `env` | object | no | Environment variables exported only for this package's isolated build. | +| `configure_args` | array | no | Arguments passed as `--configure-args` to the isolated build. | +| `makevars` | object | no | Key/value pairs written into a package-local Makevars for the isolated build. | +| `patch` | string or null | no | Path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. | +| `reason` | string | yes | Human explanation, surfaced in logs and metadata. | + +## Adding an entry + +To add a new patch entry: + +1. Add an object to the array in `registry.json` with the fields documented above. + Start with lightweight overrides (environment variables, configure arguments, Makevars) before resorting to source diffs. + +2. If a source diff is needed, place it in `local/patches//.patch` and reference its path in the `patch` field. + For example, a diff for `RcppParallel` would go in `local/patches/RcppParallel/fix.patch` and be referenced as `"patch": "RcppParallel/fix.patch"`. + +3. The `reason` field should clearly explain why the patch is needed and what problem it solves. + +## Validation + +The registry is validated and applied by bincraft during the build process. +For manual validation, run the validator from the repo root: + +```bash +Rscript local/validate-patches.R +``` + +This validates the schema, referenced patch-file existence, and checks for duplicate entries across platforms and versions. diff --git a/local/patches/registry.json b/local/patches/registry.json new file mode 100644 index 0000000..80460d6 --- /dev/null +++ b/local/patches/registry.json @@ -0,0 +1,12 @@ +[ + { + "package": "RcppParallel", + "versions": "*", + "platforms": ["alpine", "ubuntu-2604"], + "env": { "RCPP_PARALLEL_USE_TBB": "0" }, + "configure_args": [], + "makevars": {}, + "patch": null, + "reason": "bundled Intel TBB fails to build on musl and on newer toolchains (e.g. g++ 15 on ubuntu-2604); disabling TBB falls back to TinyThread" + } +] diff --git a/local/validate-patches.R b/local/validate-patches.R new file mode 100644 index 0000000..5f5aace --- /dev/null +++ b/local/validate-patches.R @@ -0,0 +1,56 @@ +#!/usr/bin/env Rscript +# Validate local/patches/registry.json: schema, referenced patch files, and +# ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. + +dir <- "local/patches" +registry_file <- file.path(dir, "registry.json") +if (!file.exists(registry_file)) { + cat("No registry.json found; nothing to validate.\n") + quit(status = 0L) +} + +or_q <- function(x) if (is.null(x)) "?" else x + +reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +required <- c("package", "versions", "platforms", "reason") +errs <- character(0L) + +for (i in seq_along(reg)) { + e <- reg[[i]] + missing <- setdiff(required, names(e)) + if (length(missing) > 0L) { + errs <- c(errs, sprintf( + "entry %d (%s): missing %s", i, + if (is.null(e$package)) "?" else e$package, toString(missing) + )) + } + if (!is.null(e$patch)) { + p <- file.path(dir, e$patch) + if (!file.exists(p)) { + errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing", + i, e$package, p)) + } + } +} + +# Ambiguous overlap: two entries for the same package with identical platforms +# and versions. +keys <- vapply(reg, function(e) { + sprintf( + "%s|%s|%s", + or_q(e$package), + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + or_q(e$versions) + ) +}, character(1L)) +dups <- keys[duplicated(keys)] +if (length(dups) > 0L) { + errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups)))) +} + +if (length(errs) > 0L) { + cat("Patch registry validation FAILED:\n") + cat(paste0(" - ", errs, "\n")) + quit(status = 1L) +} +cat(sprintf("Patch registry OK (%d %s).\n", length(reg), if (length(reg) == 1L) "entry" else "entries")) diff --git a/plans/2026-06-30-package-patching-implementation.md b/plans/2026-06-30-package-patching-implementation.md new file mode 100644 index 0000000..3a743d7 --- /dev/null +++ b/plans/2026-06-30-package-patching-implementation.md @@ -0,0 +1,1477 @@ +# Package Patching Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let a curated registry of packages be "patched" (env/configure/Makevars overrides and/or source diffs) before `pak` installs them, including when they are transitive dependencies, so compiler-/OS-specific failures like `RcppParallel` stop cascading. + +**Architecture:** The mechanism lives in `bincraft`. Before `pak::local_install_deps()` runs, bincraft pre-builds each registry-matched package as a binary (reusing `pkgbuild::build(binary=TRUE)` + `cranlike::add_PACKAGES`), caches it, and serves it from a local `file://` repo prepended to `options("repos")` so `pak` installs the patched binary — direct or transitive — without recompiling. The curated registry (`registry.json` + diff files) lives in this repo and is passed in via a new `patches` argument. + +**Tech Stack:** R, `pak`, `pkgbuild`, `cranlike` (fork `pat-s/cranlike@s3`), `pkgsearch`, `withr`, `jsonlite`, `testthat` (3e), `mockery`. Two repos: `bincraft` at `/Users/pjs/git/codefloe.com/rpkgs/bincraftr`, and this repo `build-cran-binaries`. + +## Global Constraints + +- bincraft package name is `bincraft`; repo dir is `…/rpkgs/bincraftr`. Current version `4.2.2.9999`. +- Logging uses bincraft's wrappers only: `log_info`, `log_warn`, `log_success`, `log_debug`, `log_error`, `log_header` (never bare `message`/`cat` in package code). +- `cli`-style inline markup is allowed in log messages (e.g. `{.pkg %s}`, `{.path %s}`); curly braces in dynamic/error text must be escaped as already done in `install_helpers.R`. +- Tests: `testthat` 3rd edition, files at `tests/testthat/test-.R`, network/build tests guarded with `skip_on_cran()` / `skip_if_offline()`; end-to-end build tests guarded behind `skip_if_not(nzchar(Sys.getenv("BINCRAFT_PATCH_E2E")))`. +- New exported functions need roxygen with `@keywords internal` for non-user helpers; run `devtools::document()` after adding roxygen. +- Patched-binary cache dir default: `/mnt/cache/patched-binaries`. +- Platform tokens for matching = `c(, , )`, e.g. `ubuntu-2604` → `c("ubuntu-2604","ubuntu","amd64")`. +- Registry entry required fields: `package`, `versions`, `platforms`, `reason`. Optional: `env`, `configure_args`, `makevars`, `patch`. +- One sentence per line in prose/commit messages; do not hard-wrap at 80 columns. +- Use `fj -H codefloe.com` for any PR operations (Forgejo), not `gh`. + +--- + +## Phase A — bincraft mechanism + +All Phase A paths are relative to `/Users/pjs/git/codefloe.com/rpkgs/bincraftr`. + +### Task A0: Proof of mechanism — pak installs a patched binary from a prepended `file://` repo + +This de-risks the core assumption before building anything on top: that `pak` installs a binary from a local `file://` repo in preference to CRAN for an equal version, and does so without recompiling. If this fails, the contingency (documented in Step 4) is to serve patched *source* and rely on `pkgcache` build-caching — the rest of the plan changes only inside `build_patched_binary()`. + +**Files:** +- Create: `tools/verify-patch-mechanism.R` + +**Interfaces:** +- Produces: a runnable script proving `pak::pkg_install()` resolves a local patched binary over CRAN. No package API. + +- [ ] **Step 1: Write the verification script** + +```r +# tools/verify-patch-mechanism.R +# Proves pak installs a patched binary from a prepended file:// repo instead of +# CRAN's, without recompiling. Run inside a Linux build-env container: +# Rscript tools/verify-patch-mechanism.R +# Exits 0 on success, 1 on failure. + +pkg <- "glue" # small, pure-R CRAN package +sentinel <- "PatchMechanismProof" + +work <- tempfile("verify_") +repo <- file.path(work, "repo", "src", "contrib") +lib <- file.path(work, "lib") +dir.create(repo, recursive = TRUE) +dir.create(lib, recursive = TRUE) + +# 1. Download CRAN source for the current version. +ap <- available.packages(repos = "https://cloud.r-project.org") +ver <- ap[pkg, "Version"] +src <- file.path(work, sprintf("%s_%s.tar.gz", pkg, ver)) +download.file( + sprintf("https://cloud.r-project.org/src/contrib/%s_%s.tar.gz", pkg, ver), + src, mode = "wb" +) + +# 2. Unpack, inject a sentinel field into DESCRIPTION, build a binary. +untar(src, exdir = work) +desc <- file.path(work, pkg, "DESCRIPTION") +writeLines(c(readLines(desc), sprintf("%s: yes", sentinel)), desc) +pkgbuild::build( + file.path(work, pkg), binary = TRUE, vignettes = FALSE, + dest_path = repo, quiet = TRUE +) +built <- list.files(repo, pattern = sprintf("^%s_.*\\.tar\\.gz$", pkg), full.names = TRUE) +file.rename(built[1L], file.path(repo, sprintf("%s_%s.tar.gz", pkg, ver))) +cranlike::add_PACKAGES(sprintf("%s_%s.tar.gz", pkg, ver), repo) + +# 3. Install with the local repo prepended; assert our patched build won. +withr::with_options( + list(repos = c(patched = sprintf("file://%s", dirname(dirname(repo))), + CRAN = "https://cloud.r-project.org")), + pak::pkg_install(pkg, lib = lib, ask = FALSE, upgrade = FALSE) +) + +installed_desc <- file.path(lib, pkg, "DESCRIPTION") +ok <- file.exists(installed_desc) && + any(grepl(sentinel, readLines(installed_desc))) + +if (ok) { + cat("PROOF PASSED: pak installed the patched local binary.\n") + quit(status = 0L) +} else { + cat("PROOF FAILED: pak did not install the patched local binary.\n") + quit(status = 1L) +} +``` + +- [ ] **Step 2: Run the proof in a build-env container** + +Run (amd64 example; use any supported build-env image): + +```bash +just build-single ubuntu 2604 amd64 4.5.0 glue 1.0.0 1 || true # warms the env +docker run --rm -v "$PWD":/work -w /work reg.devxy.io/rpkgs/build-env-ubuntu:2604 \ + Rscript tools/verify-patch-mechanism.R +``` + +Expected: final line `PROOF PASSED: pak installed the patched local binary.` and exit status 0. + +- [ ] **Step 3: Commit** + +```bash +git add tools/verify-patch-mechanism.R +git commit -m "test(patches): prove pak installs a patched binary from a local file:// repo" +``` + +- [ ] **Step 4: Record the outcome / contingency** + +If the proof PASSED, proceed to Task A1 unchanged. +If it FAILED (pak recompiled or picked CRAN's), the mechanism switches to serving patched *source*: in Task A4 `build_patched_binary()` skips `pkgbuild::build()` and instead repackages the patched source tree with `pkgbuild::build(binary = FALSE)`; everything else (registry, matching, cache, repo prepend) is unchanged because `pak` build-caches the compiled result via `pkgcache`. Note the chosen path in the commit message and continue. + +--- + +### Task A1: Registry loading and normalization + +**Files:** +- Create: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Produces: `load_patch_registry(patches_dir)` → `list()` of normalized entries; each entry is a named list with `package`, `versions`, `platforms` (character vector), `env` (named list), `configure_args` (character), `makevars` (named list), `reason`, and `patch_path` (absolute path or `NULL`). `normalize_patch_entry(entry, patches_dir)` → one normalized entry; errors on missing required field or missing patch file. + +- [ ] **Step 1: Write the failing test** + +```r +# tests/testthat/test-patches.R +test_that("load_patch_registry parses and normalizes entries", { + dir <- withr::local_tempdir() + writeLines("--- a patch ---", file.path(dir, "fix.patch")) + jsonlite::write_json( + list(list( + package = "RcppParallel", versions = "*", + platforms = list("alpine", "ubuntu-2604"), + env = list(RCPP_PARALLEL_USE_TBB = "0"), + patch = "fix.patch", reason = "bundled TBB fails" + )), + file.path(dir, "registry.json"), auto_unbox = TRUE + ) + + reg <- load_patch_registry(dir) + + expect_length(reg, 1L) + expect_identical(reg[[1L]]$package, "RcppParallel") + expect_identical(reg[[1L]]$platforms, c("alpine", "ubuntu-2604")) + expect_identical(reg[[1L]]$env$RCPP_PARALLEL_USE_TBB, "0") + expect_identical(reg[[1L]]$configure_args, character(0L)) + expect_true(file.exists(reg[[1L]]$patch_path)) +}) + +test_that("load_patch_registry returns empty list when no registry", { + expect_identical(load_patch_registry(NULL), list()) + expect_identical(load_patch_registry(withr::local_tempdir()), list()) +}) + +test_that("normalize_patch_entry errors on missing required field", { + expect_error( + normalize_patch_entry(list(package = "x"), tempdir()), + "missing required field" + ) +}) + +test_that("normalize_patch_entry errors on missing patch file", { + expect_error( + normalize_patch_entry( + list(package = "x", versions = "*", platforms = "alpine", + reason = "r", patch = "nope.patch"), + tempdir() + ), + "does not exist" + ) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function load_patch_registry". + +- [ ] **Step 3: Write minimal implementation** + +```r +# R/patches.R + +#' Load and validate the patch registry +#' +#' Reads `registry.json` from `patches_dir` and returns normalized entries. +#' +#' @param patches_dir Directory containing `registry.json` and any patch files, +#' or `NULL` to disable patching. +#' @return A list of normalized patch entries (possibly empty). +#' @keywords internal +load_patch_registry <- function(patches_dir) { + if (is.null(patches_dir)) { + return(list()) + } + registry_file <- file.path(patches_dir, "registry.json") + if (!file.exists(registry_file)) { + log_warn(sprintf( + "Patch directory {.path %s} has no registry.json; patching disabled.", + patches_dir + )) + return(list()) + } + raw <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) + lapply(raw, normalize_patch_entry, patches_dir = patches_dir) +} + +#' Normalize and validate a single patch registry entry +#' +#' @param entry A list parsed from `registry.json`. +#' @param patches_dir Directory used to resolve a relative `patch` path. +#' @return The entry with defaults filled and `patch_path` resolved. +#' @keywords internal +normalize_patch_entry <- function(entry, patches_dir) { + required <- c("package", "versions", "platforms", "reason") + missing <- setdiff(required, names(entry)) + if (length(missing) > 0L) { + stop( + sprintf("Patch entry is missing required field(s): %s", toString(missing)), + call. = FALSE + ) + } + entry$platforms <- as.character(unlist(entry$platforms)) + entry$env <- if (is.null(entry$env)) list() else entry$env + entry$configure_args <- if (is.null(entry$configure_args)) { + character(0L) + } else { + as.character(unlist(entry$configure_args)) + } + entry$makevars <- if (is.null(entry$makevars)) list() else entry$makevars + if (!is.null(entry$patch)) { + patch_path <- file.path(patches_dir, entry$patch) + if (!file.exists(patch_path)) { + stop( + sprintf( + "Patch file '%s' for package '%s' does not exist.", + patch_path, entry$package + ), + call. = FALSE + ) + } + entry$patch_path <- patch_path + } else { + entry$patch_path <- NULL + } + entry +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): load and validate the patch registry" +``` + +--- + +### Task A2: Platform matching and version-constraint satisfaction + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: normalized entries from Task A1. +- Produces: `build_platform_tokens(platform, arch)` → character vector; `entry_matches_platform(entry, tokens)` → logical; `match_patch_entries(registry, platform, arch)` → filtered list; `version_satisfies(version, constraint)` → logical (constraint forms: `"*"` handled by caller, `"x.y.z"` exact, `">=x"`, `"<=x"`, `">x"`, `"=5.1.0")) + expect_false(version_satisfies("5.0.0", ">=5.1.0")) + expect_true(version_satisfies("5.1.11-2", "<=5.1.11-2")) + expect_false(version_satisfies("5.1.12", "<=5.1.11-2")) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function build_platform_tokens". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Build platform tokens for patch matching +#' @keywords internal +build_platform_tokens <- function(platform, arch) { + family <- sub("-.*$", "", platform) + unique(c(platform, family, arch)) +} + +#' Does a patch entry apply to the current platform tokens? +#' @keywords internal +entry_matches_platform <- function(entry, tokens) { + any(entry$platforms == "*") || + length(intersect(entry$platforms, tokens)) > 0L +} + +#' Filter registry entries applicable to the current build +#' @keywords internal +match_patch_entries <- function(registry, platform, arch) { + if (length(registry) == 0L) { + return(list()) + } + tokens <- build_platform_tokens(platform, arch) + Filter(function(e) entry_matches_platform(e, tokens), registry) +} + +#' Test whether a version satisfies a single constraint +#' +#' @param version A version string (CRAN style, may contain `-`). +#' @param constraint One of `"x.y.z"`, `"==x"`, `">=x"`, `"<=x"`, `">x"`, `"=|<=|==|>|<)?\\s*(.+)$", constraint) + )[[1L]] + op <- parts[2L] + target <- parts[3L] + v <- package_version(version) + t <- package_version(target) + if (op == "" || op == "==") { + return(v == t) + } + switch( + op, + ">=" = v >= t, + "<=" = v <= t, + ">" = v > t, + "<" = v < t, + FALSE + ) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): platform matching and version-constraint checks" +``` + +--- + +### Task A3: Cache key and version resolution + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: normalized entries. +- Produces: `patch_cache_key(entry, version, platform, arch, r_minor)` → string `"_____"`, where `hash12` covers `env`/`configure_args`/`makevars`/patch bytes; `resolve_patch_version(entry)` → latest CRAN version satisfying `entry$versions`, or `NA_character_`; `describe_patch(entry)` → short human label. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("patch_cache_key is stable and sensitive to env/patch changes", { + e1 <- list(package = "P", env = list(A = "1"), + configure_args = character(0L), makevars = list(), + patch_path = NULL) + e2 <- e1; e2$env <- list(A = "2") + + k1 <- patch_cache_key(e1, "1.0", "alpine-324", "amd64", "4.5") + expect_identical(k1, patch_cache_key(e1, "1.0", "alpine-324", "amd64", "4.5")) + expect_false(identical( + k1, patch_cache_key(e2, "1.0", "alpine-324", "amd64", "4.5") + )) + expect_match(k1, "^P_1.0_alpine-324_amd64_4.5_[0-9a-f]{12}$") +}) + +test_that("resolve_patch_version returns latest for wildcard, NA when unmet", { + local_mocked_bindings( + cran_package = function(pkg) list(Version = "5.1.12"), + .package = "pkgsearch" + ) + expect_identical( + resolve_patch_version(list(package = "RcppParallel", versions = "*")), + "5.1.12" + ) + expect_identical( + resolve_patch_version(list(package = "RcppParallel", versions = ">=9.0")), + NA_character_ + ) +}) + +test_that("describe_patch summarizes the active overrides", { + expect_match( + describe_patch(list(env = list(RCPP_PARALLEL_USE_TBB = "0"), + configure_args = character(0L), makevars = list(), + patch_path = NULL)), + "env: RCPP_PARALLEL_USE_TBB=0" + ) + expect_match( + describe_patch(list(env = list(), configure_args = character(0L), + makevars = list(), patch_path = "/x/fix.patch")), + "source patch" + ) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function patch_cache_key". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Compute the cache key for a patched binary +#' @keywords internal +patch_cache_key <- function(entry, version, platform, arch, r_minor) { + payload <- list( + env = entry$env, + configure_args = entry$configure_args, + makevars = entry$makevars, + patch = if (!is.null(entry$patch_path)) { + readBin(entry$patch_path, "raw", file.size(entry$patch_path)) + } else { + raw(0L) + } + ) + tmp <- tempfile() + on.exit(unlink(tmp), add = TRUE) + saveRDS(payload, tmp) + hash <- substr(unname(tools::md5sum(tmp)), 1L, 12L) + sprintf( + "%s_%s_%s_%s_%s_%s", + entry$package, version, platform, arch, r_minor, hash + ) +} + +#' Resolve the CRAN version to build for a patch entry +#' +#' Returns the latest CRAN version satisfying the entry's `versions` constraint, +#' or `NA_character_` when CRAN's latest does not satisfy it or lookup fails. +#' @keywords internal +resolve_patch_version <- function(entry) { + latest <- tryCatch( + pkgsearch::cran_package(entry$package)$Version, + error = function(e) NA_character_ + ) + if (is.na(latest)) { + return(NA_character_) + } + if (identical(entry$versions, "*") || version_satisfies(latest, entry$versions)) { + return(latest) + } + NA_character_ +} + +#' Short human label describing a patch entry's overrides +#' @keywords internal +describe_patch <- function(entry) { + bits <- character(0L) + if (length(entry$env) > 0L) { + bits <- c(bits, sprintf( + "env: %s", + paste( + names(entry$env), + unlist(entry$env), + sep = "=", collapse = "," + ) + )) + } + if (length(entry$configure_args) > 0L) { + bits <- c(bits, sprintf("configure: %s", toString(entry$configure_args))) + } + if (length(entry$makevars) > 0L) { + bits <- c(bits, "makevars") + } + if (!is.null(entry$patch_path)) { + bits <- c(bits, "source patch") + } + if (length(bits) == 0L) "no-op" else paste(bits, collapse = "; ") +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. (Note: `local_mocked_bindings` requires testthat >= 3.1.7; bincraft uses 3e.) + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): cache key, version resolution, and patch description" +``` + +--- + +### Task A4: Build a patched binary in isolation + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: a normalized entry, a resolved `version`, a `dest_dir`. +- Produces: `download_cran_source(package, version, dest_dir, cran)` → path or `NULL`; `apply_source_patch(patch_path, pkg_src)` → logical; `configure_args_to_build_args(configure_args)` → character; `build_patched_binary(entry, version, dest_dir)` → path to built binary tarball or `NULL`. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("configure_args_to_build_args formats configure args", { + expect_identical(configure_args_to_build_args(character(0L)), character(0L)) + expect_identical( + configure_args_to_build_args(c("--with-foo", "--no-bar")), + "--configure-args=--with-foo --no-bar" + ) +}) + +test_that("apply_source_patch returns FALSE when patch does not apply", { + src <- withr::local_tempdir() + writeLines("unrelated content", file.path(src, "file.txt")) + bad_patch <- tempfile(fileext = ".patch") + writeLines(c( + "--- a/missing.txt", "+++ b/missing.txt", + "@@ -1 +1 @@", "-nope", "+nope2" + ), bad_patch) + expect_false(apply_source_patch(bad_patch, src)) +}) + +test_that("build_patched_binary returns NULL when download fails", { + local_mocked_bindings(download_cran_source = function(...) NULL) + expect_null( + build_patched_binary( + list(package = "P", env = list(), configure_args = character(0L), + makevars = list(), patch_path = NULL), + "1.0", withr::local_tempdir() + ) + ) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function configure_args_to_build_args". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Download a CRAN source tarball for an exact version +#' @keywords internal +download_cran_source <- function( + package, + version, + dest_dir, + cran = "https://cloud.r-project.org" +) { + fname <- sprintf("%s_%s.tar.gz", package, version) + urls <- c( + sprintf("%s/src/contrib/%s", cran, fname), + sprintf("%s/src/contrib/Archive/%s/%s", cran, package, fname) + ) + dest <- file.path(dest_dir, fname) + for (u in urls) { + ok <- tryCatch( + { + utils::download.file(u, dest, mode = "wb", quiet = TRUE) + file.exists(dest) && file.size(dest) > 0L + }, + error = function(e) FALSE + ) + if (isTRUE(ok)) { + return(dest) + } + } + log_warn(sprintf( + "Could not download CRAN source for {.pkg %s} %s.", + package, version + )) + NULL +} + +#' Apply a unified diff to an unpacked source tree +#' +#' Uses `patch -p1 --forward` so an already-applied or non-applying patch fails +#' cleanly (returns FALSE) instead of corrupting the tree. +#' @keywords internal +apply_source_patch <- function(patch_path, pkg_src) { + status <- system2( + "patch", + args = c( + "-p1", "--forward", "--batch", + "-d", shQuote(pkg_src), + "-i", shQuote(patch_path) + ), + stdout = FALSE, stderr = FALSE + ) + identical(status, 0L) +} + +#' Format configure args for `pkgbuild::build(args = ...)` +#' @keywords internal +configure_args_to_build_args <- function(configure_args) { + if (length(configure_args) == 0L) { + return(character(0L)) + } + sprintf("--configure-args=%s", paste(configure_args, collapse = " ")) +} + +#' Build a patched binary for one registry entry, in isolation +#' +#' Downloads CRAN source for `version`, applies the source patch (if any), and +#' builds a binary with the entry's env / configure / Makevars overrides scoped +#' to this build only. Returns the built tarball path, or `NULL` on any failure. +#' @keywords internal +build_patched_binary <- function(entry, version, dest_dir) { + workdir <- tempfile("patch_build_") + dir.create(workdir, recursive = TRUE, showWarnings = FALSE) + on.exit(unlink(workdir, recursive = TRUE, force = TRUE), add = TRUE) + + src_tarball <- download_cran_source(entry$package, version, workdir) + if (is.null(src_tarball)) { + return(NULL) + } + + utils::untar(src_tarball, exdir = workdir) + pkg_src <- file.path(workdir, entry$package) + + if (!is.null(entry$patch_path)) { + if (!apply_source_patch(entry$patch_path, pkg_src)) { + log_warn(sprintf( + "Patch for {.pkg %s} %s did not apply cleanly; skipping patched build.", + entry$package, version + )) + return(NULL) + } + } + + build_env <- entry$env + if (length(entry$makevars) > 0L) { + mk <- tempfile(fileext = ".mk") + writeLines( + vapply( + names(entry$makevars), + function(k) sprintf("%s=%s", k, entry$makevars[[k]]), + character(1L) + ), + mk + ) + build_env$R_MAKEVARS_USER <- mk + } + + tryCatch( + withr::with_envvar(build_env, { + pkgbuild::build( + path = pkg_src, + binary = TRUE, + vignettes = FALSE, + dest_path = dest_dir, + args = configure_args_to_build_args(entry$configure_args), + quiet = TRUE + ) + }), + error = function(e) { + log_warn(sprintf( + "Isolated patched build of {.pkg %s} %s failed: %s", + entry$package, version, conditionMessage(e) + )) + NULL + } + ) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): build a patched binary in isolation from CRAN source" +``` + +--- + +### Task A5: Orchestrate the local patched repo (cache + index) + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: all helpers above. +- Produces: `prepare_patched_repo(patches_dir, platform, arch, r_minor, cache_dir, repo_dir)` → path to a `src/contrib`-style dir containing patched binaries + a `PACKAGES` index, or `NULL` when nothing matched/built. On a cache hit it copies the cached tarball into `repo_dir`; on a miss it builds, then writes the result into `cache_dir`. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("prepare_patched_repo returns NULL when no entries match", { + dir <- withr::local_tempdir() + jsonlite::write_json( + list(list(package = "A", versions = "*", platforms = list("redhat"), + reason = "r")), + file.path(dir, "registry.json"), auto_unbox = TRUE + ) + expect_null( + prepare_patched_repo(dir, "ubuntu-2604", "amd64", "4.5", + cache_dir = withr::local_tempdir(), + repo_dir = withr::local_tempdir()) + ) +}) + +test_that("prepare_patched_repo serves a cached binary and writes an index", { + dir <- withr::local_tempdir() + jsonlite::write_json( + list(list(package = "glue", versions = "*", platforms = list("*"), + env = list(A = "1"), reason = "r")), + file.path(dir, "registry.json"), auto_unbox = TRUE + ) + cache <- withr::local_tempdir() + repo <- withr::local_tempdir() + + local_mocked_bindings( + resolve_patch_version = function(entry) "1.0.0", + build_patched_binary = function(entry, version, dest_dir) { + f <- file.path(dest_dir, sprintf("%s_%s.tar.gz", entry$package, version)) + writeLines("fake binary", f) + f + } + ) + + out <- prepare_patched_repo(dir, "ubuntu-2604", "amd64", "4.5", + cache_dir = cache, repo_dir = repo) + + expect_identical(out, repo) + expect_true(file.exists(file.path(repo, "glue_1.0.0.tar.gz"))) + expect_true(file.exists(file.path(repo, "PACKAGES"))) + # The build result was cached under the key. + expect_length(list.files(cache, pattern = "^glue_1.0.0_.*\\.tar\\.gz$"), 1L) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function prepare_patched_repo". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Prepare a local repo of patched binaries for the current build +#' +#' For each registry entry matching the current platform, ensures a patched +#' binary is present in `repo_dir` (from `cache_dir` if available, else built +#' and then cached) and writes a `PACKAGES` index over them. +#' +#' @param patches_dir Directory with `registry.json`, or `NULL`. +#' @param platform Build platform, e.g. `"ubuntu-2604"`. +#' @param arch Build arch, e.g. `"amd64"`. +#' @param r_minor R `"major.minor"` string, e.g. `"4.5"`. +#' @param cache_dir Persistent cache for patched binaries. +#' @param repo_dir Directory to assemble the local repo in. +#' @return `repo_dir` if at least one patched binary was produced, else `NULL`. +#' @keywords internal +prepare_patched_repo <- function( + patches_dir, + platform, + arch, + r_minor, + cache_dir = file.path("/mnt", "cache", "patched-binaries"), + repo_dir = tempfile("patched_repo_") +) { + entries <- match_patch_entries( + load_patch_registry(patches_dir), platform, arch + ) + if (length(entries) == 0L) { + return(NULL) + } + + dir.create(repo_dir, recursive = TRUE, showWarnings = FALSE) + dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE) + + produced <- 0L + for (entry in entries) { + version <- resolve_patch_version(entry) + if (is.na(version)) { + log_warn(sprintf( + "No CRAN version of {.pkg %s} satisfies '%s'; patch skipped.", + entry$package, entry$versions + )) + next + } + + key <- patch_cache_key(entry, version, platform, arch, r_minor) + cached <- file.path(cache_dir, sprintf("%s.tar.gz", key)) + target <- file.path( + repo_dir, sprintf("%s_%s.tar.gz", entry$package, version) + ) + + if (file.exists(cached)) { + log_info(sprintf( + "Using cached patched binary for {.pkg %s} %s.", + entry$package, version + )) + file.copy(cached, target, overwrite = TRUE) + } else { + log_info(sprintf( + "Applying patch to {.pkg %s} %s [%s]: %s", + entry$package, version, describe_patch(entry), entry$reason + )) + built <- build_patched_binary(entry, version, repo_dir) + if (is.null(built)) { + next + } + if (!identical(normalizePath(built), normalizePath(target))) { + file.copy(built, target, overwrite = TRUE) + } + file.copy(target, cached, overwrite = TRUE) + } + produced <- produced + 1L + } + + if (produced == 0L) { + return(NULL) + } + cranlike::add_PACKAGES( + list.files(repo_dir, pattern = "\\.tar\\.gz$"), + repo_dir + ) + repo_dir +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 5: Update imports and document** + +Add to `DESCRIPTION` `Imports:` (alphabetical) `jsonlite` if not present, and ensure `pkgsearch`, `pkgbuild`, `cranlike`, `withr` are listed (they are). Then: + +```bash +Rscript -e 'devtools::document()' +git add R/patches.R tests/testthat/test-patches.R DESCRIPTION NAMESPACE +git commit -m "feat(patches): orchestrate local patched-binary repo with caching" +``` + +--- + +### Task A6: Wire patched repo into the pak install path + +**Files:** +- Modify: `R/install_helpers.R:333-389` (`run_pak_install_with_mutex`) +- Modify: `R/install-deps.R:23-75` (`install_pkg_sys_deps`) +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: `prepare_patched_repo()`. +- Produces: `run_pak_install_with_mutex(local_clone_dir_single, env_vars, patched_repo = NULL)` — prepends `file://` to `options("repos")` for the install; `install_pkg_sys_deps(package_name, tag, local_clone_dir, platform, aggressive_cleanup = FALSE, patches = NULL, arch = NULL)` — builds the patched repo before installing. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("run_pak_install_with_mutex prepends the patched repo to repos", { + seen <- NULL + local_mocked_bindings( + acquire_pak_mutex = function(...) tempfile(), + release_pak_mutex = function(...) invisible(NULL), + retry_with_backoff = function(func, ...) func() + ) + local_mocked_bindings( + local_install_deps = function(...) { + seen <<- getOption("repos") + invisible(TRUE) + }, + .package = "pak" + ) + + run_pak_install_with_mutex( + tempfile(), list(), patched_repo = "/tmp/patched" + ) + + expect_true(any(grepl("file:///tmp/patched", seen))) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL — `patched_repo` argument not yet accepted / repos not prepended. + +- [ ] **Step 3: Edit `run_pak_install_with_mutex` in `R/install_helpers.R`** + +Change the signature line: + +```r +run_pak_install_with_mutex <- function(local_clone_dir_single, env_vars) { +``` + +to: + +```r +run_pak_install_with_mutex <- function( + local_clone_dir_single, + env_vars, + patched_repo = NULL +) { +``` + +Replace the inner `retry_with_backoff(...)` block (the one wrapping `pak::local_install_deps`) with: + +```r + retry_with_backoff(function() { + withr::with_envvar(env_vars, { + repos <- getOption("repos") + if (!is.null(patched_repo)) { + repos <- c( + patched = sprintf("file://%s", patched_repo), + repos + ) + } + withr::with_options(list(repos = repos), { + # Default to non-verbose (suppressed messages) + suppressMessages(pak::local_install_deps(sprintf( + "%s", + local_clone_dir_single + ))) + }) + }) + }) +``` + +- [ ] **Step 4: Edit `install_pkg_sys_deps` in `R/install-deps.R`** + +Change the signature to add `patches` and `arch`: + +```r +install_pkg_sys_deps <- function( + package_name, + tag, + local_clone_dir, + platform = platform, + aggressive_cleanup = FALSE, + patches = NULL, + arch = NULL +) { +``` + +Immediately before the `run_pak_install_with_mutex(...)` call, insert: + +```r + # Build a local repo of patched binaries (if any apply) and serve it to pak. + r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." + ) + patched_repo <- tryCatch( + prepare_patched_repo(patches, platform, arch, r_minor), + error = function(e) { + log_warn(sprintf("Patch preparation failed: %s", conditionMessage(e))) + NULL + } + ) +``` + +and change the call from: + +```r + run_pak_install_with_mutex( + local_clone_dir_single, + env_vars + ) +``` + +to: + +```r + run_pak_install_with_mutex( + local_clone_dir_single, + env_vars, + patched_repo = patched_repo + ) +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add R/install_helpers.R R/install-deps.R tests/testthat/test-patches.R +git commit -m "feat(patches): serve patched binaries to pak during dep install" +``` + +--- + +### Task A7: Thread `patches` through the public build API + +**Files:** +- Modify: `R/build_binaries.R` (`build_binary_package`, `execute_package_builds`, `build_single_tag`, `handle_system_dependencies`) +- Create: `man-roxygen/param-patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Produces: `build_binary_package(..., patches = NULL)` and the internal chain each carry `patches` down to `install_pkg_sys_deps()`. `handle_system_dependencies(..., patches = NULL)` passes `patches` and `arch` through. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("handle_system_dependencies forwards patches and arch", { + captured <- list() + local_mocked_bindings( + install_pkg_sys_deps = function(package_name, tag, local_clone_dir_single, + platform, patches = NULL, arch = NULL) { + captured <<- list(patches = patches, arch = arch) + invisible(TRUE) + } + ) + handle_system_dependencies( + "RcppParallel", "5.1.11-2", "ubuntu-2604", tempfile(), "amd64", + NULL, NULL, NULL, NULL, NULL, NULL, NULL, + patches = "local/patches" + ) + expect_identical(captured$patches, "local/patches") + expect_identical(captured$arch, "amd64") +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL — `handle_system_dependencies` has no `patches` argument. + +- [ ] **Step 3: Create the roxygen template** + +```r +# man-roxygen/param-patches.R +#' @param patches Optional path to a patch registry directory containing a +#' `registry.json` (and any referenced diff files). When set, matching +#' packages are pre-built as patched binaries and served to `pak` during +#' dependency installation. Defaults to `NULL` (no patching). +``` + +- [ ] **Step 4: Edit the four functions in `R/build_binaries.R`** + +In `build_single_tag()`'s call to `handle_system_dependencies(...)`, add `patches = patches` as the final argument, and add `patches = NULL` to `build_single_tag`'s own signature plus `#' @template param-patches` to its roxygen block. + +Change `handle_system_dependencies` signature to end with `metadata_db_sslmode,` then add `patches = NULL`, and change its inner `install_pkg_sys_deps(...)` call from: + +```r + install_pkg_sys_deps( + package_name, + tag, + local_clone_dir_single, + platform + ) +``` + +to: + +```r + install_pkg_sys_deps( + package_name, + tag, + local_clone_dir_single, + platform, + patches = patches, + arch = arch + ) +``` + +In `execute_package_builds()`, add `patches = NULL` to the signature and pass `patches = patches` into its `build_single_tag(...)` call inside `worker_function`. + +In `build_binary_package()`, add `patches = NULL` to the signature (after `s3_package_cache`), add `#' @template param-patches` to its roxygen, and pass `patches = patches` into the `execute_package_builds(...)` call. + +- [ ] **Step 5: Document, test, and run package check** + +Run: + +```bash +Rscript -e 'devtools::document()' +Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")' +``` + +Expected: PASS for the new test; `man/build_binary_package.Rd` etc. regenerated. + +- [ ] **Step 6: Commit** + +```bash +git add R/build_binaries.R man-roxygen/param-patches.R man NAMESPACE tests/testthat/test-patches.R +git commit -m "feat(patches): thread patches argument through build_binary_package" +``` + +--- + +### Task A8: End-to-end patch test (guarded) and version bump + +**Files:** +- Modify: `tests/testthat/test-patches.R` +- Modify: `DESCRIPTION` (version), `NEWS.md` + +**Interfaces:** +- Produces: a guarded e2e test proving a dependent package builds when its failing dependency is patched. + +- [ ] **Step 1: Add the guarded e2e test** + +```r +test_that("a patched dependency unblocks a dependent build (e2e)", { + skip_if_not(nzchar(Sys.getenv("BINCRAFT_PATCH_E2E"))) + skip_if_offline() + + patches_dir <- withr::local_tempdir() + jsonlite::write_json( + list(list( + package = "RcppParallel", versions = "*", platforms = list("*"), + env = list(RCPP_PARALLEL_USE_TBB = "0"), + reason = "bundled TBB fails on this toolchain" + )), + file.path(patches_dir, "registry.json"), auto_unbox = TRUE + ) + + out <- withr::local_tempdir() + result <- build_binary_package( + "rts2", tag = "latest", local_output_dir_root = out, + upload = FALSE, archive = FALSE, patches = patches_dir + ) + expect_true(isTRUE(result) || identical(result, "skipped")) +}) +``` + +- [ ] **Step 2: Run the e2e test in a container** + +Run: + +```bash +docker run --rm -e BINCRAFT_PATCH_E2E=1 -v "$PWD":/work -w /work \ + reg.devxy.io/rpkgs/build-env-ubuntu:2604 \ + Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")' +``` + +Expected: the e2e test runs (not skipped) and PASSES; build log shows `Applying patch to RcppParallel`. + +- [ ] **Step 3: Bump version and changelog** + +In `DESCRIPTION`, bump `Version:` to `4.3.0.9999`. Prepend to `NEWS.md`: + +```markdown +# bincraft 4.3.0 + +* `build_binary_package()` gains a `patches` argument: a registry of + per-package env / configure / Makevars overrides and source diffs that are + pre-built into patched binaries and served to pak, fixing compiler- and + OS-specific failures (e.g. RcppParallel) including for transitive deps. +``` + +- [ ] **Step 4: Commit** + +```bash +git add tests/testthat/test-patches.R DESCRIPTION NEWS.md +git commit -m "test(patches): guarded end-to-end test; bump to 4.3.0.9999" +``` + +--- + +## Phase B — build-cran-binaries registry and wiring + +All Phase B paths are relative to this repo (`build-cran-binaries`). Phase B depends on a bincraft build that includes Phase A (install the updated bincraft in the build containers, or bump the pinned version in `.crow/build-all-versions-install-deps.yaml`). + +### Task B1: Create the patch registry with the RcppParallel entry + +**Files:** +- Create: `local/patches/registry.json` +- Create: `local/patches/README.md` + +**Interfaces:** +- Produces: the curated registry consumed by bincraft's `patches` argument. + +- [ ] **Step 1: Write the registry** + +```json +[ + { + "package": "RcppParallel", + "versions": "*", + "platforms": ["alpine", "ubuntu-2604"], + "env": { "RCPP_PARALLEL_USE_TBB": "0" }, + "configure_args": [], + "makevars": {}, + "patch": null, + "reason": "bundled Intel TBB fails to build on musl and on newer toolchains (e.g. g++ 15 on ubuntu-2604); disabling TBB falls back to TinyThread" + } +] +``` + +- [ ] **Step 2: Write the README** + +`local/patches/README.md` documents the schema (copy the field table from `specs/2026-06-30-package-patching-design.md`), how to add an entry, and that source diffs go in `local/patches//.patch` referenced by the `patch` field. + +- [ ] **Step 3: Verify it parses** + +Run: `Rscript -e 'x <- jsonlite::fromJSON("local/patches/registry.json", simplifyVector = FALSE); stopifnot(length(x) == 1L, x[[1]]$package == "RcppParallel"); cat("ok\n")'` +Expected: `ok`. + +- [ ] **Step 4: Commit** + +```bash +git add local/patches/registry.json local/patches/README.md +git commit -m "feat(patches): add patch registry with RcppParallel TBB workaround" +``` + +--- + +### Task B2: Registry validator script + +**Files:** +- Create: `local/validate-patches.R` + +**Interfaces:** +- Consumes: `local/patches/registry.json`. +- Produces: a script that exits non-zero on schema violations, missing patch files, or ambiguous overlapping entries. + +- [ ] **Step 1: Write the validator** + +```r +#!/usr/bin/env Rscript +# Validate local/patches/registry.json: schema, referenced patch files, and +# ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. + +dir <- "local/patches" +registry_file <- file.path(dir, "registry.json") +if (!file.exists(registry_file)) { + cat("No registry.json found; nothing to validate.\n") + quit(status = 0L) +} + +reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +required <- c("package", "versions", "platforms", "reason") +errs <- character(0L) + +for (i in seq_along(reg)) { + e <- reg[[i]] + missing <- setdiff(required, names(e)) + if (length(missing) > 0L) { + errs <- c(errs, sprintf( + "entry %d (%s): missing %s", i, + if (is.null(e$package)) "?" else e$package, toString(missing) + )) + } + if (!is.null(e$patch)) { + p <- file.path(dir, e$patch) + if (!file.exists(p)) { + errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing", + i, e$package, p)) + } + } +} + +# Ambiguous overlap: two entries for the same package with identical platforms +# and versions. +keys <- vapply(reg, function(e) { + sprintf("%s|%s|%s", e$package, + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + e$versions) +}, character(1L)) +dups <- keys[duplicated(keys)] +if (length(dups) > 0L) { + errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups)))) +} + +if (length(errs) > 0L) { + cat("Patch registry validation FAILED:\n") + cat(paste0(" - ", errs, "\n")) + quit(status = 1L) +} +cat(sprintf("Patch registry OK (%d entrie(s)).\n", length(reg))) +``` + +- [ ] **Step 2: Run it (expect success on the B1 registry)** + +Run: `Rscript local/validate-patches.R` +Expected: `Patch registry OK (1 entrie(s)).` and exit 0. + +- [ ] **Step 3: Run it against a broken registry (expect failure)** + +Run: + +```bash +cp local/patches/registry.json /tmp/reg.bak +Rscript -e 'writeLines("[{\"package\":\"X\"}]", "local/patches/registry.json")' +Rscript local/validate-patches.R; echo "exit=$?" +cp /tmp/reg.bak local/patches/registry.json +``` + +Expected: prints `validation FAILED` with a missing-field message and `exit=1`. + +- [ ] **Step 4: Commit** + +```bash +git add local/validate-patches.R +git commit -m "feat(patches): add registry validator script" +``` + +--- + +### Task B3: Hook the validator into pre-commit + +**Files:** +- Modify: `.pre-commit-config.yaml` + +**Interfaces:** +- Produces: a local hook that runs `local/validate-patches.R` when the registry or patch files change. + +- [ ] **Step 1: Add the hook** + +Add a `repo: local` hook entry to `.pre-commit-config.yaml`: + +```yaml + - repo: local + hooks: + - id: validate-patches + name: validate patch registry + entry: Rscript local/validate-patches.R + language: system + files: ^local/patches/ + pass_filenames: false +``` + +- [ ] **Step 2: Verify the hook runs** + +Run: `pre-commit run validate-patches --all-files` +Expected: hook passes (`Patch registry OK`). + +- [ ] **Step 3: Commit** + +```bash +git add .pre-commit-config.yaml +git commit -m "ci(patches): validate patch registry in pre-commit" +``` + +--- + +### Task B4: Pass `patches` through the build entry points + +**Files:** +- Modify: `local/build-one.R:99-119` (the `build_binary_package` call) +- Modify: `local/build-all.R:125-145` (the `build_binary_package` call) + +**Interfaces:** +- Consumes: the new bincraft `patches` argument (Phase A) and `local/patches/`. +- Produces: both entry points pass `patches = "local/patches"`. + +- [ ] **Step 1: Edit `local/build-one.R`** + +In the `bincraft::build_binary_package(` call, add as a new argument (e.g. after `archive = TRUE,`): + +```r + patches = "local/patches", +``` + +- [ ] **Step 2: Edit `local/build-all.R`** + +In the `bincraft::build_binary_package(` call, add: + +```r + patches = "local/patches", +``` + +- [ ] **Step 3: Verify the scripts still parse** + +Run: `Rscript -e 'invisible(parse("local/build-one.R")); invisible(parse("local/build-all.R")); cat("parse ok\n")'` +Expected: `parse ok`. + +- [ ] **Step 4: Commit** + +```bash +git add local/build-one.R local/build-all.R +git commit -m "feat(patches): pass patch registry to bincraft build calls" +``` + +--- + +### Task B5: Document the feature in the README + +**Files:** +- Modify: `README.md` + +**Interfaces:** +- Produces: a short "Patching packages" section explaining the registry and linking the design spec. + +- [ ] **Step 1: Add a README section** + +Add a `## Patching packages` section after the "Build Process" section describing: why patching exists (compiler/OS-specific failures cascading via shared deps like RcppParallel), where the registry lives (`local/patches/registry.json`), the two tiers (env/configure/Makevars overrides vs source diffs), and that bincraft pre-builds patched binaries served to pak. Link `specs/2026-06-30-package-patching-design.md`. + +- [ ] **Step 2: Commit** + +```bash +git add README.md +git commit -m "docs(patches): document the package patching workflow" +``` + +--- + +## Self-Review + +**Spec coverage:** +- Registry in this repo, passed to bincraft → Task A7 (`patches` arg), B1 (registry), B4 (wiring). ✓ +- Mechanism in bincraft (resolve → pre-build → prepend repo → install) → A4–A6. ✓ +- Both tiers, env-first → A4 (`env`, `configure_args`, `makevars`, `patch` all applied; env is the cheap default and the RcppParallel entry uses only env). ✓ +- Pre-built patched binary served from a prepended local repo → A0 proof + A5/A6. ✓ +- Cache keyed by pkg/version/platform/arch/rminor/patchhash → A3 (`patch_cache_key`) + A5 (cache use). ✓ +- S3 upload of patched binaries → **partial**: A5 caches locally only. Cross-machine S3 reuse is deferred (see note below) to keep the first cut shippable; local `/mnt/cache` reuse already removes per-dependent rebuilds within a container. Recorded as out-of-first-cut, not dropped. +- Error handling (diff fails → skip+warn; build fails → skip+warn; no match → skip; ambiguous overlap → validation error) → A4 (`apply_source_patch` FALSE path, build tryCatch), A5 (`resolve_patch_version` NA path), B2 (overlap validation). ✓ +- Observability (log line per applied patch) → A5 (`log_info` with `describe_patch`). DB metadata recording of applied patches is deferred with S3 (same note). +- Testing (registry parse/match, cache key, integration, failure path) → A1–A5 unit tests, A8 e2e, A4 patch-fail test. ✓ + +**Deferred from spec (call out to user):** S3 upload/reuse of patched binaries and recording applied patches in the Postgres build-metadata row. The local `/mnt/cache/patched-binaries` cache already prevents repeated rebuilds within a container; S3 reuse across CI jobs is a follow-up. If you want it in the first cut, add a Task A5b (upload `cached` tarball to a `…/patched/` S3 slot and check there before building) and a metadata column — say so and I'll insert them. + +**Placeholder scan:** No TBD/TODO; every code step shows complete code. README/registry-README prose steps describe exact content to write (acceptable for docs). + +**Type consistency:** `prepare_patched_repo` signature is identical across A5 (definition) and A6 (call site, via defaults). `patch_cache_key(entry, version, platform, arch, r_minor)` argument order matches between A3 and A5. `run_pak_install_with_mutex(..., patched_repo)` matches between A6 definition and the A6 test. `handle_system_dependencies(..., patches)` matches between A7 edit and A7 test. `build_patched_binary(entry, version, dest_dir)` matches A4 and A5. + +**Open risk:** Task A0 gates the whole approach; if it fails, the documented source-serving contingency keeps every later task valid with a localized change in A4. diff --git a/specs/2026-06-30-package-patching-design.md b/specs/2026-06-30-package-patching-design.md new file mode 100644 index 0000000..1344173 --- /dev/null +++ b/specs/2026-06-30-package-patching-design.md @@ -0,0 +1,157 @@ +# Design: Dynamic per-package patching during binary builds + +Date: 2026-06-30 +Status: Approved (pending spec review) + +## Problem + +Some CRAN packages fail to compile on specific build platforms due to compiler- or OS-specific issues that have nothing to do with the package being built. +The canonical example is `RcppParallel`: its bundled Intel TBB sources fail to build on musl (Alpine) and on newer OS/compiler combinations. +Observed failure on `ubuntu-2604` ("resolute") with `g++ 15.2.0`: + +``` +../build/common.inc:74: *** "" is not supported. Add build/.inc file with os-specific settings . Stop. +make: *** [Makevars:163: tbb] Error 2 +ERROR: compilation failed for package 'RcppParallel' +``` + +Because `RcppParallel` is a dependency of many packages, a single such failure cascades: every dependent package (e.g. `rts2`) also fails, even though nothing is wrong with the dependent itself. + +Today there is no way to intervene. +A package can only be **excluded** (`local/excluded-packages.json`), which is all-or-nothing and does not help dependents. + +## Goal + +Allow a curated set of packages to be "patched" — via lightweight build-time overrides or, when necessary, real source diffs — **before** they are installed, whether the package is a direct build target or a transitive dependency pulled in by `pak`. + +## Key constraint that drives the design + +When `RcppParallel` fails here, it is being installed as a **transitive dependency** by `pak`, inside `bincraft::build_binary_package()`. +`pak` downloads, configures, and compiles it in one subprocess; this repo never touches that source. +For a fix to reach a dependency-of-a-dependency, the fixed package must be visible to `pak` itself, where `pak`'s repositories/sources are configured — which is inside `bincraft`. + +Decisions taken during brainstorming: + +- **Mechanism lives in `bincraft`** (the engine), because only there can transitive deps be influenced. +- **Patch tiers: both, env-overrides first.** Support cheap per-package env vars / configure args / Makevars (version-independent) *and* true source diffs (version-pinned), preferring the lightweight override. +- **Registry data lives in this repo** (`build-cran-binaries`) and is passed into `bincraft`, keeping `bincraft` as pure mechanism and the frequently-changing policy data with operational config. + +## Approaches considered + +| Approach | How pak sees the fix | Verdict | +|---|---|---| +| A. Patched **source** repo — drop patched `.tar.gz` source into a local repo, prepend it | pak recompiles from your source | Simple, but env-tier overrides leak globally (one subprocess builds everything) and the dep recompiles on every dependent build | +| **B. Pre-built patched binary repo (chosen)** | pak installs a ready binary by repo priority | Per-package scoping is free; no recompile; the binary is a cacheable/uploadable artifact that fits the existing system | +| C. pkgdepends per-build hook | intercept each build | No clean per-package pre-compile hook exists; fragile | + +Chosen: **B**. + +## Architecture + +### Registry (this repo) + +``` +local/patches/ + registry.json # the manifest + RcppParallel/ + fix.patch # optional source diff, referenced by an entry +``` + +`registry.json` is an array of entries: + +```json +[ + { + "package": "RcppParallel", + "versions": "*", + "platforms": ["alpine", "ubuntu-2604"], + "env": { "RCPP_PARALLEL_USE_TBB": "0" }, + "configure_args": [], + "makevars": {}, + "patch": null, + "reason": "bundled TBB fails to build on musl / newer compilers" + } +] +``` + +Field semantics: + +- `package` (string, required): CRAN package name. +- `versions` (string, required): `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. + Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. +- `platforms` (array of strings, required): matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). + An entry matches if any listed token matches any build token. + `["*"]` matches all platforms. +- `env` (object, optional): environment variables exported only for this package's isolated build. +- `configure_args` (array, optional): passed as `--configure-args` to the isolated build. +- `makevars` (object, optional): key/value pairs written into a package-local Makevars for the isolated build. +- `patch` (string or null, optional): path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. +- `reason` (string, required): human explanation, surfaced in logs and metadata. + +A fix is any combination of `env`, `configure_args`, `makevars`, and `patch`. +"Env-first" is an authoring guideline (prefer the lightweight override) and an ordering of effort, not a runtime branch — all present fields are applied together for the isolated build. + +### Flow (inside bincraft, around existing pak resolution) + +1. **Resolve** the dependency set (dry-run) to learn the concrete versions `pak` will install. + Reuse bincraft's existing resolution where possible (e.g. a `pkgdepends` proposal: `$resolve()` → inspect resolution → ... → `$solve()` / `$install()` after the local repo is prepended). +2. For each resolved package that matches a registry entry (name + `versions` + `platforms`): obtain a **patched binary** for the exact `version × platform × arch × R-minor`: + - **Cache hit** (local `/mnt/cache/patched-binaries/` or S3): fetch it into the local repo. + - **Cache miss**: download the CRAN **source** for that version, apply the source `patch` (if any) to the unpacked tree, build the binary in isolation with `env` / `configure_args` / `makevars` applied, then place the binary in the local repo and write it to the cache (and S3 if uploading is enabled). +3. **Prepend** the local binary repo (`file://…`) to `pak`'s repo list, and regenerate its `PACKAGES` index. +4. Run the **normal install**. + `pak` resolves the patched binary for the matched package — direct or transitive — because it wins on repo priority for an equal version, and installs it without recompiling. + +### Caching (essential) + +`RcppParallel` is a dependency of dozens of packages; without caching the fix would be rebuilt on every dependent build. +Patched binaries are keyed by: + +``` +_____ +``` + +`patchhash` is a hash of the normalized registry entry plus the referenced diff file contents. +Editing a patch therefore changes the hash and auto-invalidates stale cached binaries. + +- Local cache: `/mnt/cache/patched-binaries/`. +- Optional S3 cache for cross-build reuse: a dedicated `…/patched/` slot under the existing arch/codename structure, mirroring how normal binaries are stored. + +### S3 upload + +Patched binaries **are** uploaded to S3 (in addition to the local cache) so they are reused across CI jobs and machines, not just within one container. +They live in a separate `patched/` slot and are not published into the user-facing `src/contrib` index — they are an internal build accelerator, not a distributed artifact. + +## Error handling + +- **Source diff fails to apply** (CRAN moved past the pinned version): log a clear warning, skip that entry, and proceed. + The package builds unpatched (status quo) and may fail. + The skipped/failed-to-apply patch is surfaced in build metadata. +- **Pre-build of the patched binary fails**: log a warning, skip, proceed. +- **No version or platform match**: skip silently (the entry simply does not apply to this build). +- **Overlapping entries for one package**: the most specific entry wins (a concrete `platforms`/`versions` beats `"*"`). + Genuine ambiguity (two equally specific, conflicting entries) is a validation error reported before the build. + +## Observability + +- One log line per applied patch, e.g.: `Applying patch to RcppParallel 5.1.11-2 [env: RCPP_PARALLEL_USE_TBB=0]: bundled TBB fails on musl / newer compilers` +- The set of applied patches (package, version, `patchhash`) is recorded in the Postgres build-metadata row for the build, so it is queryable later. + +## Testing + +- **Unit (registry):** parsing and matching — version constraints, platform token matching, precedence/specificity, and detection of ambiguous overlaps. +- **Unit (cache key):** `patchhash` changes when the entry or diff changes; is stable otherwise. +- **Integration:** `RcppParallel` on `resolute` (and/or Alpine) fails to build without a registry entry and succeeds with one; a dependent package such as `rts2` succeeds once the dependency is patched. +- **Failure path:** an entry pinned to an old version against a newer CRAN release → graceful skip with a warning, build continues. + +## Out of scope + +- Shipping a default registry inside `bincraft` (registry is repo-local for now; a baseline-in-engine + repo-override model can come later if needed). +- Publishing patched binaries into the public `src/contrib` index. +- Automatic detection of which packages need patches — entries are curated by hand. + +## Split of work + +- **bincraft:** the mechanism — registry ingestion, resolution hook, isolated patched-binary build, caching/upload, local-repo prepend, logging, metadata recording. + A new `patches` argument on `build_binary_package()`. +- **build-cran-binaries (this repo):** the `local/patches/` registry and diffs, passing `patches = "local/patches"` through `build-one.R` / `build-all.R`, and documentation. From 00db47398d0eefb0bab011186bc962dbb944b60b Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 30 Jun 2026 14:07:22 +0200 Subject: [PATCH 047/125] chore: 4.4.0 instead of 4.3.1 --- .crow/archive-missed-packages.yaml | 2 +- .crow/build-all-versions-install-deps.yaml | 2 +- .crow/build-all-versions.yaml | 4 ++-- .crow/process-updates.yaml | 4 ++-- .crow/weekly-rebuild-missing.yaml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 3feb157..22ca365 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,7 +62,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' backend_options: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 3a7d92c..886bfc6 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -75,7 +75,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 076e482..3de6fa1 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -121,7 +121,7 @@ steps: # to a zero-length value and breaks every metadata query and the sysdeps # install). Pin bincraft here, exactly like the R-minor pass below. - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -133,7 +133,7 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages; first arg is the codename (e.g. "alpine324"), diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index b943b2d..24b3fd3 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -192,7 +192,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run @@ -208,7 +208,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 5d88dec..8f7a263 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -129,7 +129,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.3.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.3.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' From b744ae1bb1bf7eeff565d8119849af7e01ed3a13 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 30 Jun 2026 12:15:39 +0000 Subject: [PATCH 048/125] fix: build-one image bincraft v4.4.0 + ship patch registry (#104) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes the **targeted rebuild** path (`just rebuild` → `docker/build-one.Dockerfile`) so it actually applies patches. Two problems, both of which would make `build-one.R`'s `patches = "local/patches"` a silent no-op or a hard error: 1. **Stale bincraft pin.** The Dockerfile pinned bincraft `v4.2.1`, which predates the `patches` argument — so `build_binary_package(patches = ...)` would fail with `unused argument`. Bumped to **v4.4.0** to match the `.crow` workflows. 2. **Registry not shipped into the image.** Only `build-one.R` was copied in; `local/patches/registry.json` was absent, so `patches = "local/patches"` resolved to a nonexistent `/work/local/patches` and silently applied nothing. Added `COPY patches /work/local/patches` (build context is `local/`, CWD is `/work`). ## Why This is the path used to verify the patching end-to-end, e.g.: ```bash just rebuild alpine 3.23 amd64 rts2 1.0.3 ``` rts2 depends on RcppParallel; with this fix the container installs bincraft v4.4.0, ships the registry, and the patched RcppParallel binary (`RCPP_PARALLEL_USE_TBB=0`) is served to pak during dependency install. Expect `Applying patch to RcppParallel …` in the log, RcppParallel installed as a binary (no recompile), then rts2 building and uploading. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/104 --- docker/build-one.Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 2cd4d51..2c1af97 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -16,6 +16,9 @@ ARG CACHEBUST WORKDIR /work COPY build-one.R /work/build-one.R +# Ship the patch registry so build-one.R's `patches = "local/patches"` resolves +# (build context is `local/`, CWD is /work). +COPY patches /work/local/patches RUN --mount=type=secret,id=b2_access,required=true \ --mount=type=secret,id=b2_secret,required=true \ @@ -37,7 +40,7 @@ RUN --mount=type=secret,id=b2_access,required=true \ echo "No working virtual display; building without xvfb" >&2; \ fi; \ run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \ - ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.1")'; }; \ + ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")'; }; \ PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ seen=" $PRIMARY_MINOR "; \ prc=0; failed=""; \ From b93d1ef6bd4bdaa9d3534228c59d5cc8933164ca Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 30 Jun 2026 12:41:32 +0000 Subject: [PATCH 049/125] chore: silence otelsdk warnings across build paths (#105) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Silences the recurring `OpenTelemetry error: there is no package called 'otelsdk'` warnings during builds. ## Root cause The `build-env-*` images configure an OTel exporter (traces/logs/metrics), but `otelsdk` (the R OTel SDK backend) isn't installed. pak's `otel` instrumentation therefore tries to load `otelsdk` on every run and logs the error, falling back to a no-op. `OTEL_SDK_DISABLED=true` (already set in `build-one.Dockerfile`) does **not** help — the R `otel` package ignores it and gates purely on `OTEL_R__EXPORTER` (then the standard `OTEL__EXPORTER`). When that resolves to a real exporter (`otlp`/`http`/…) with no SDK present, you get the error. ## Fix Set `OTEL_R_TRACES_EXPORTER`, `OTEL_R_LOGS_EXPORTER`, and `OTEL_R_METRICS_EXPORTER` to `none` so the R otel providers are clean no-ops: - `docker/build-one.Dockerfile` — exported alongside the existing OTel var. - `.crow/process-updates.yaml`, `weekly-rebuild-missing.yaml`, `build-all-versions.yaml`, `build-all-versions-install-deps.yaml` — added to each step's `environment` block. Using the R-specific variables (not the standard `OTEL_*_EXPORTER`) keeps OTel intact for any non-R tooling in the images. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/105 --- .crow/build-all-versions-install-deps.yaml | 3 +++ .crow/build-all-versions.yaml | 6 ++++++ .crow/process-updates.yaml | 6 ++++++ .crow/weekly-rebuild-missing.yaml | 3 +++ docker/build-one.Dockerfile | 3 +++ 5 files changed, 21 insertions(+) diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 886bfc6..68d9569 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -53,6 +53,9 @@ steps: image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN GITHUB_PAT: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 3de6fa1..9364b4e 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -90,6 +90,9 @@ steps: image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW B2_S3_ACCESS_KEY: @@ -152,6 +155,9 @@ steps: image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none B2_S3_ACCESS_KEY: from_secret: B2_S3_ACCESS_KEY B2_S3_SECRET_KEY: diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 24b3fd3..5e4eccc 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -158,6 +158,9 @@ steps: image: reg.devxy.io/rpkgs/build-env-${IMG} pull: true environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW B2_S3_ACCESS_KEY: @@ -230,6 +233,9 @@ steps: - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY SUBDOMAIN1: 'cran.devxy.io' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 8f7a263..8877c45 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -103,6 +103,9 @@ steps: image: reg.devxy.io/rpkgs/build-env-${IMG} pull: true environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW B2_S3_ACCESS_KEY: diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 2c1af97..a99db07 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -30,6 +30,9 @@ RUN --mount=type=secret,id=b2_access,required=true \ export GITHUB_PAT="$(cat /run/secrets/github_pat 2>/dev/null || true)" && \ export GIT_TERMINAL_PROMPT=0 && \ export OTEL_SDK_DISABLED=true && \ + export OTEL_R_TRACES_EXPORTER=none && \ + export OTEL_R_LOGS_EXPORTER=none && \ + export OTEL_R_METRICS_EXPORTER=none && \ XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run || true); \ XVFB_ARGS=""; \ if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi; \ From e59590e2d69163a128bbcfe7fcb0cf32fbbf2882 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 30 Jun 2026 13:27:01 +0000 Subject: [PATCH 050/125] fix: RcppParallel disable-TBB source patch (env var was a no-op) (#106) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes the RcppParallel patch, which was a **no-op** and left the build hanging. The previous registry entry set `env: { RCPP_PARALLEL_USE_TBB: "0" }`. But `RCPP_PARALLEL_USE_TBB` is a **compile-time `-D` flag** in RcppParallel's Makevars — it is never read from the environment. So the override did nothing: `USE_TBB=Linux` (hardcoded from `uname`) still triggered the **bundled Intel TBB build**, which hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604). The `Applying patch …` log only meant the env was set, not that it had any effect. ## Fix Replace the env entry with a **source patch** (`local/patches/RcppParallel/disable-tbb.patch`) on `src/Makevars.in` that, on Linux: - leaves `USE_TBB` unset → the whole bundled-TBB build/link path is skipped (no hang), and - forces `PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=0` → the sources compile the **TinyThread** backend (needed because `RcppParallel.h` otherwise auto-defaults TBB on for glibc Linux). ## Verification In a Linux container, applying the patch and running `R CMD INSTALL RcppParallel`: ``` bundled_TBB_build=0 # bundled TBB build never runs * DONE (RcppParallel) # installs via TinyThread ``` ## Note bincraft's `apply_source_patch` shells out to `patch`. If a build-env image lacks the `patch` tool (common on Alpine), the patch will report "did not apply cleanly" and fall back to an unpatched (hanging) build. If that happens, the follow-up is to switch bincraft's patch application to `git apply` (git is always present) — happy to do that if needed. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/106 --- local/patches/RcppParallel/disable-tbb.patch | 16 ++++++++++++++++ local/patches/registry.json | 6 +++--- 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 local/patches/RcppParallel/disable-tbb.patch diff --git a/local/patches/RcppParallel/disable-tbb.patch b/local/patches/RcppParallel/disable-tbb.patch new file mode 100644 index 0000000..0fe8ad9 --- /dev/null +++ b/local/patches/RcppParallel/disable-tbb.patch @@ -0,0 +1,16 @@ +diff --git a/src/Makevars.in b/src/Makevars.in +index be8445f..faee771 100644 +--- a/src/Makevars.in ++++ b/src/Makevars.in +@@ -60,7 +60,10 @@ else + endif + + ifeq ($(UNAME), Linux) +- USE_TBB=Linux ++ # bincraft patch: the bundled Intel TBB build hangs/fails on musl (Alpine) ++ # and newer toolchains (g++ 15). Skip it (leave USE_TBB unset) and force the ++ # TinyThread backend so RcppParallel still builds. ++ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=0 + endif + + ifeq ($(UNAME), SunOS) diff --git a/local/patches/registry.json b/local/patches/registry.json index 80460d6..cdeef99 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -3,10 +3,10 @@ "package": "RcppParallel", "versions": "*", "platforms": ["alpine", "ubuntu-2604"], - "env": { "RCPP_PARALLEL_USE_TBB": "0" }, + "env": {}, "configure_args": [], "makevars": {}, - "patch": null, - "reason": "bundled Intel TBB fails to build on musl and on newer toolchains (e.g. g++ 15 on ubuntu-2604); disabling TBB falls back to TinyThread" + "patch": "RcppParallel/disable-tbb.patch", + "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); patch unsets USE_TBB and forces -DRCPP_PARALLEL_USE_TBB=0 so RcppParallel skips the bundled build and uses the TinyThread backend" } ] From d496029a79d0e0621ed46008023b8eedd252e1a1 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 30 Jun 2026 15:51:02 +0200 Subject: [PATCH 051/125] chore: bump bincraft to 4.4.1 --- .crow/archive-missed-packages.yaml | 2 +- .crow/build-all-versions-install-deps.yaml | 2 +- .crow/build-all-versions.yaml | 4 ++-- .crow/process-updates.yaml | 4 ++-- .crow/weekly-rebuild-missing.yaml | 2 +- docker/build-one.Dockerfile | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 22ca365..eef7ea0 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,7 +62,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' backend_options: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 68d9569..1e9cb32 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -78,7 +78,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 9364b4e..ac4e0ae 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -124,7 +124,7 @@ steps: # to a zero-length value and breaks every metadata query and the sysdeps # install). Pin bincraft here, exactly like the R-minor pass below. - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -136,7 +136,7 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages; first arg is the codename (e.g. "alpine324"), diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 5e4eccc..baaa34c 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -195,7 +195,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run @@ -211,7 +211,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 8877c45..0edf55a 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -132,7 +132,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index a99db07..7f93ae8 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -43,7 +43,7 @@ RUN --mount=type=secret,id=b2_access,required=true \ echo "No working virtual display; building without xvfb" >&2; \ fi; \ run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \ - ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.0")'; }; \ + ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")'; }; \ PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ seen=" $PRIMARY_MINOR "; \ prc=0; failed=""; \ From b4ec6291a9b8a3b3963fe4c8bc22949aa03f161c Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 30 Jun 2026 16:09:44 +0200 Subject: [PATCH 052/125] chore: bump bincraft to 4.4.2 --- .crow/archive-missed-packages.yaml | 2 +- .crow/build-all-versions-install-deps.yaml | 2 +- .crow/build-all-versions.yaml | 4 ++-- .crow/process-updates.yaml | 4 ++-- .crow/weekly-rebuild-missing.yaml | 2 +- README.md | 2 +- docker/build-one.Dockerfile | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index eef7ea0..ca21aa5 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,7 +62,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2", dependencies = TRUE)' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' backend_options: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 1e9cb32..050db51 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -78,7 +78,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index ac4e0ae..f1e5f00 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -124,7 +124,7 @@ steps: # to a zero-length value and breaks every metadata query and the sysdeps # install). Pin bincraft here, exactly like the R-minor pass below. - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -136,7 +136,7 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages; first arg is the codename (e.g. "alpine324"), diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index baaa34c..9ffcaed 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -195,7 +195,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run @@ -211,7 +211,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 0edf55a..9996182 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -132,7 +132,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")' + - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/README.md b/README.md index d030948..28541a1 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ Tag does not have a NAMESPACE file and hence cannot be built. Dependency not available: Either because the dependency was not declared or errored itself during installation. ```text - In function '\033[01m\033[KRcpp::List solveRRBLUP(const mat&, const mat&, const mat&)\033[m\033[K':\n\033[01m\033[KMME.cpp:162:61:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope\n 162 | double ll = -0.5*(double(optRes[\"objective\"])+df+df*log(2*\033[01;31m\033[KPI\033[m\033[K/df));\n | \033[01;31m\033[K^~\033[m\033[K\n\033[01m\033[KMME.cpp:\033[m\033[K In function '\033[01m\033[KRcpp::List solveRRBLUPMV(const mat&, const mat&, const mat&, int, double)\033[m\033[K':\n\033[01m\033[KMME.cpp:277:31:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope; did you mean '\033[01m\033[KHI\033[m\033[K'?\n 277 | ll -= double(n*m)/2.0*log(2*\033[01;31m\033[KPI\033[m\033[K);\n | \033[01;31m\033[K^~\033[m\033[K\n | \033[32m\033[KHI\033[m\033[K\nmake: *** [/opt/R/4.4.1/lib/R/etc/Makeconf:204: MME.o] Error 1\nERROR: compilation failed for package 'AlphaSimR'\n* removing '/tmp/RtmpclI5CE/temp_libpath11135d215d5/AlphaSimR'\n + In function '\033[01m\033[KRcpp::List solveRRBLUP(const mat&, const mat&, const mat&)\033[m\033[K':\n\033[01m\033[KMME.cpp:162:61:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope\n 162 | double ll = -0.5*(double(optRes[\"objective\"])+df+df*log(2*\033[01;31m\033[KPI\033[m\033[K/df));\n | \033[01;31m\033[K^~\033[m\033[K\n\033[01m\033[KMME.cpp:\033[m\033[K In function '\033[01m\033[KRcpp::List solveRRBLUPMV(const mat&, const mat&, const mat&, int, double)\033[m\033[K':\n\033[01m\033[KMME.cpp:277:31:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope; did you mean '\033[01m\033[KHI\033[m\033[K'?\n 277 | ll -= double(n*m)/2.0*log(2*\033[01;31m\033[KPI\033[m\033[K);\n | \033[01;31m\033[K^~\033[m\033[K\n | \033[32m\033[KHI\033[m\033[K\nmake: *** [/opt/R/4.4.2/lib/R/etc/Makeconf:204: MME.o] Error 1\nERROR: compilation failed for package 'AlphaSimR'\n* removing '/tmp/RtmpclI5CE/temp_libpath11135d215d5/AlphaSimR'\n ``` Compiler error: Possible reasons: too old CXX code which cannot be compiled anymore with CXX14 or CXX17. diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 7f93ae8..1eb5ff7 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -43,7 +43,7 @@ RUN --mount=type=secret,id=b2_access,required=true \ echo "No working virtual display; building without xvfb" >&2; \ fi; \ run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \ - ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.1") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.1")'; }; \ + ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")'; }; \ PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ seen=" $PRIMARY_MINOR "; \ prc=0; failed=""; \ From 2cf5714ad919928b5b2193f6e41fc3f77c618038 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Wed, 1 Jul 2026 00:32:49 +0000 Subject: [PATCH 053/125] chore(deps): update pre-commit hook rbubley/mirrors-prettier to v3.9.4 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7448d6c..0f62ef9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.1 + rev: v3.9.4 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit From 1ad7a6bfe9ae8798f96cbbae23a8a9718f9872a1 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 1 Jul 2026 08:10:02 +0000 Subject: [PATCH 054/125] chore: resolve latest bincraft release dynamically (no hardcoded pins) (#107) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Stop hardcoding the bincraft version. Every `.crow` workflow and the build-one image pinned `@vX.Y.Z` (and a `packageVersion() != "X.Y.Z"` guard), so each bincraft release meant editing the version in ~8 places — and it was easy to miss one (the Dockerfile lagged at v4.2.1; v4.4.1 shipped without the empty-env fix because of exactly this churn). ## Change New `local/install-bincraft.R` resolves the **latest release tag dynamically**: - `git ls-remote --tags` on the public repo (no token), - keep `vX.Y.Z` tags, pick the highest version (filtered/sorted in R for portability, not via git `--sort`/refspec which behaved inconsistently under `system2()`), - `pak::pak("git::…@")` — idempotent on the git ref, so re-runs keep the package unless a newer tag exists. All call sites now invoke the helper instead of a pinned version: - `.crow/build-all-versions.yaml` (primary + per-minor pass) - `.crow/build-all-versions-install-deps.yaml` - `.crow/process-updates.yaml` (primary + per-minor pass) - `.crow/weekly-rebuild-missing.yaml` - `.crow/archive-missed-packages.yaml` - `docker/build-one.Dockerfile` (ships the helper into the image; `ensure_bincraft` sources it) ## Effect Tag a new bincraft release → the next CI run / `just rebuild` picks it up automatically. No more pin edits, and no more "forgot to bump the Dockerfile" drift. ## Verified - Resolver returns the current latest tag (`v4.4.2`) via `git ls-remote` + R-side version sort. - All five workflow YAMLs parse; helper R parses; air/editorconfig clean. Note: this tracks the latest **tag**, so cutting a release is still the deliberate gate — CI won't pick up un-tagged main. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/107 --- .crow/archive-missed-packages.yaml | 2 +- .crow/build-all-versions-install-deps.yaml | 2 +- .crow/build-all-versions.yaml | 4 +- .crow/process-updates.yaml | 4 +- .crow/weekly-rebuild-missing.yaml | 2 +- docker/build-one.Dockerfile | 4 +- local/install-bincraft.R | 52 ++++++++++++++++++++++ 7 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 local/install-bincraft.R diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index ca21aa5..71f808b 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,7 +62,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' backend_options: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 050db51..8649ca0 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -78,7 +78,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index f1e5f00..615d273 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -124,7 +124,7 @@ steps: # to a zero-length value and breaks every metadata query and the sysdeps # install). Pin bincraft here, exactly like the R-minor pass below. - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -136,7 +136,7 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages; first arg is the codename (e.g. "alpine324"), diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 9ffcaed..3da41fd 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -195,7 +195,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run @@ -211,7 +211,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 9996182..934c518 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -132,7 +132,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 1eb5ff7..5bc107a 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -16,6 +16,8 @@ ARG CACHEBUST WORKDIR /work COPY build-one.R /work/build-one.R +# Resolve and install the latest bincraft release dynamically (no hardcoded pin). +COPY install-bincraft.R /work/install-bincraft.R # Ship the patch registry so build-one.R's `patches = "local/patches"` resolves # (build context is `local/`, CWD is /work). COPY patches /work/local/patches @@ -43,7 +45,7 @@ RUN --mount=type=secret,id=b2_access,required=true \ echo "No working virtual display; building without xvfb" >&2; \ fi; \ run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \ - ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")'; }; \ + ensure_bincraft() { "$1" -q -e 'source("/work/install-bincraft.R")'; }; \ PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ seen=" $PRIMARY_MINOR "; \ prc=0; failed=""; \ diff --git a/local/install-bincraft.R b/local/install-bincraft.R new file mode 100644 index 0000000..4665b3b --- /dev/null +++ b/local/install-bincraft.R @@ -0,0 +1,52 @@ +#!/usr/bin/env Rscript + +# Install the latest tagged bincraft release, resolved dynamically, so the CI +# workflows and the build-one image never pin a hardcoded version (no more +# editing `@vX.Y.Z` in many places on every release). +# +# Run with the R whose library should receive bincraft: +# Rscript local/install-bincraft.R +# or, to target a specific R from a shell loop: +# "$RBIN" -q -e 'source("local/install-bincraft.R")' +# +# How it works: list the remote tags with `git ls-remote` (no token needed for +# the public repo), keep the `vX.Y.Z` release tags, pick the highest version, +# and install it with pak. pak is idempotent on the git ref, so re-running keeps +# the package when it is already current and only updates when a newer tag ships. +# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is +# identical across git versions and `system2()` argument handling. + +repo_url <- Sys.getenv( + "BINCRAFT_GIT_URL", + unset = "https://codefloe.com/rpkgs/bincraft.git" +) + +# GIT_TERMINAL_PROMPT=0 keeps a non-interactive run from hanging on auth. +refs <- system2( + "git", + c("ls-remote", "--tags", repo_url), + stdout = TRUE, + stderr = FALSE, + env = "GIT_TERMINAL_PROMPT=0" +) +tags <- sub(".*refs/tags/", "", refs) +tags <- tags[!grepl("\\^\\{\\}$", tags)] # drop dereferenced "...^{}" lines +tags <- grep("^v[0-9]", tags, value = TRUE) # only vX.Y.Z release tags +if (length(tags) == 0L) { + stop( + "Could not resolve any bincraft release tag from ", + repo_url, + call. = FALSE + ) +} +latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][ + 1L +] + +message(sprintf("Installing latest bincraft release: %s", latest)) +pak::pak(sprintf("git::%s@%s", repo_url, latest)) +message(sprintf( + "bincraft %s installed (%s)", + as.character(utils::packageVersion("bincraft")), + latest +)) From 5f10863a9c52d2997d8511494276e9f260a4944f Mon Sep 17 00:00:00 2001 From: automation-bot Date: Thu, 2 Jul 2026 00:32:11 +0000 Subject: [PATCH 055/125] chore(deps): update pre-commit hook davidanson/markdownlint-cli2 to v0.23.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f62ef9..a0f4422 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.22.1 + rev: v0.23.0 hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier From 1e0657608492670cc039144bd123cf4e33b2b8ba Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 2 Jul 2026 08:43:20 +0200 Subject: [PATCH 056/125] docs: add CLAUDE.md with B2 storage gotchas and build pipeline conventions for agents --- CLAUDE.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..faae5f1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,14 @@ +# CLAUDE.md + +Crow CI pipelines (`.crow/`) and local tooling (`local/`) that build CRAN binary packages (incl. Alpine/musl) and upload them to Backblaze B2. + +## Conventions + +- **PRs:** the remote is Forgejo on `codefloe.com`; use `fj -H codefloe.com` (not `gh`). +- **Storage:** Backblaze B2 bucket `devxy-r-builds` (endpoints configured in the `.crow/` pipelines). + +## Gotchas + +- **B2 requires authentication for the list-bucket API.** Anonymous GET only works for individual public-read objects — an empty listing means missing credentials, not an empty bucket. +- Weekly-rebuild pipelines run for hours; watch them as background tasks and fetch Crow logs yourself instead of having the user paste progress. +- musl builds of packages with bundled native deps (e.g. RcppParallel/TBB) recur as failures; check for an existing patch before re-deriving a fix. From 36d3bc8604d179b3baabcf396157e9304862ae06 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 7 Jul 2026 18:12:36 +0000 Subject: [PATCH 057/125] fix(patches): force fs to build vendored static libuv (#111) ## Problem The `fs` 2.1.0 binary links **system libuv** (`readelf -d fs.so` shows `NEEDED libuv.so.1`). fs's `configure` prefers system libuv whenever `pkg-config` resolves it, and our build images ship `libuv-devel` (installed as a pak build-time system requirement), so the resulting binary is dynamically linked against `libuv.so.1`. That binary fails to load on any consumer machine without runtime libuv: ```text unable to load shared object '.../fs/libs/fs.so': libuv.so.1: cannot open shared object file: No such file or directory ``` `install.packages()`/renv do **not** install `SystemRequirements` (only `pak` does, and only inside the build container), so most consumers hit this. Older fs 1.6.x always vendored libuv, so only the 2.x binaries regressed. Reproduced in a clean `reg.devxy.io/r/r-alma:4.5-9`. ## Fix Add `local/patches/fs/force-vendored-libuv.patch`, registered for all platforms. It short-circuits `configure` to `cp -f src/Makevars.vendor src/Makevars; exit 0` before the pkg-config detection, forcing the bundled static libuv build (`tools/libuv-v1.52.0.tar.gz`, built via cmake). An env/pkg-config override (`PKG_CONFIG_LIBDIR`) was tried first but the rebuilt binary still linked `libuv.so.1` (the registry `env` tier does not reach fs's configure step), so a source patch is used instead. ## Verification Built end-to-end inside the real `build-env-redhat:9` image (system libuv present): - patch fires (`Building static libuv (bincraft: forced vendored)`), - cmake compiles the vendored libuv, - resulting `fs.so` has **no `libuv.so.1`** in `NEEDED` (only libR, libstdc++, libm, libgcc_s, libc). `Rscript local/validate-patches.R` passes (2 entries). cmake confirmed present in the build-env images. ## Follow-up (not in this PR) - Rebuild `fs 2.1.0` on every affected platform (rhel8/9/10, ubuntu jammy/noble, alpine 3.22/3.23; amd64 + arm64) and purge the CDN binary paths. - CDN delivery gap: `purge_cdn_cache.sh` only purges `PACKAGES*`, never package binaries, so rebuilt binaries stay masked until their `.tar.gz` path is purged. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/111 --- local/patches/fs/force-vendored-libuv.patch | 19 +++++++++++++++++++ local/patches/registry.json | 10 ++++++++++ 2 files changed, 29 insertions(+) create mode 100644 local/patches/fs/force-vendored-libuv.patch diff --git a/local/patches/fs/force-vendored-libuv.patch b/local/patches/fs/force-vendored-libuv.patch new file mode 100644 index 0000000..c69e8b9 --- /dev/null +++ b/local/patches/fs/force-vendored-libuv.patch @@ -0,0 +1,19 @@ +diff --git a/configure b/configure +--- a/configure ++++ b/configure +@@ -11,6 +11,15 @@ + PKG_TEST_HEADER="" + PKG_LIBS="-luv" + ++# bincraft patch: force the vendored static libuv so the resulting binary ++# is self-contained. fs configure otherwise links system libuv whenever ++# pkg-config finds libuv-devel (installed as a build-time sysreq), yielding ++# an fs.so with NEEDED libuv.so.1 that fails to dyn.load on machines lacking ++# runtime libuv (install.packages/renv do not install SystemRequirements). ++echo "Building static libuv (bincraft: forced vendored)" 1>&2 ++cp -f src/Makevars.vendor src/Makevars ++exit 0 ++ + # Use pkg-config if available + if [ `command -v pkg-config` ]; then + PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` diff --git a/local/patches/registry.json b/local/patches/registry.json index cdeef99..7a197d7 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -8,5 +8,15 @@ "makevars": {}, "patch": "RcppParallel/disable-tbb.patch", "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); patch unsets USE_TBB and forces -DRCPP_PARALLEL_USE_TBB=0 so RcppParallel skips the bundled build and uses the TinyThread backend" + }, + { + "package": "fs", + "versions": "*", + "platforms": ["*"], + "env": {}, + "configure_args": [], + "makevars": {}, + "patch": "fs/force-vendored-libuv.patch", + "reason": "fs 2.x configure links system libuv whenever pkg-config finds libuv-devel (installed as a build-time sysreq), producing an fs.so with NEEDED libuv.so.1. That binary fails to dyn.load on consumer machines lacking runtime libuv, because install.packages/renv do not install SystemRequirements (only pak does, and only in the build container). The patch short-circuits configure to copy src/Makevars.vendor and build the bundled static libuv (needs cmake) so the binary is self-contained on every platform. An env/pkg-config override was tried first but the rebuilt binary still linked libuv.so.1, so a source patch is used instead." } ] From 09536cddd63cb7f390927dab873799d3736b24c8 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 8 Jul 2026 08:22:03 +0000 Subject: [PATCH 058/125] feat(build): raise BuildKit cache-mount budget for remote builders (#112) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BuildKit's default GC caps the ephemeral cache tier — `RUN --mount=type=cache` mounts, local build context, git checkouts — at a hardcoded **512 MB** (shown as `488.3 MiB` in `buildx inspect`). Across our 7-distro build matrix that fills instantly and forces re-downloads of system + R packages on every rebuild. ## Changes - **`docker/buildkitd.toml`** (new) — GC config passed via `--config` when creating the `docker-container` builders `artemis` (amd64) and `gaia` (arm64): - cache-mount tier: **512 MB → 8 GB**, retained 7 days - total cache bounded at **40 GB** with **20 GB min-free** - **`justfile`** — document the `--config docker/buildkitd.toml` flag on the builder-create commands so a recreate does not silently revert to the 512 MB default. ## Notes - Limits are absolute (not `%`) because the two hosts differ ~6× in free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB). The 20 GB min-free is the safety valve on the disk-tight Hetzner host; the old default wanted 64 GiB free, which does not exist there. - Both live builders were already recreated with this config and verified running. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/112 --- docker/buildkitd.toml | 41 +++++++++++++++++++++++++++++++++++++++++ justfile | 6 ++++-- 2 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 docker/buildkitd.toml diff --git a/docker/buildkitd.toml b/docker/buildkitd.toml new file mode 100644 index 0000000..ebbeb38 --- /dev/null +++ b/docker/buildkitd.toml @@ -0,0 +1,41 @@ +# BuildKit GC config for the remote buildx builders (artemis/amd64, gaia/arm64). +# +# Apply when creating the docker-container builders: +# docker buildx create --name artemis --driver docker-container \ +# --config docker/buildkitd.toml ssh:// +# docker buildx create --name gaia --driver docker-container \ +# --config docker/buildkitd.toml ssh:// +# +# Why: BuildKit's default GC caps the ephemeral cache tier — RUN +# --mount=type=cache mounts, local build context, git checkouts — at a +# hardcoded 512 MB (shown as "488.3 MiB" in `buildx inspect`). Across our +# 7-distro build matrix that fills instantly and forces re-downloads of +# system + R packages every rebuild. The first rule below raises that tier. +# +# Limits are absolute (not %) on purpose: artemis and gaia have very +# different free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB), so a +# percentage would mean wildly different real budgets. minFreeSpace = 20 GB +# keeps the tight Hetzner host safe while staying modest on the Mac mini. + +[worker.oci] + gc = true + + # Tier 1 — ephemeral caches (cache mounts, local context, git checkouts). + # Raised from the 512 MB default to 8 GB, retained for 7 days so weekly + # rebuilds reuse downloaded packages instead of re-fetching them. + [[worker.oci.gcpolicy]] + filters = [ + "type==source.local", + "type==exec.cachemount", + "type==source.git.checkout", + ] + keepDuration = "168h" + maxUsedSpace = "8GB" + + # Tier 2 — everything else (image layers, RUN exec results). Bounds the + # whole buildkit cache and always leaves 20 GB free on the host disk. + [[worker.oci.gcpolicy]] + all = true + reservedSpace = "2GB" + maxUsedSpace = "40GB" + minFreeSpace = "20GB" diff --git a/justfile b/justfile index 2d9388d..a9a705e 100644 --- a/justfile +++ b/justfile @@ -10,8 +10,10 @@ # - buildx builders named `artemis` (amd64) and `gaia` (arm64), created with the # docker-container driver (runs BuildKit on the remote host's docker daemon over # SSH). The default `remote` driver does NOT work with an ssh:// docker host. -# docker buildx create --name artemis --driver docker-container ssh:// -# docker buildx create --name gaia --driver docker-container ssh:// +# Pass --config docker/buildkitd.toml so BuildKit's GC keeps a usable cache +# (the default caps the cache-mount tier at 512 MB, forcing re-downloads). +# docker buildx create --name artemis --driver docker-container --config docker/buildkitd.toml ssh:// +# docker buildx create --name gaia --driver docker-container --config docker/buildkitd.toml ssh:// # - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional) # # Overridable (env or `just VAR=… rebuild …`): From ffc2319558a0577539a860ba9e530071ca3ed5ef Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 13 Jul 2026 09:28:45 +0000 Subject: [PATCH 059/125] fix(build-all): exclude previously-errored versions in prefilter (#113) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Build jobs were cycling through hundreds of packages that were only ever printed as `Skipping … due to previous build error recorded in metadata DB`, wasting wall-clock on per-package preparation before dropping each one. ## Cause The prefilter query in `local/build-all.R` selected only successfully-built versions (`error_occurred = FALSE`) into `built`, so line 112 removed only those from the chunk. Every previously-errored version stayed in the work list and was walked one-by-one, each hitting the internal skip in `build_binary_package()`. This also explains the misleading `Skipped 0 already-built package versions` line for alphabetical chunks whose leading packages only have error records. ## Changes - `local/build-all.R`: drop the `AND error_occurred = FALSE` clause so `built` holds every version already attempted (built or errored) for this platform/arch; the existing filter then removes all of them up front. - Rename the log line to `already-attempted` so the reported count reflects successes and errors. - Update the surrounding comment to explain why errored versions are excluded. ## Behaviour change Previously-errored versions are now dropped before the build loop instead of being iterated and individually skipped. No package that would otherwise build is affected, `build_binary_package()` already skipped these internally. Retrying errored versions is out of scope and would need a separate opt-in flag on both the prefilter and the in-loop skip. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/113 --- local/build-all.R | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/local/build-all.R b/local/build-all.R index c0fe7f7..7983be3 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -64,10 +64,13 @@ sprintf("# of package versions for this job: %s", nrow(chunk)) exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] chunk <- chunk[!chunk$Package %in% exclude, ] -# Skip package versions already built in a previous run. +# Skip package versions already attempted in a previous run (built or errored). # pkgs_to_build.rds is a static snapshot from the install-deps step, so on a # restart it still lists everything an interrupted run already produced. The # metadata DB reflects that progress, so we re-derive the remaining set here. +# We exclude *all* attempted versions, not just successful ones: a previously +# errored version is skipped by build_binary_package() anyway, so leaving it in +# the chunk only makes the job cycle through it one-by-one for no benefit. # Derive platform + arch from the running container, mirroring the codename -> # platform mapping bincraft uses internally. The OS/OS_VERSION selectors are # workflow-level CI variables that are not injected into the container @@ -104,13 +107,13 @@ con <- DBI::dbConnect( ) built <- DBI::dbGetQuery( con, - "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2 AND error_occurred = FALSE", + "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2", params = list(platform, arch) ) DBI::dbDisconnect(con) before <- nrow(chunk) chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] -sprintf("Skipped %d already-built package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) +sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, From a4b274f28148d7d257007c6abd1d51e6cbd69227 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 13 Jul 2026 09:31:29 +0000 Subject: [PATCH 060/125] fix(ci): bound pkgcache _metadata growth to stop macmini disk-fill (#110) ## Problem The arm64 `build-all` pipeline fills the macmini (gaia) host disk despite an 8h prune. Root cause is not images or job volumes: it is the persistent dep-cache volume, specifically `pkgcache/R/pkgcache/_metadata`, which grew to ~165 GB. `{pkgcache}` mints a new content hash for the "patched" binaries repo on every PACKAGES change, so each per-package build writes a fresh ~70 MB `pkgs-.rds` (+ `patched-/`) that is never evicted (2407 snapshots observed). When the disk hits 100% OrbStack stops and the on-host prune can no longer connect to the daemon, so it never self-heals. ## Change (Workstream A of the disk-fill fix) - Add `trim_pkgcache_metadata()` to `local/r-minor-helpers.R`: keeps the newest `keep` (default 20) `patched-*`/`pkgs-*.rds` entries under `_metadata`, deleting only entries older than `min_age_secs` (default 600s) so it never races the up-to-4 concurrent split-jobs sharing the volume. Preserves `pkg/` downloads and the stable CRAN/BioC/INLA repo dirs. No-op when `R_PKG_CACHE_DIR` is empty (amd64) or `_metadata` is absent (first run). - Call it every 25 packages inside the build loop in `local/build-all.R`. - Add a defensive start-of-run cleanup of `_metadata/patched-*` + `pkgs-*.rds` to the two workflows that mount the persistent volume (`build-all-versions.yaml`, `build-all-versions-install-deps.yaml`). Only these paths are touched; `process-updates.yaml`/`weekly-rebuild-missing.yaml` (no persistent volume) are unchanged. Follow-ups (separate workstreams): on-host self-healing prune watcher + OrbStack disk cap (ansible), and Prometheus/Grafana alerting (k8s-talos). Upstream: bincraft patched-repo hash churn is the true source fix. New unit tests (6) for the helper; full suite 24/24 green. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/110 --- .crow/build-all-versions-install-deps.yaml | 8 ++- .crow/build-all-versions.yaml | 9 ++- local/build-all.R | 10 ++++ local/r-minor-helpers.R | 39 +++++++++++++ local/tests/test-trim-pkgcache.R | 68 ++++++++++++++++++++++ 5 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 local/tests/test-trim-pkgcache.R diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 8649ca0..e0cca86 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -4,7 +4,7 @@ # they are merged with build-all-versions' identical declarations. variables: target_arch: - description: "Architecture to build." + description: 'Architecture to build.' options: - amd64 - arm64 @@ -27,7 +27,7 @@ variables: - "noble" default: "3.24" R_VERSION: - description: "Primary R version under /opt/R." + description: 'Primary R version under /opt/R.' options: - 4.5.3 - 4.4.3 @@ -74,6 +74,10 @@ steps: commands: # one-time full wipe to fix corrupted .so files from previous failed builds # - rm -rf /mnt/cache/R-pkgs + # Clear churny pkgcache metadata left by a prior crashed run (the "patched" + # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). + # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. + - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 615d273..3932811 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -5,7 +5,7 @@ # Skip list lives in local/excluded-packages.json (read by local/build-all.R). variables: target_arch: - description: "Architecture to build." + description: 'Architecture to build.' options: - amd64 - arm64 @@ -31,7 +31,7 @@ variables: - "resolute" default: "3.24" R_VERSION: - description: "Primary R version under /opt/R." + description: 'Primary R version under /opt/R.' options: - 4.5.3 - 4.4.3 @@ -116,6 +116,11 @@ steps: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + # Clear churny pkgcache metadata left by a prior crashed run (the "patched" + # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). + # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. Within-run + # growth is bounded separately by trim_pkgcache_metadata() in build-all.R. + - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # The primary pass must not rely on build-all-versions-install-deps having # run on *this* agent: depends_on only orders the steps, but the cache diff --git a/local/build-all.R b/local/build-all.R index 7983be3..1df1f83 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -122,6 +122,10 @@ s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") sprintf("S3 cache: %s files", length(s3_cache)) n <- nrow(chunk) +# Every `trim_every` packages, bound the pkgcache _metadata dir so a full-platform +# run does not accumulate thousands of ~70 MB snapshots and fill the host disk. +# No-op on amd64 (R_PKG_CACHE_DIR is empty / cache not persisted). +trim_every <- 25L mapply( function(pkg, ver, sens, i) { cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) @@ -147,6 +151,12 @@ mapply( upload = TRUE, store_build_metadata = TRUE ) + if (i %% trim_every == 0L) { + removed <- trim_pkgcache_metadata() + if (removed > 0L) { + cat(sprintf(" [pkgcache trim] removed %d stale _metadata entries\n", removed)) + } + } }, chunk$Package, chunk$Version, diff --git a/local/r-minor-helpers.R b/local/r-minor-helpers.R index 65aafdf..6f01c29 100644 --- a/local/r-minor-helpers.R +++ b/local/r-minor-helpers.R @@ -31,3 +31,42 @@ parse_build_args <- function(args) { ncpus = as.integer(pos[3L]) ) } + +# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the +# "patched" repo mints a new content hash on every PACKAGES change, so each build +# writes a fresh ~70 MB _metadata/pkgs-.rds (+ patched-/) that is +# never reused. Keep the `keep` newest entries by mtime; only remove entries +# older than `min_age_secs`, so a concurrent split-job's in-flight files are +# never deleted (each build uses a unique hash, so aged entries are +# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are +# not matched and thus preserved. Returns the number of entries removed. +trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"), + keep = 20L, + min_age_secs = 600) { + meta <- file.path(cache_dir, "R", "pkgcache", "_metadata") + if (!nzchar(cache_dir) || !dir.exists(meta)) { + return(0L) + } + entries <- c( + Sys.glob(file.path(meta, "patched-*")), + Sys.glob(file.path(meta, "pkgs-*.rds")) + ) + if (length(entries) == 0L) { + return(0L) + } + info <- file.info(entries) + order_new_first <- order(info$mtime, decreasing = TRUE) + ranked <- entries[order_new_first] + ranked_mtime <- info$mtime[order_new_first] + if (length(ranked) <= keep) { + return(0L) + } + candidates <- ranked[(keep + 1L):length(ranked)] + candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)]) + removable <- candidates[candidate_age >= min_age_secs] + if (length(removable) == 0L) { + return(0L) + } + unlink(removable, recursive = TRUE, force = TRUE) + length(removable) +} diff --git a/local/tests/test-trim-pkgcache.R b/local/tests/test-trim-pkgcache.R new file mode 100644 index 0000000..1a5b418 --- /dev/null +++ b/local/tests/test-trim-pkgcache.R @@ -0,0 +1,68 @@ +source(file.path("..", "r-minor-helpers.R")) + +# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is +# set to `age_secs` in the past so we can exercise the age gate deterministically. +make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) { + root <- tempfile("pkgcache-") + meta <- file.path(root, "R", "pkgcache", "_metadata") + dir.create(meta, recursive = TRUE) + old <- Sys.time() - age_secs + mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p } + mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p } + for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i))) + for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i))) + if (keep_repos) { + mk_dir(file.path(meta, "CRAN-075c426938")) + mk_dir(file.path(meta, "BioCsoft-1ac964ed6c")) + mk_file(file.path(meta, "bioc-sysreqs.dcf.gz")) + mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive + } + root +} + +n_churn <- function(root) { + meta <- file.path(root, "R", "pkgcache", "_metadata") + length(Sys.glob(file.path(meta, "patched-*"))) + + length(Sys.glob(file.path(meta, "pkgs-*.rds"))) +} + +test_that("empty cache_dir is a no-op", { + expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L) +}) + +test_that("missing _metadata dir is a no-op", { + expect_identical( + trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0), + 0L + ) +}) + +test_that("fewer than keep entries removes nothing", { + root <- make_meta(patched = 2L, pkgs = 2L) + expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L) + expect_identical(n_churn(root), 4L) +}) + +test_that("trims down to keep newest, leaving churn == keep", { + root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old + removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0) + expect_identical(removed, 40L) + expect_identical(n_churn(root), 20L) +}) + +test_that("entries younger than min_age_secs are protected", { + root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60) + # keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed + expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L) + expect_identical(n_churn(root), 30L) +}) + +test_that("stable repo dirs and pkg downloads are never touched", { + root <- make_meta(patched = 30L, pkgs = 30L) + trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0) + meta <- file.path(root, "R", "pkgcache", "_metadata") + expect_true(dir.exists(file.path(meta, "CRAN-075c426938"))) + expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c"))) + expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz"))) + expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg"))) +}) From 04059f20e39f944cbc00a48093f52bc4b7e03f94 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 13:52:17 +0000 Subject: [PATCH 061/125] fix(patches): add rstan entry to skip removed tbb_stddef.h (#114) ## Motivation `rstan` fails to compile when it is pulled in as a transitive dependency (e.g. of `ctsem`), aborting the dependency install: ``` /mnt/cache/R-pkgs/StanHeaders/include/stan/math/prim/core/init_threadpool_tbb.hpp:9:10: fatal error: tbb/tbb_stddef.h: No such file or directory 9 | #include ``` StanHeaders' `init_threadpool_tbb.hpp` unconditionally includes the legacy `` to detect the TBB version, but that header was removed in oneTBB 2021+, which is the TBB that StanHeaders now bundles. ## Fix Add an `rstan` entry to `local/patches/registry.json` that pre-defines `TBB_INTERFACE_NEW` via `CPPFLAGS`. The header's own guard skips the removed include when that macro is already defined and selects the modern `tbb/global_control.h` + `tbb/task_arena.h` path that the bundled oneTBB actually provides: ```cpp #ifndef TBB_INTERFACE_NEW #include // removed in oneTBB 2021+ #if TBB_VERSION_MAJOR >= 2020 #define TBB_INTERFACE_NEW #endif #endif #ifdef TBB_INTERFACE_NEW #include // taken when we pre-define the macro #include ``` This is a makevars-tier override (no source diff), applied through bincraft's `R_MAKEVARS_USER`, so `CPPFLAGS` stays additive and does not clobber rstan's own `PKG_CPPFLAGS`. `-I/usr/local/include` is kept because the override replaces the default `CPPFLAGS`. ## Files changed - `local/patches/registry.json`: add the `rstan` entry (`versions: "*"`, `platforms: ["*"]`, `makevars.CPPFLAGS = "-DTBB_INTERFACE_NEW -I/usr/local/include"`). ## Behaviour change When bincraft resolves `rstan` (direct or transitive) it builds a patched binary with `TBB_INTERFACE_NEW` defined, so `rstan` (and dependents such as `ctsem`) compile against the bundled oneTBB. The scope is `["*"]` because the failure is bound to StanHeaders' bundled TBB version, not the OS or toolchain. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/114 --- local/patches/registry.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/local/patches/registry.json b/local/patches/registry.json index 7a197d7..e53f1c9 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -18,5 +18,17 @@ "makevars": {}, "patch": "fs/force-vendored-libuv.patch", "reason": "fs 2.x configure links system libuv whenever pkg-config finds libuv-devel (installed as a build-time sysreq), producing an fs.so with NEEDED libuv.so.1. That binary fails to dyn.load on consumer machines lacking runtime libuv, because install.packages/renv do not install SystemRequirements (only pak does, and only in the build container). The patch short-circuits configure to copy src/Makevars.vendor and build the bundled static libuv (needs cmake) so the binary is self-contained on every platform. An env/pkg-config override was tried first but the rebuilt binary still linked libuv.so.1, so a source patch is used instead." + }, + { + "package": "rstan", + "versions": "*", + "platforms": ["*"], + "env": {}, + "configure_args": [], + "makevars": { + "CPPFLAGS": "-DTBB_INTERFACE_NEW -I/usr/local/include" + }, + "patch": null, + "reason": "StanHeaders' init_threadpool_tbb.hpp unconditionally includes the legacy (removed in oneTBB 2021+) for version detection, breaking compilation of Module.cpp against the bundled oneTBB. Pre-defining TBB_INTERFACE_NEW skips that include and selects the modern tbb/global_control.h + tbb/task_arena.h path that the bundled TBB provides (-I/usr/local/include preserves the default CPPFLAGS the override replaces)" } ] From f11ba7172f25704248db231947ae1eb290cad601 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 14:45:25 +0000 Subject: [PATCH 062/125] feat(local): classify failing binary builds and pre-fill registry suggestions (#116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements steps 1 + 2 of #115: turn recorded build failures into triaged patch suggestions instead of hand-scraping Crow logs. ## What this adds A **read-only** reporting pipeline over the `single_builds` metadata table. It never writes to the DB or the registry. - `local/failing-builds-classify.R` — pure, DB-free helpers: - `normalise_error()` strips temp paths, version numbers, hex addresses, and the package name so the same root cause collapses to one fingerprint. - `fingerprint_error()` extracts the salient error line and normalises it. - `classify_error()` matches against a seed signature set; unmatched errors are never guessed at. - `propose_registry_entry()` renders a schema-valid `registry.json` entry. - `local/failing-builds-report.R` — entrypoint: queries `single_builds WHERE error_occurred = TRUE AND removed = FALSE`, groups by root cause (signature when classified, fingerprint otherwise), classifies each group, and prints a triaged report. Flags: `--platform`, `--arch`, `--min`, `--json`; `PLATFORM`/`ARCH` env fallbacks. - `local/tests/test-failing-builds-classify.R` — unit tests for the helpers. - `local/patches/README.md` — documents the workflow. ## Seed signatures Each rule carries a fix tier, confidence, and an auto/human-only flag: | Signature | Fix | Disposition | | --- | --- | --- | | `tbb/tbb_stddef.h: No such file` | makevars `-DTBB_INTERFACE_NEW` | auto-proposable | | RcppParallel bundled TBB (musl / new g++) | curated `disable-tbb.patch` | auto-proposable | | system `libuv.so` link leak | force vendored/static lib | **human triage** (novel source diff) | | unmatched | none | **human triage** | ## Guardrails honored - No autonomous novel source diffs: only known env/makevars levers and already-curated package patches are auto-proposable; anything needing a brand-new diff, and any unknown signature, is routed to human triage. - No DB or registry writes; no change to the public `src/contrib` index. - Reuses `single_builds.error_text`; no new failure-capture pipeline. ## Verification - All helper unit tests pass under the Dockerized R 4.5.3 build env. - Pre-commit hooks pass (`air-format`, `validate-patches`, prettier, etc.). - Smoke-tested the full report path with a stubbed DB; generated proposals pass the real `local/validate-patches.R`. Steps 3 (auto-open PRs) and 4 (feedback loop) are intentionally deferred, per the issue's suggestion to validate the signature set first. Closes #115 Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/116 --- local/failing-builds-classify.R | 204 +++++++++++++++ local/failing-builds-report.R | 290 +++++++++++++++++++++ local/patches/README.md | 38 ++- local/tests/test-failing-builds-classify.R | 87 +++++++ 4 files changed, 609 insertions(+), 10 deletions(-) create mode 100644 local/failing-builds-classify.R create mode 100644 local/failing-builds-report.R create mode 100644 local/tests/test-failing-builds-classify.R diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R new file mode 100644 index 0000000..b6f3430 --- /dev/null +++ b/local/failing-builds-classify.R @@ -0,0 +1,204 @@ +# Pure, side-effect-free helpers for triaging `single_builds` failures: +# normalise a raw `error_text` into a stable fingerprint, and classify it +# against a seed set of known failure signatures (issue #115, steps 1 + 2). +# +# Kept free of DB/IO so it can be sourced by both `failing-builds-report.R` +# and the unit tests in `local/tests/`. + +# --------------------------------------------------------------------------- +# Signature table +# --------------------------------------------------------------------------- +# Each rule maps a recurring compile/link/load error to a suggested fix tier. +# `pattern` is a case-insensitive regex matched against the raw `error_text`. +# `auto` marks whether the fix is a *known lever* safe to auto-propose as a PR +# (env / makevars / an already-curated package-specific patch). Rules that +# would require a brand-new source diff for a previously-unseen package stay +# `auto = FALSE` -> classified, but always routed to human triage, per the +# issue's guardrail against shipping autonomous novel source diffs. +# +# Seeded from the existing registry entries and known recurring failures; add +# a row here as new signatures are confirmed. Order matters: the first match +# wins, so keep more specific patterns above broader ones. +build_signatures <- function() { + list( + list( + id = "tbb-stddef-removed", + label = "removed TBB header tbb/tbb_stddef.h", + pattern = "tbb/tbb_stddef\\.h.*No such file", + tier = "makevars", + confidence = "high", + auto = TRUE, + fix = "add CPPFLAGS += -DTBB_INTERFACE_NEW so the source stops including the removed tbb/tbb_stddef.h header", + example = "StanHeaders / rstan (#114)", + registry = list( + env = NULL, + configure_args = NULL, + makevars = list(CPPFLAGS = "-DTBB_INTERFACE_NEW"), + patch = NULL, + reason = "package includes the removed tbb/tbb_stddef.h; -DTBB_INTERFACE_NEW selects the new oneTBB interface path" + ) + ), + list( + id = "rcppparallel-bundled-tbb", + label = "bundled Intel TBB build fails/hangs (musl / new g++)", + pattern = "USE_TBB[^\\n]*(not supported|unsupported)|RcppParallel[^\\n]*TBB|tbb[^\\n]*(Alpine|musl)", + tier = "patch", + confidence = "high", + auto = TRUE, + fix = "apply the curated RcppParallel/disable-tbb.patch so the bundled TBB build is skipped and the TinyThread backend is used", + example = "RcppParallel", + registry = list( + env = NULL, + configure_args = NULL, + makevars = NULL, + patch = "RcppParallel/disable-tbb.patch", + reason = "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains; patch forces the TinyThread backend" + ) + ), + list( + id = "system-libuv-link-leak", + label = "binary links system libuv (NEEDED libuv.so.1)", + pattern = "libuv\\.so", + tier = "patch", + confidence = "medium", + # A novel per-package source diff is required to force the vendored lib; + # never auto-propose, only surface for a human (guardrail). + auto = FALSE, + fix = "force the vendored/static library instead of the system one via a human-authored source patch (see fs/force-vendored-libuv.patch as precedent)", + example = "fs", + registry = list( + env = NULL, + configure_args = NULL, + makevars = NULL, + patch = "/force-vendored-.patch", + reason = "binary links the system library and fails to dyn.load on consumer machines; force the vendored/static build" + ) + ) + ) +} + +# --------------------------------------------------------------------------- +# Normalisation +# --------------------------------------------------------------------------- +# Collapse build-specific noise (temp paths, version numbers, hex addresses, +# the package name) so the same root cause across packages/platforms/versions +# maps to one fingerprint bucket. +normalise_error <- function(error_text, package = NULL) { + if (length(error_text) == 0L || is.na(error_text) || !nzchar(error_text)) { + return("") + } + x <- as.character(error_text) + # Package-specific token first (before version/number stripping mangles it). + if (!is.null(package) && length(package) == 1L && nzchar(package)) { + # \Q..\E quotes the name literally so metachars (e.g. data.table's dot) + # are matched verbatim rather than as regex. + x <- gsub(paste0("\\b\\Q", package, "\\E\\b"), "", x, perl = TRUE) + } + # R temp dirs/files: /tmp/RtmpAbC123, RtmpXXXX, /tmp/Rtmp.../file123. + x <- gsub("/tmp/[^ \t\n]*", "", x, perl = TRUE) + x <- gsub("\\bRtmp[A-Za-z0-9]+", "Rtmp", x, perl = TRUE) + # Hex addresses and version-like number runs. + x <- gsub("0x[0-9a-fA-F]+", "0x", x, perl = TRUE) + x <- gsub("[0-9]+(\\.[0-9]+)+", "", x, perl = TRUE) + x <- gsub("\\b[0-9]{2,}\\b", "", x, perl = TRUE) + # Whitespace and case. + x <- tolower(x) + x <- gsub("[ \t\r\n]+", " ", x, perl = TRUE) + trimws(x) +} + +# Extract the single most informative line from a multi-line error, then +# normalise it. This is the grouping key; a short salient line groups far +# better than the whole (often huge) transcript. +fingerprint_error <- function(error_text, package = NULL, max_chars = 200L) { + if (length(error_text) == 0L || is.na(error_text) || !nzchar(error_text)) { + return("") + } + lines <- strsplit(as.character(error_text), "\n", fixed = TRUE)[[1L]] + lines <- trimws(lines) + lines <- lines[nzchar(lines)] + if (length(lines) == 0L) { + return("") + } + salient_re <- paste( + "error:", + "fatal error:", + "no such file", + "undefined reference", + "cannot find -l", + "cannot open shared object", + "configuration failed", + "non-zero exit", + "installation of package", + "compilation failed", + sep = "|" + ) + hit <- lines[grepl(salient_re, lines, ignore.case = TRUE)] + chosen <- if (length(hit) > 0L) hit[[1L]] else lines[[length(lines)]] + fp <- normalise_error(chosen, package) + if (nchar(fp) > max_chars) { + fp <- paste0(substr(fp, 1L, max_chars), "...") + } + fp +} + +# --------------------------------------------------------------------------- +# Classification +# --------------------------------------------------------------------------- +# Return the first matching signature (as a list) enriched with `matched`, or +# the unclassified fallback. Never guesses: an unmatched error is routed to a +# human, not assigned a fix. +classify_error <- function(error_text, signatures = build_signatures()) { + txt <- if (length(error_text) == 0L || is.na(error_text)) { + "" + } else { + as.character(error_text) + } + for (sig in signatures) { + if ( + nzchar(txt) && grepl(sig$pattern, txt, ignore.case = TRUE, perl = TRUE) + ) { + sig$matched <- TRUE + return(sig) + } + } + list( + id = "unclassified", + label = "unknown signature", + tier = NA_character_, + confidence = NA_character_, + auto = FALSE, + fix = "no known signature; flag for human triage", + example = NA_character_, + registry = NULL, + matched = FALSE + ) +} + +# Render a suggested registry.json entry (as a pretty JSON string) for a +# classified group, filling package/versions/platforms from the observed +# failures. Only meaningful when the signature carries a `registry` template. +propose_registry_entry <- function( + signature, + package, + platforms, + versions = "*" +) { + if (is.null(signature$registry)) { + return(NULL) + } + tmpl <- signature$registry + entry <- list( + package = package, + versions = versions, + # I() keeps this a JSON array even when a single platform is affected. + platforms = I(as.character(platforms)) + ) + for (k in c("env", "configure_args", "makevars", "patch")) { + if (!is.null(tmpl[[k]])) { + entry[[k]] <- tmpl[[k]] + } + } + entry$reason <- tmpl$reason + jsonlite::toJSON(entry, auto_unbox = TRUE, pretty = TRUE, null = "null") +} diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R new file mode 100644 index 0000000..855f042 --- /dev/null +++ b/local/failing-builds-report.R @@ -0,0 +1,290 @@ +#!/usr/bin/env Rscript + +# Read-only failure triage over the `single_builds` metadata table (issue #115, +# steps 1 + 2): query every recorded build failure, group by a normalised error +# fingerprint, classify each bucket against the known signature set, and print a +# triaged report with a pre-filled `registry.json` suggestion where a *known, +# safe* fix lever applies. Novel source diffs and unknown signatures are routed +# to human triage; this script never writes to the DB or the registry. +# +# Usage: +# PGPASS=... Rscript local/failing-builds-report.R [--platform P] [--arch A] +# [--json out.json] [--min N] +# +# --platform / --arch restrict to one platform/arch (default: all) +# --min N only show fingerprint groups with >= N failing builds +# --json PATH also write the machine-readable report to PATH +# +# Env fallbacks: PLATFORM, ARCH (same effect as the flags). + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(DBI, quietly = TRUE) + library(RPostgres, quietly = TRUE) + library(jsonlite, quietly = TRUE) +}) + +# Locate helpers relative to this script so it runs from any CWD. +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "failing-builds-classify.R")) + +# --------------------------------------------------------------------------- +# Arguments +# --------------------------------------------------------------------------- +args <- commandArgs(trailingOnly = TRUE) +opt_val <- function(flag, default = NA_character_) { + i <- match(flag, args) + if (!is.na(i) && i < length(args)) args[[i + 1L]] else default +} +platform <- opt_val("--platform", Sys.getenv("PLATFORM", "")) +arch <- opt_val("--arch", Sys.getenv("ARCH", "")) +json_out <- opt_val("--json") +min_count <- suppressWarnings(as.integer(opt_val("--min", "1"))) +if (is.na(min_count)) { + min_count <- 1L +} + +if (nchar(Sys.getenv("PGPASS")) == 0L) { + stop( + "PGPASS env var is not set; a DB password is required (no read-only role exists)." + ) +} + +# --------------------------------------------------------------------------- +# Query failing builds +# --------------------------------------------------------------------------- +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(DBI::dbDisconnect(con), add = TRUE) + +where <- "error_occurred = TRUE AND removed = FALSE" +params <- list() +if (nzchar(platform)) { + where <- paste0(where, " AND platform = $", length(params) + 1L) + params <- c(params, platform) +} +if (nzchar(arch)) { + where <- paste0(where, " AND arch = $", length(params) + 1L) + params <- c(params, arch) +} +failures <- DBI::dbGetQuery( + con, + paste0( + "SELECT name, tag, platform, arch, r_version, timestamp, error_text ", + "FROM single_builds WHERE ", + where + ), + params = if (length(params) > 0L) params else NULL +) + +cat(sprintf( + "Queried single_builds: %d failing builds%s\n", + nrow(failures), + if (nzchar(platform) || nzchar(arch)) { + sprintf( + " (filter: platform=%s arch=%s)", + if (nzchar(platform)) platform else "*", + if (nzchar(arch)) arch else "*" + ) + } else { + "" + } +)) +if (nrow(failures) == 0L) { + cat("No failing builds to triage.\n") + q(status = 0) +} + +# --------------------------------------------------------------------------- +# Packages that already carry a registry entry (so we don't re-propose) +# --------------------------------------------------------------------------- +`%||%` <- function(a, b) if (is.null(a)) b else a +registry_file <- file.path(script_dir, "patches", "registry.json") +registered_pkgs <- character(0L) +if (file.exists(registry_file)) { + reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) + registered_pkgs <- unique(vapply( + reg, + function(e) as.character(e$package %||% ""), + character(1L) + )) +} + +# --------------------------------------------------------------------------- +# Fingerprint + classify every failure, then group +# --------------------------------------------------------------------------- +signatures <- build_signatures() +failures$fingerprint <- vapply( + seq_len(nrow(failures)), + function(i) fingerprint_error(failures$error_text[[i]], failures$name[[i]]), + character(1L) +) +failures$sig_id <- vapply( + seq_len(nrow(failures)), + function(i) classify_error(failures$error_text[[i]], signatures)$id, + character(1L) +) + +# Group by root cause: classified failures collapse by signature id (so the +# same fix candidate is one bucket regardless of surrounding log noise); +# unclassified failures fall back to the fingerprint so distinct unknowns stay +# separate for discovery. +failures$group_key <- ifelse( + failures$sig_id == "unclassified", + failures$fingerprint, + failures$sig_id +) +groups <- split(failures, failures$group_key) +# Order groups by number of affected builds, descending. +groups <- groups[order(-vapply(groups, nrow, integer(1L)))] + +report <- list() +cat(sprintf( + "\n%d distinct failure fingerprint(s); showing groups with >= %d build(s).\n", + length(groups), + min_count +)) +cat(strrep("=", 78L), "\n", sep = "") + +for (g in groups) { + n <- nrow(g) + if (n < min_count) { + next + } + + # Representative classification: most common signature id in the group. + sig_id <- names(sort(table(g$sig_id), decreasing = TRUE))[[1L]] + sig <- Filter(function(s) s$id == sig_id, signatures) + if (length(sig) > 0L) { + sig <- sig[[1L]] + sig$matched <- TRUE + } else { + sig <- classify_error("", signatures) # unclassified fallback (matched = FALSE) + } + + fp_tab <- sort(table(g$fingerprint), decreasing = TRUE) + rep_fp <- names(fp_tab)[[1L]] + fp_variants <- length(fp_tab) + pkgs <- sort(unique(g$name)) + plats <- sort(unique(g$platform)) + arches <- sort(unique(g$arch)) + unregistered <- setdiff(pkgs, registered_pkgs) + + status <- if (!sig$matched) { + "HUMAN TRIAGE (unknown signature)" + } else if (!sig$auto) { + sprintf("HUMAN TRIAGE (classified: %s; novel source diff)", sig$id) + } else { + sprintf("AUTO-PROPOSABLE (%s, %s confidence)", sig$id, sig$confidence) + } + + cat(sprintf( + "\n[%d builds | %d pkgs | %s] %s\n", + n, + length(pkgs), + toString(plats), + status + )) + cat(sprintf( + " fingerprint: %s%s\n", + rep_fp, + if (fp_variants > 1L) { + sprintf(" (+%d fingerprint variant(s))", fp_variants - 1L) + } else { + "" + } + )) + if (sig$matched) { + cat(sprintf(" signature : %s\n", sig$label)) + cat(sprintf(" suggested : [%s] %s\n", sig$tier, sig$fix)) + } + cat(sprintf(" arch : %s\n", toString(arches))) + cat(sprintf( + " packages : %s\n", + paste( + vapply( + pkgs, + function(p) { + if (p %in% registered_pkgs) paste0(p, " (has entry)") else p + }, + character(1L) + ), + collapse = ", " + ) + )) + + proposals <- list() + if (sig$matched && sig$auto && length(unregistered) > 0L) { + cat(" proposed registry entries (validate + trial-build before merge):\n") + for (p in unregistered) { + p_plats <- sort(unique(g$platform[g$name == p])) + entry <- propose_registry_entry(sig, p, p_plats) + proposals[[p]] <- entry + cat(paste0(" ", gsub("\n", "\n ", entry)), "\n", sep = "") + } + } else if (sig$matched && sig$auto && length(unregistered) == 0L) { + cat(" (all affected packages already have a registry entry)\n") + } + + report[[length(report) + 1L]] <- list( + fingerprint = rep_fp, + fingerprint_variants = fp_variants, + build_count = n, + packages = pkgs, + packages_without_entry = unregistered, + platforms = plats, + arches = arches, + signature = sig$id, + matched = sig$matched, + auto_proposable = isTRUE(sig$auto) && sig$matched, + tier = sig$tier, + confidence = sig$confidence, + suggested_fix = sig$fix, + proposed_entries = if (length(proposals) > 0L) { + lapply(proposals, jsonlite::fromJSON) + } else { + NULL + } + ) +} + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +matched <- Filter(function(r) r$matched, report) +auto <- Filter(function(r) r$auto_proposable, report) +cat("\n", strrep("=", 78L), "\n", sep = "") +cat(sprintf( + "Summary: %d groups | %d classified | %d auto-proposable | %d for human triage\n", + length(report), + length(matched), + length(auto), + length(report) - length(matched) +)) + +if (!is.na(json_out)) { + jsonlite::write_json( + report, + json_out, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf("Wrote machine-readable report to %s\n", json_out)) +} diff --git a/local/patches/README.md b/local/patches/README.md index af24b2b..b57cc1b 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -8,16 +8,16 @@ The registry is defined in `registry.json` as an array of patch entries. Each en ### Field semantics -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| `package` | string | yes | CRAN package name. | -| `versions` | string | yes | `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. | -| `platforms` | array of strings | yes | Matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). An entry matches if any listed token matches any build token. `["*"]` matches all platforms. | -| `env` | object | no | Environment variables exported only for this package's isolated build. | -| `configure_args` | array | no | Arguments passed as `--configure-args` to the isolated build. | -| `makevars` | object | no | Key/value pairs written into a package-local Makevars for the isolated build. | -| `patch` | string or null | no | Path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. | -| `reason` | string | yes | Human explanation, surfaced in logs and metadata. | +| Field | Type | Required | Description | +| ---------------- | ---------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `package` | string | yes | CRAN package name. | +| `versions` | string | yes | `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. | +| `platforms` | array of strings | yes | Matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). An entry matches if any listed token matches any build token. `["*"]` matches all platforms. | +| `env` | object | no | Environment variables exported only for this package's isolated build. | +| `configure_args` | array | no | Arguments passed as `--configure-args` to the isolated build. | +| `makevars` | object | no | Key/value pairs written into a package-local Makevars for the isolated build. | +| `patch` | string or null | no | Path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. | +| `reason` | string | yes | Human explanation, surfaced in logs and metadata. | ## Adding an entry @@ -41,3 +41,21 @@ Rscript local/validate-patches.R ``` This validates the schema, referenced patch-file existence, and checks for duplicate entries across platforms and versions. + +## Triaging failures into entries + +`local/failing-builds-report.R` turns recorded build failures into triaged patch suggestions instead of hand-scraping Crow logs (issue #115, steps 1 + 2). +It is read-only: it queries `single_builds WHERE error_occurred`, groups failures by a normalised error fingerprint, classifies each group against the known signature set in `local/failing-builds-classify.R`, and prints a report. + +```bash +# All platforms/arches; needs the DB password. +PGPASS=... Rscript local/failing-builds-report.R +# Restrict scope and also emit a machine-readable report. +PGPASS=... Rscript local/failing-builds-report.R --platform alpine-321 --arch amd64 --json report.json +``` + +Each group is tagged `AUTO-PROPOSABLE` (a known env/makevars lever, or an already-curated package patch, safe to pre-fill as a `registry.json` entry) or `HUMAN TRIAGE` (unknown signature, or a fix that needs a novel source diff). +For auto-proposable groups it prints a ready-to-review registry entry; still run `validate-patches.R` and an isolated trial build before merging. +Novel source diffs and unknown signatures stay human-reviewed by design. + +Add a new signature by appending a rule to `build_signatures()` in `local/failing-builds-classify.R`; the pure helpers are covered by `local/tests/test-failing-builds-classify.R`. diff --git a/local/tests/test-failing-builds-classify.R b/local/tests/test-failing-builds-classify.R new file mode 100644 index 0000000..66cd26b --- /dev/null +++ b/local/tests/test-failing-builds-classify.R @@ -0,0 +1,87 @@ +source(file.path("..", "failing-builds-classify.R")) + +test_that("normalise_error collapses temp paths, versions, and package tokens", { + a <- normalise_error( + "In file /tmp/RtmpAb12/foo.c: RcppParallel 5.1.9 failed at 0xdeadbeef", + package = "RcppParallel" + ) + expect_false(grepl("RtmpAb12", a)) + expect_false(grepl("5\\.1\\.9", a)) + expect_false(grepl("0xdeadbeef", a)) + expect_true(grepl("", a)) + expect_true(grepl("", a)) + + # Same root error across two versions collapses to one fingerprint. + e1 <- "StanHeaders 2.32.1: tbb/tbb_stddef.h: No such file or directory" + e2 <- "StanHeaders 2.33.0: tbb/tbb_stddef.h: No such file or directory" + expect_identical( + fingerprint_error(e1, "StanHeaders"), + fingerprint_error(e2, "StanHeaders") + ) +}) + +test_that("normalise_error is safe on NA/empty input", { + expect_identical(normalise_error(NA_character_), "") + expect_identical(normalise_error(""), "") + expect_identical(fingerprint_error(NA_character_), "") +}) + +test_that("fingerprint_error picks the salient error line, not the last line", { + txt <- paste( + "* installing *source* package 'foo' ...", + "error: bar.h: No such file or directory", + "* removing '/tmp/lib/foo'", + sep = "\n" + ) + fp <- fingerprint_error(txt, "foo") + expect_true(grepl("no such file", fp)) + expect_false(grepl("removing", fp)) +}) + +test_that("classify_error matches the removed TBB header signature (makevars, auto)", { + sig <- classify_error( + "fatal error: tbb/tbb_stddef.h: No such file or directory" + ) + expect_identical(sig$id, "tbb-stddef-removed") + expect_identical(sig$tier, "makevars") + expect_true(sig$auto) + expect_true(sig$matched) +}) + +test_that("classify_error matches RcppParallel bundled-TBB failures", { + sig <- classify_error( + "Error: USE_TBB=Linux is not supported on this toolchain" + ) + expect_identical(sig$id, "rcppparallel-bundled-tbb") + expect_identical(sig$tier, "patch") + expect_true(sig$auto) +}) + +test_that("system libuv link leak is classified but stays human-only (novel diff)", { + sig <- classify_error("cannot open shared object file: libuv.so.1") + expect_identical(sig$id, "system-libuv-link-leak") + expect_true(sig$matched) + expect_false(sig$auto) +}) + +test_that("unknown signatures are never guessed at", { + sig <- classify_error("segfault: memory not mapped at address") + expect_identical(sig$id, "unclassified") + expect_false(sig$matched) + expect_false(sig$auto) + expect_true(is.na(sig$tier)) +}) + +test_that("propose_registry_entry fills a schema-valid entry for a known lever", { + sig <- classify_error("tbb/tbb_stddef.h: No such file") + json <- propose_registry_entry(sig, "StanHeaders", c("alpine", "ubuntu-2604")) + entry <- jsonlite::fromJSON(json, simplifyVector = FALSE) + expect_identical(entry$package, "StanHeaders") + expect_identical(entry$versions, "*") + expect_identical(entry$platforms, list("alpine", "ubuntu-2604")) + expect_identical(entry$makevars$CPPFLAGS, "-DTBB_INTERFACE_NEW") + expect_true(nzchar(entry$reason)) + + # No template -> no proposal (unclassified path). + expect_null(propose_registry_entry(classify_error("weird"), "x", "alpine")) +}) From 1c297c01bf14af1100b6ec3bff79b13d86ec47eb Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 15:03:24 +0000 Subject: [PATCH 063/125] feat(local): auto-propose registry patches and track the feedback loop (#117) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements steps 3 + 4 of #115, building on the classifier merged in #116. Now that bincraft **v4.4.3** applies registry `patch`/`makevars`/`configure_args` to the *target* package build (previously deps-only), a trial patched build is a meaningful acceptance gate, so the "propose" half is viable. ## Step 3 — propose, do not apply - **`local/propose-patches.R`** — for each classified, safe fix affecting a package with no current registry entry, emits a pre-filled `registry.json` entry and validates the candidate set against a *temporary* merged registry (the real one is never touched unless asked). - default: print candidates + validation, **take no action** - `--write`: append entries to `registry.json` + the proposals ledger (you commit + open the PR) - `--open-issue`: post/update a Forgejo tracking issue (reuses the weekly-audit `httr2` + `FORGEJO_TOKEN` pattern) - **`local/trial-build-patch.R`** — isolated bincraft build of one package with the registry applied (no upload/archive/metadata; `patchhash` keeps it out of the real cache). Exit 0/1, so it gates a CI step or manual pre-merge check. The human gate stays: nothing merges. Acceptance = `validate-patches.R` passes (checked automatically) **and** the trial build succeeds. Novel source diffs and unknown signatures are never proposed (they carry `auto = FALSE`). ## Step 4 — feedback loop - **`local/proposal-tracking.R`** (read-only) — signature hit rate (builds/pkgs/addressed/open per signature), proposed-vs-merged (a proposal counts merged once its package is in the registry), and retirement candidates (registry entries whose package no longer fails, i.e. likely fixed upstream). - **`local/proposal-tracking-lib.R`** — the pure metric/ledger helpers. ## Supporting changes - Refactored the classify helpers to expose a pure `build_triage_report()` + a list-returning entry builder; `failing-builds-report.R` now renders from the shared function (no behaviour change). - `validate-patches.R` gains optional `PATCH_DIR`/`REGISTRY_FILE` overrides (backward-compatible) so a candidate registry can be validated in isolation. - Documented the propose/trial-build/tracking workflow in `local/patches/README.md`. ## Verification - 71 unit tests pass (incl. new `test-proposal-tracking-lib.R`) under the Dockerized R 4.5.3 build env. - All pre-commit hooks pass (`air-format`, `validate-patches`, prettier, etc.). - Smoke-tested all three entrypoints end-to-end with a stubbed DB: dry-run, `--write` (produces a registry that passes the canonical validator + a valid ledger, then reverted), and the tracker. Closes #115 Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/117 --- local/failing-builds-classify.R | 116 +++++++- local/failing-builds-report.R | 127 ++------ local/patches/README.md | 34 +++ local/proposal-tracking-lib.R | 103 +++++++ local/proposal-tracking.R | 135 +++++++++ local/propose-patches.R | 362 +++++++++++++++++++++++ local/tests/test-proposal-tracking-lib.R | 105 +++++++ local/trial-build-patch.R | 65 ++++ local/validate-patches.R | 62 ++-- 9 files changed, 990 insertions(+), 119 deletions(-) create mode 100644 local/proposal-tracking-lib.R create mode 100644 local/proposal-tracking.R create mode 100644 local/propose-patches.R create mode 100644 local/tests/test-proposal-tracking-lib.R create mode 100644 local/trial-build-patch.R diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index b6f3430..30c168d 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -175,10 +175,10 @@ classify_error <- function(error_text, signatures = build_signatures()) { ) } -# Render a suggested registry.json entry (as a pretty JSON string) for a -# classified group, filling package/versions/platforms from the observed -# failures. Only meaningful when the signature carries a `registry` template. -propose_registry_entry <- function( +# Build a suggested registry.json entry (as an R list) for a classified group, +# filling package/versions/platforms from the observed failures. Only +# meaningful when the signature carries a `registry` template. +propose_registry_entry_list <- function( signature, package, platforms, @@ -192,7 +192,7 @@ propose_registry_entry <- function( package = package, versions = versions, # I() keeps this a JSON array even when a single platform is affected. - platforms = I(as.character(platforms)) + platforms = I(sort(unique(as.character(platforms)))) ) for (k in c("env", "configure_args", "makevars", "patch")) { if (!is.null(tmpl[[k]])) { @@ -200,5 +200,111 @@ propose_registry_entry <- function( } } entry$reason <- tmpl$reason + entry +} + +# Same, rendered as a pretty JSON string. +propose_registry_entry <- function( + signature, + package, + platforms, + versions = "*" +) { + entry <- propose_registry_entry_list(signature, package, platforms, versions) + if (is.null(entry)) { + return(NULL) + } jsonlite::toJSON(entry, auto_unbox = TRUE, pretty = TRUE, null = "null") } + +# Resolve the representative signature for a group by its id, re-attaching the +# `matched` flag the report/proposer rely on. Unknown ids -> unclassified. +resolve_signature <- function(sig_id, signatures = build_signatures()) { + hit <- Filter(function(s) s$id == sig_id, signatures) + if (length(hit) > 0L) { + s <- hit[[1L]] + s$matched <- TRUE + s + } else { + classify_error("", signatures) # unclassified fallback (matched = FALSE) + } +} + +# --------------------------------------------------------------------------- +# Triage report (shared by the report printer and the patch proposer) +# --------------------------------------------------------------------------- +# `failures` is a data.frame with at least: name, platform, arch, error_text. +# `registered_pkgs` is the set of packages that already carry a registry entry. +# Returns a list of group records (one per root-cause bucket), ordered by +# number of affected builds descending. Pure: no IO, no printing. +build_triage_report <- function( + failures, + registered_pkgs = character(0L), + signatures = build_signatures() +) { + if (nrow(failures) == 0L) { + return(list()) + } + idx <- seq_len(nrow(failures)) + failures$fingerprint <- vapply( + idx, + function(i) fingerprint_error(failures$error_text[[i]], failures$name[[i]]), + character(1L) + ) + failures$sig_id <- vapply( + idx, + function(i) classify_error(failures$error_text[[i]], signatures)$id, + character(1L) + ) + # Group by root cause: classified failures collapse by signature id (same fix + # candidate is one bucket regardless of log noise); unclassified failures fall + # back to the fingerprint so distinct unknowns stay separate for discovery. + failures$group_key <- ifelse( + failures$sig_id == "unclassified", + failures$fingerprint, + failures$sig_id + ) + groups <- split(failures, failures$group_key) + groups <- groups[order(-vapply(groups, nrow, integer(1L)))] + + lapply(unname(groups), function(g) { + sig_id <- names(sort(table(g$sig_id), decreasing = TRUE))[[1L]] + sig <- resolve_signature(sig_id, signatures) + + fp_tab <- sort(table(g$fingerprint), decreasing = TRUE) + pkgs <- sort(unique(g$name)) + plats <- sort(unique(g$platform)) + arches <- sort(unique(g$arch)) + unregistered <- setdiff(pkgs, registered_pkgs) + auto_proposable <- isTRUE(sig$auto) && sig$matched + + proposed <- list() + if (auto_proposable) { + for (p in unregistered) { + proposed[[p]] <- propose_registry_entry_list( + sig, + p, + g$platform[g$name == p] + ) + } + } + + list( + signature = sig$id, + label = sig$label, + matched = sig$matched, + auto_proposable = auto_proposable, + tier = sig$tier, + confidence = sig$confidence, + suggested_fix = sig$fix, + fingerprint = names(fp_tab)[[1L]], + fingerprint_variants = length(fp_tab), + build_count = nrow(g), + packages = pkgs, + packages_without_entry = unregistered, + platforms = plats, + arches = arches, + proposed_entries = if (length(proposed) > 0L) proposed else NULL + ) + }) +} diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R index 855f042..60f65ea 100644 --- a/local/failing-builds-report.R +++ b/local/failing-builds-report.R @@ -127,99 +127,53 @@ if (file.exists(registry_file)) { } # --------------------------------------------------------------------------- -# Fingerprint + classify every failure, then group +# Classify + group (shared logic), then render # --------------------------------------------------------------------------- -signatures <- build_signatures() -failures$fingerprint <- vapply( - seq_len(nrow(failures)), - function(i) fingerprint_error(failures$error_text[[i]], failures$name[[i]]), - character(1L) -) -failures$sig_id <- vapply( - seq_len(nrow(failures)), - function(i) classify_error(failures$error_text[[i]], signatures)$id, - character(1L) -) +report <- build_triage_report(failures, registered_pkgs) +report <- Filter(function(r) r$build_count >= min_count, report) -# Group by root cause: classified failures collapse by signature id (so the -# same fix candidate is one bucket regardless of surrounding log noise); -# unclassified failures fall back to the fingerprint so distinct unknowns stay -# separate for discovery. -failures$group_key <- ifelse( - failures$sig_id == "unclassified", - failures$fingerprint, - failures$sig_id -) -groups <- split(failures, failures$group_key) -# Order groups by number of affected builds, descending. -groups <- groups[order(-vapply(groups, nrow, integer(1L)))] - -report <- list() cat(sprintf( - "\n%d distinct failure fingerprint(s); showing groups with >= %d build(s).\n", - length(groups), + "\n%d distinct failure group(s); showing groups with >= %d build(s).\n", + length(report), min_count )) cat(strrep("=", 78L), "\n", sep = "") -for (g in groups) { - n <- nrow(g) - if (n < min_count) { - next - } - - # Representative classification: most common signature id in the group. - sig_id <- names(sort(table(g$sig_id), decreasing = TRUE))[[1L]] - sig <- Filter(function(s) s$id == sig_id, signatures) - if (length(sig) > 0L) { - sig <- sig[[1L]] - sig$matched <- TRUE - } else { - sig <- classify_error("", signatures) # unclassified fallback (matched = FALSE) - } - - fp_tab <- sort(table(g$fingerprint), decreasing = TRUE) - rep_fp <- names(fp_tab)[[1L]] - fp_variants <- length(fp_tab) - pkgs <- sort(unique(g$name)) - plats <- sort(unique(g$platform)) - arches <- sort(unique(g$arch)) - unregistered <- setdiff(pkgs, registered_pkgs) - - status <- if (!sig$matched) { +for (r in report) { + status <- if (!r$matched) { "HUMAN TRIAGE (unknown signature)" - } else if (!sig$auto) { - sprintf("HUMAN TRIAGE (classified: %s; novel source diff)", sig$id) + } else if (!r$auto_proposable) { + sprintf("HUMAN TRIAGE (classified: %s; novel source diff)", r$signature) } else { - sprintf("AUTO-PROPOSABLE (%s, %s confidence)", sig$id, sig$confidence) + sprintf("AUTO-PROPOSABLE (%s, %s confidence)", r$signature, r$confidence) } cat(sprintf( "\n[%d builds | %d pkgs | %s] %s\n", - n, - length(pkgs), - toString(plats), + r$build_count, + length(r$packages), + toString(r$platforms), status )) cat(sprintf( " fingerprint: %s%s\n", - rep_fp, - if (fp_variants > 1L) { - sprintf(" (+%d fingerprint variant(s))", fp_variants - 1L) + r$fingerprint, + if (r$fingerprint_variants > 1L) { + sprintf(" (+%d fingerprint variant(s))", r$fingerprint_variants - 1L) } else { "" } )) - if (sig$matched) { - cat(sprintf(" signature : %s\n", sig$label)) - cat(sprintf(" suggested : [%s] %s\n", sig$tier, sig$fix)) + if (r$matched) { + cat(sprintf(" signature : %s\n", r$label)) + cat(sprintf(" suggested : [%s] %s\n", r$tier, r$suggested_fix)) } - cat(sprintf(" arch : %s\n", toString(arches))) + cat(sprintf(" arch : %s\n", toString(r$arches))) cat(sprintf( " packages : %s\n", paste( vapply( - pkgs, + r$packages, function(p) { if (p %in% registered_pkgs) paste0(p, " (has entry)") else p }, @@ -229,39 +183,20 @@ for (g in groups) { ) )) - proposals <- list() - if (sig$matched && sig$auto && length(unregistered) > 0L) { + if (!is.null(r$proposed_entries)) { cat(" proposed registry entries (validate + trial-build before merge):\n") - for (p in unregistered) { - p_plats <- sort(unique(g$platform[g$name == p])) - entry <- propose_registry_entry(sig, p, p_plats) - proposals[[p]] <- entry - cat(paste0(" ", gsub("\n", "\n ", entry)), "\n", sep = "") + for (entry in r$proposed_entries) { + j <- jsonlite::toJSON( + entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "") } - } else if (sig$matched && sig$auto && length(unregistered) == 0L) { + } else if (r$auto_proposable) { cat(" (all affected packages already have a registry entry)\n") } - - report[[length(report) + 1L]] <- list( - fingerprint = rep_fp, - fingerprint_variants = fp_variants, - build_count = n, - packages = pkgs, - packages_without_entry = unregistered, - platforms = plats, - arches = arches, - signature = sig$id, - matched = sig$matched, - auto_proposable = isTRUE(sig$auto) && sig$matched, - tier = sig$tier, - confidence = sig$confidence, - suggested_fix = sig$fix, - proposed_entries = if (length(proposals) > 0L) { - lapply(proposals, jsonlite::fromJSON) - } else { - NULL - } - ) } # --------------------------------------------------------------------------- diff --git a/local/patches/README.md b/local/patches/README.md index b57cc1b..7dd769c 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -59,3 +59,37 @@ For auto-proposable groups it prints a ready-to-review registry entry; still run Novel source diffs and unknown signatures stay human-reviewed by design. Add a new signature by appending a rule to `build_signatures()` in `local/failing-builds-classify.R`; the pure helpers are covered by `local/tests/test-failing-builds-classify.R`. + +### Proposing entries (step 3: propose, do not apply) + +`local/propose-patches.R` takes the auto-proposable candidates one step further: for each classified, safe fix affecting a package with no current entry, it emits a pre-filled `registry.json` entry and validates the candidate set against a temporary merged registry (the real registry is never touched unless you ask). +The human gate stays: it never merges. + +```bash +# Default: print candidates + validation, take no action. +PGPASS=... Rscript local/propose-patches.R +# Append the candidates to registry.json + the proposals ledger (you commit + open the PR). +PGPASS=... Rscript local/propose-patches.R --write +# Or post/update a Forgejo tracking issue instead (needs FORGEJO_TOKEN). +PGPASS=... FORGEJO_TOKEN=... Rscript local/propose-patches.R --open-issue +``` + +The acceptance criteria before merging a proposal are: `validate-patches.R` passes (checked automatically), and an isolated trial build succeeds. +Run the trial build inside the failing platform's build-env image; it uploads/archives nothing and writes no metadata: + +```bash +Rscript local/trial-build-patch.R +``` + +`--write` and `--open-issue` also append to `local/patches/proposals-log.json`, a ledger of what was proposed. + +### Feedback loop (step 4) + +`local/proposal-tracking.R` reports the signature hit rate, proposed-vs-merged status (a proposal counts as merged once its package appears in the registry), and retirement candidates (registry entries whose package no longer appears in any current failure, so the upstream cause was likely fixed). +It is read-only. + +```bash +PGPASS=... Rscript local/proposal-tracking.R --json metrics.json +``` + +The pure metric/ledger helpers live in `local/proposal-tracking-lib.R` and are covered by `local/tests/test-proposal-tracking-lib.R`. diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R new file mode 100644 index 0000000..4e8a468 --- /dev/null +++ b/local/proposal-tracking-lib.R @@ -0,0 +1,103 @@ +# Pure, IO-free helpers for the proposal feedback loop (issue #115, step 4): +# a small ledger of what the classifier proposed, plus metrics derived from the +# live triage report and the registry (signature hit rate, proposed-vs-merged, +# retirement candidates). +# +# Kept free of DB/HTTP/clock so it can be sourced by the proposer, the tracker +# entrypoint, and the unit tests. Timestamps are passed in by callers. + +# Stable identity of a ledger record: one proposal per (package, signature). +ledger_key <- function(record) { + paste0( + if (is.null(record$package)) "?" else record$package, + "|", + if (is.null(record$signature)) "?" else record$signature + ) +} + +# Merge freshly-generated proposals into an existing ledger without clobbering +# history: a record whose (package, signature) already exists is left as-is +# (its status/PR/issue are preserved); genuinely new proposals are appended. +# Returns the combined list. Pure: callers stamp `proposed_at` before passing. +merge_ledger <- function(existing, new_records) { + if (is.null(existing)) { + existing <- list() + } + seen <- vapply(existing, ledger_key, character(1L)) + out <- existing + for (rec in new_records) { + if (!(ledger_key(rec) %in% seen)) { + out[[length(out) + 1L]] <- rec + seen <- c(seen, ledger_key(rec)) + } + } + out +} + +# Per-signature hit rate from a triage report (list of group records from +# build_triage_report) crossed with the set of already-registered packages. +# For each *matched* signature: how many failing builds/packages it explains, +# and how many of those packages are already addressed by a registry entry. +signature_hit_rate <- function(report, registered_pkgs = character(0L)) { + matched <- Filter(function(g) isTRUE(g$matched), report) + by_sig <- split( + matched, + vapply(matched, function(g) g$signature, character(1L)) + ) + lapply(names(by_sig), function(sig) { + grps <- by_sig[[sig]] + pkgs <- unique(unlist(lapply(grps, function(g) g$packages))) + addressed <- intersect(pkgs, registered_pkgs) + list( + signature = sig, + builds = sum(vapply(grps, function(g) g$build_count, integer(1L))), + packages = length(pkgs), + addressed = length(addressed), + open = length(setdiff(pkgs, registered_pkgs)), + auto_proposable = any(vapply( + grps, + function(g) isTRUE(g$auto_proposable), + logical(1L) + )) + ) + }) +} + +# Proposed-vs-merged: a ledger record counts as "merged" once its package +# appears in the registry. Returns per-record status plus a rollup. +proposed_vs_merged <- function(ledger, registered_pkgs = character(0L)) { + if (is.null(ledger)) { + ledger <- list() + } + rows <- lapply(ledger, function(rec) { + merged <- !is.null(rec$package) && rec$package %in% registered_pkgs + list( + package = rec$package, + signature = rec$signature, + status = if (merged) "merged" else (rec$status %||% "proposed"), + merged = merged + ) + }) + list( + records = rows, + total = length(rows), + merged = sum(vapply(rows, function(r) isTRUE(r$merged), logical(1L))) + ) +} + +# Registry entries whose package no longer appears in any current failure are +# retirement candidates: the upstream cause was likely fixed, so the entry can +# be reviewed for removal. `failing_pkgs` is the set of currently-failing +# package names (from the live report). +retirement_candidates <- function(registry_entries, failing_pkgs) { + if (is.null(registry_entries)) { + registry_entries <- list() + } + keep <- Filter( + function(e) !is.null(e$package) && !(e$package %in% failing_pkgs), + registry_entries + ) + vapply(keep, function(e) as.character(e$package), character(1L)) +} + +`%||%` <- function(a, b) if (is.null(a)) b else a diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R new file mode 100644 index 0000000..60bbd10 --- /dev/null +++ b/local/proposal-tracking.R @@ -0,0 +1,135 @@ +#!/usr/bin/env Rscript + +# Feedback loop for the failure classifier (issue #115, step 4): report the +# signature hit rate, proposed-vs-merged status, and registry entries that look +# retirable, so the rule set can improve and stale entries can be pruned. +# Read-only: queries `single_builds` and reads the registry + proposals ledger. +# +# Usage: +# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(DBI, quietly = TRUE) + library(RPostgres, quietly = TRUE) + library(jsonlite, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "failing-builds-classify.R")) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +args <- commandArgs(trailingOnly = TRUE) +json_out <- local({ + i <- match("--json", args) + if (!is.na(i) && i < length(args)) args[[i + 1L]] else NA_character_ +}) + +if (nchar(Sys.getenv("PGPASS")) == 0L) { + stop("PGPASS env var is not set; a DB password is required.") +} + +registry_file <- file.path(script_dir, "patches", "registry.json") +ledger_file <- file.path(script_dir, "patches", "proposals-log.json") + +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(DBI::dbDisconnect(con), add = TRUE) + +failures <- DBI::dbGetQuery( + con, + paste0( + "SELECT name, tag, platform, arch, r_version, timestamp, error_text ", + "FROM single_builds WHERE error_occurred = TRUE AND removed = FALSE" + ) +) + +existing_entries <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +registered_pkgs <- unique(vapply( + existing_entries, + function(e) as.character(e$package %||% ""), + character(1L) +)) +ledger <- if (file.exists(ledger_file)) { + jsonlite::fromJSON(ledger_file, simplifyVector = FALSE) +} else { + list() +} + +report <- build_triage_report(failures, registered_pkgs) +failing_pkgs <- unique(unlist(lapply(report, function(g) g$packages))) + +hit <- signature_hit_rate(report, registered_pkgs) +pvm <- proposed_vs_merged(ledger, registered_pkgs) +retire <- retirement_candidates(existing_entries, failing_pkgs) + +cat(sprintf( + "Feedback loop over %d failing builds, %d registry entries, %d ledger records.\n", + nrow(failures), + length(existing_entries), + length(ledger) +)) + +cat("\nSignature hit rate:\n") +cat(strrep("-", 60L), "\n", sep = "") +for (h in hit) { + cat(sprintf( + " %-26s %3d builds | %2d pkgs | %2d addressed | %2d open%s\n", + h$signature, + h$builds, + h$packages, + h$addressed, + h$open, + if (isTRUE(h$auto_proposable)) " [auto]" else "" + )) +} + +cat(sprintf("\nProposed vs merged: %d / %d merged.\n", pvm$merged, pvm$total)) +for (r in pvm$records) { + cat(sprintf(" %-24s %-26s %s\n", r$package, r$signature, r$status)) +} + +cat(sprintf( + "\nRetirement candidates (%d): registry entries with no current failure.\n", + length(retire) +)) +if (length(retire) > 0L) { + cat(paste0(" - ", retire, "\n"), sep = "") +} else { + cat(" (none)\n") +} + +if (!is.na(json_out)) { + jsonlite::write_json( + list( + signature_hit_rate = hit, + proposed_vs_merged = pvm, + retirement_candidates = retire + ), + json_out, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf("\nWrote metrics to %s\n", json_out)) +} diff --git a/local/propose-patches.R b/local/propose-patches.R new file mode 100644 index 0000000..20583ba --- /dev/null +++ b/local/propose-patches.R @@ -0,0 +1,362 @@ +#!/usr/bin/env Rscript + +# Propose registry patches for classified, auto-proposable build failures +# (issue #115, step 3: "propose, do not apply"). Queries `single_builds`, +# classifies failures, and for each *known, safe* fix lever affecting a package +# with no current registry entry, emits a pre-filled `registry.json` entry. +# +# The human gate stays: this never merges. Acceptance before merge is +# 1. `local/validate-patches.R` passes (checked here against a candidate +# registry), and +# 2. an isolated trial build succeeds -- run `local/trial-build-patch.R ` +# (this script prints the exact command per candidate). +# Novel source diffs and unknown signatures are never proposed (they carry +# `auto = FALSE` in the signature table). +# +# Usage: +# PGPASS=... Rscript local/propose-patches.R [--platform P] [--arch A] [--min N] +# (default) print candidates + validation, take no action +# --write append candidates to local/patches/registry.json and +# the proposals ledger (commit + open a PR yourself) +# --open-issue post/update a Forgejo tracking issue (needs FORGEJO_TOKEN) +# --json PATH also write the machine-readable candidate list to PATH + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(DBI, quietly = TRUE) + library(RPostgres, quietly = TRUE) + library(jsonlite, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "failing-builds-classify.R")) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +# --------------------------------------------------------------------------- +# Arguments +# --------------------------------------------------------------------------- +args <- commandArgs(trailingOnly = TRUE) +opt_val <- function(flag, default = NA_character_) { + i <- match(flag, args) + if (!is.na(i) && i < length(args)) args[[i + 1L]] else default +} +platform <- opt_val("--platform", Sys.getenv("PLATFORM", "")) +arch <- opt_val("--arch", Sys.getenv("ARCH", "")) +json_out <- opt_val("--json") +min_count <- suppressWarnings(as.integer(opt_val("--min", "1"))) +if (is.na(min_count)) { + min_count <- 1L +} +do_write <- "--write" %in% args +do_issue <- "--open-issue" %in% args + +if (nchar(Sys.getenv("PGPASS")) == 0L) { + stop("PGPASS env var is not set; a DB password is required.") +} + +registry_file <- file.path(script_dir, "patches", "registry.json") +ledger_file <- file.path(script_dir, "patches", "proposals-log.json") + +# --------------------------------------------------------------------------- +# Query failing builds (same shape as failing-builds-report.R) +# --------------------------------------------------------------------------- +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(DBI::dbDisconnect(con), add = TRUE) + +where <- "error_occurred = TRUE AND removed = FALSE" +params <- list() +if (nzchar(platform)) { + where <- paste0(where, " AND platform = $", length(params) + 1L) + params <- c(params, platform) +} +if (nzchar(arch)) { + where <- paste0(where, " AND arch = $", length(params) + 1L) + params <- c(params, arch) +} +failures <- DBI::dbGetQuery( + con, + paste0( + "SELECT name, tag, platform, arch, r_version, timestamp, error_text ", + "FROM single_builds WHERE ", + where + ), + params = if (length(params) > 0L) params else NULL +) +cat(sprintf("Queried single_builds: %d failing builds\n", nrow(failures))) + +# --------------------------------------------------------------------------- +# Existing registry + classification +# --------------------------------------------------------------------------- +existing_entries <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +registered_pkgs <- unique(vapply( + existing_entries, + function(e) as.character(e$package %||% ""), + character(1L) +)) + +report <- build_triage_report(failures, registered_pkgs) +report <- Filter(function(r) r$build_count >= min_count, report) + +# Flatten auto-proposable groups into candidate records. +candidates <- list() +for (r in report) { + if (is.null(r$proposed_entries)) { + next + } + for (pkg in names(r$proposed_entries)) { + candidates[[length(candidates) + 1L]] <- list( + package = pkg, + signature = r$signature, + confidence = r$confidence, + tier = r$tier, + entry = r$proposed_entries[[pkg]] + ) + } +} + +if (length(candidates) == 0L) { + cat( + "No auto-proposable candidates (nothing classified, safe, and unregistered).\n" + ) + q(status = 0) +} + +cat(sprintf( + "\n%d candidate registry %s:\n", + length(candidates), + if (length(candidates) == 1L) "entry" else "entries" +)) +cat(strrep("=", 78L), "\n", sep = "") +for (c in candidates) { + j <- jsonlite::toJSON( + c$entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf( + "\n# %s [signature: %s, %s confidence]\n", + c$package, + c$signature, + c$confidence + )) + cat(j, "\n", sep = "") + cat(sprintf( + " acceptance gate: PGPASS=... Rscript local/trial-build-patch.R %s\n", + c$package + )) +} + +# --------------------------------------------------------------------------- +# Validate the candidate set against a temporary merged registry (never +# touches the real registry unless --write succeeds). +# --------------------------------------------------------------------------- +rscript <- file.path(R.home("bin"), "Rscript") +candidate_entries <- lapply(candidates, function(c) c$entry) +merged <- c(existing_entries, candidate_entries) + +validate_registry <- function(entries) { + tmp <- tempfile(fileext = ".json") + on.exit(unlink(tmp), add = TRUE) + jsonlite::write_json( + entries, + tmp, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + status <- system2( + rscript, + file.path(script_dir, "validate-patches.R"), + env = c( + paste0("PATCH_DIR=", file.path(script_dir, "patches")), + paste0("REGISTRY_FILE=", tmp) + ), + stdout = TRUE, + stderr = TRUE + ) + cat(paste0(" ", status, "\n"), sep = "") + identical(attr(status, "status"), NULL) # NULL status attr == exit 0 +} + +cat("\n", strrep("=", 78L), "\n", sep = "") +cat("Validating candidate registry:\n") +ok <- validate_registry(merged) +if (!ok) { + stop("Candidate registry failed validation; not writing or proposing.") +} + +if (!is.na(json_out)) { + jsonlite::write_json( + candidates, + json_out, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf("Wrote candidate list to %s\n", json_out)) +} + +# --------------------------------------------------------------------------- +# Actions (default: none) +# --------------------------------------------------------------------------- +now <- format(Sys.time(), "%Y-%m-%d %H:%M:%S") +new_ledger_records <- lapply(candidates, function(c) { + list( + package = c$package, + signature = c$signature, + tier = c$tier, + confidence = c$confidence, + versions = c$entry$versions, + platforms = as.character(c$entry$platforms), + proposed_at = now, + status = "proposed" + ) +}) +load_ledger <- function() { + if (file.exists(ledger_file)) { + jsonlite::fromJSON(ledger_file, simplifyVector = FALSE) + } else { + list() + } +} +save_ledger <- function(led) { + jsonlite::write_json( + led, + ledger_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) +} + +if (do_write) { + jsonlite::write_json( + merged, + registry_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) + cat(sprintf( + "\nWrote %d entries to %s and updated %s.\n", + length(candidate_entries), + registry_file, + ledger_file + )) + cat( + "Next: run the acceptance-gate trial build(s), then commit and open a PR.\n" + ) +} else if (do_issue) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-issue requires FORGEJO_TOKEN.") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + issue_title <- "Auto-proposed registry patches (classifier #115)" + body_lines <- c( + sprintf("_Generated %s from `single_builds` failures._", now), + "", + "The failure classifier proposes these registry entries for packages with no current entry.", + "Each is a known, safe fix lever; **review, trial-build, and open a PR** -- nothing is applied automatically.", + "" + ) + for (c in candidates) { + j <- jsonlite::toJSON( + c$entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + body_lines <- c( + body_lines, + sprintf( + "### %s (%s, %s confidence)", + c$package, + c$signature, + c$confidence + ), + "", + "```json", + as.character(j), + "```", + sprintf( + "Acceptance gate: `PGPASS=... Rscript local/trial-build-patch.R %s`", + c$package + ), + "" + ) + } + new_body <- paste(body_lines, collapse = "\n") + + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(issue_title, reserved = TRUE) + ) + existing <- httr2::request(search_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + existing, + function(x) identical(x$title, issue_title), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- existing[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("Updated tracking issue #%d.\n", num)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/issues", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = issue_title, body = new_body)) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf("Opened tracking issue #%d.\n", created$number)) + } + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) +} else { + cat( + "\nDry run: no changes made. Re-run with --write or --open-issue to act.\n" + ) +} diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R new file mode 100644 index 0000000..55da4b6 --- /dev/null +++ b/local/tests/test-proposal-tracking-lib.R @@ -0,0 +1,105 @@ +source(file.path("..", "proposal-tracking-lib.R")) +source(file.path("..", "failing-builds-classify.R")) + +mk_failures <- function() { + data.frame( + name = c("StanHeaders", "rstan", "RcppParallel", "somepkg"), + platform = c("alpine-321", "alpine-321", "alpine-320", "redhat-9"), + arch = c("amd64", "arm64", "amd64", "amd64"), + error_text = c( + "fatal error: tbb/tbb_stddef.h: No such file or directory", + "In file: tbb/tbb_stddef.h: No such file or directory", + "Error: USE_TBB=Linux is not supported; bundled TBB on musl", + "some unmatched failure" + ), + stringsAsFactors = FALSE + ) +} + +test_that("merge_ledger appends new proposals and preserves existing history", { + existing <- list(list( + package = "fs", + signature = "system-libuv-link-leak", + status = "merged" + )) + new <- list( + list( + package = "fs", + signature = "system-libuv-link-leak", + status = "proposed" + ), + list( + package = "StanHeaders", + signature = "tbb-stddef-removed", + status = "proposed" + ) + ) + merged <- merge_ledger(existing, new) + expect_length(merged, 2L) # fs is deduped, StanHeaders added + fs <- Filter(function(r) r$package == "fs", merged)[[1L]] + expect_identical(fs$status, "merged") # existing status preserved, not clobbered +}) + +test_that("merge_ledger handles an empty/NULL starting ledger", { + new <- list(list(package = "x", signature = "s")) + expect_length(merge_ledger(NULL, new), 1L) + expect_length(merge_ledger(list(), new), 1L) +}) + +test_that("signature_hit_rate splits addressed vs open per signature", { + report <- build_triage_report(mk_failures(), registered_pkgs = "RcppParallel") + hit <- signature_hit_rate(report, registered_pkgs = "RcppParallel") + tbb <- Filter(function(h) h$signature == "tbb-stddef-removed", hit)[[1L]] + expect_identical(tbb$packages, 2L) # StanHeaders + rstan + expect_identical(tbb$addressed, 0L) + expect_identical(tbb$open, 2L) + expect_true(tbb$auto_proposable) + + rcpp <- Filter(function(h) h$signature == "rcppparallel-bundled-tbb", hit)[[ + 1L + ]] + expect_identical(rcpp$addressed, 1L) # already registered + expect_identical(rcpp$open, 0L) + + # Unclassified failures never appear as a signature. + expect_false( + "unclassified" %in% vapply(hit, function(h) h$signature, character(1L)) + ) +}) + +test_that("proposed_vs_merged marks a package merged once it is registered", { + ledger <- list( + list( + package = "StanHeaders", + signature = "tbb-stddef-removed", + status = "proposed" + ), + list( + package = "rstan", + signature = "tbb-stddef-removed", + status = "proposed" + ) + ) + pvm <- proposed_vs_merged(ledger, registered_pkgs = "StanHeaders") + expect_identical(pvm$total, 2L) + expect_identical(pvm$merged, 1L) + stan <- Filter(function(r) r$package == "StanHeaders", pvm$records)[[1L]] + expect_identical(stan$status, "merged") +}) + +test_that("retirement_candidates flags entries whose package no longer fails", { + entries <- list( + list(package = "RcppParallel"), + list(package = "oldpkg") + ) + # RcppParallel still fails; oldpkg does not -> only oldpkg is retirable. + out <- retirement_candidates( + entries, + failing_pkgs = c("RcppParallel", "StanHeaders") + ) + expect_identical(out, "oldpkg") + expect_length( + retirement_candidates(entries, failing_pkgs = c("RcppParallel", "oldpkg")), + 0L + ) +}) diff --git a/local/trial-build-patch.R b/local/trial-build-patch.R new file mode 100644 index 0000000..4e8030a --- /dev/null +++ b/local/trial-build-patch.R @@ -0,0 +1,65 @@ +#!/usr/bin/env Rscript + +# Acceptance gate for a proposed registry patch (issue #115, step 3): build one +# package in isolation with the current `local/patches` registry applied, and +# report whether it succeeds. Nothing is uploaded, archived, or written to the +# metadata DB -- `patchhash` keeps the trial binary out of the real cache. +# +# Must run inside a build-env image (the same image the failing platform uses), +# because it invokes the real compiler toolchain via bincraft. +# +# Usage: +# Rscript local/trial-build-patch.R [tag] +# +# Exit status: 0 if the patched build succeeds, 1 otherwise -- so it can gate a +# CI step or a manual pre-merge check. + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 1L) { + stop("usage: Rscript local/trial-build-patch.R [tag]") +} +package <- args[[1L]] +tag <- if (length(args) >= 2L) args[[2L]] else NULL + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +patches_dir <- file.path( + if (is.na(script_path)) "local" else dirname(script_path), + "patches" +) + +suppressPackageStartupMessages(library(bincraft, quietly = TRUE)) + +cat(sprintf( + "Trial build: %s%s with registry %s (no upload/archive/metadata)\n", + package, + if (is.null(tag)) "" else sprintf(" @ %s", tag), + patches_dir +)) + +ok <- tryCatch( + { + bincraft::build_binary_package( + package, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ) + TRUE + }, + error = function(e) { + cat(sprintf("Trial build FAILED: %s\n", conditionMessage(e))) + FALSE + } +) + +if (ok) { + cat(sprintf("Trial build OK: %s builds with the proposed patch.\n", package)) + q(status = 0) +} +q(status = 1) diff --git a/local/validate-patches.R b/local/validate-patches.R index 5f5aace..b20aea1 100644 --- a/local/validate-patches.R +++ b/local/validate-patches.R @@ -1,9 +1,17 @@ #!/usr/bin/env Rscript # Validate local/patches/registry.json: schema, referenced patch files, and # ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. +# +# Defaults to local/patches/registry.json. To validate a candidate registry +# without touching the real one (e.g. from the patch proposer), set: +# PATCH_DIR directory patch-file paths resolve against (default local/patches) +# REGISTRY_FILE registry.json to validate (default /registry.json) -dir <- "local/patches" -registry_file <- file.path(dir, "registry.json") +dir <- Sys.getenv("PATCH_DIR", unset = "local/patches") +registry_file <- Sys.getenv( + "REGISTRY_FILE", + unset = file.path(dir, "registry.json") +) if (!file.exists(registry_file)) { cat("No registry.json found; nothing to validate.\n") quit(status = 0L) @@ -19,33 +27,47 @@ for (i in seq_along(reg)) { e <- reg[[i]] missing <- setdiff(required, names(e)) if (length(missing) > 0L) { - errs <- c(errs, sprintf( - "entry %d (%s): missing %s", i, - if (is.null(e$package)) "?" else e$package, toString(missing) - )) + errs <- c( + errs, + sprintf( + "entry %d (%s): missing %s", + i, + if (is.null(e$package)) "?" else e$package, + toString(missing) + ) + ) } if (!is.null(e$patch)) { p <- file.path(dir, e$patch) if (!file.exists(p)) { - errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing", - i, e$package, p)) + errs <- c( + errs, + sprintf("entry %d (%s): patch file '%s' missing", i, e$package, p) + ) } } } # Ambiguous overlap: two entries for the same package with identical platforms # and versions. -keys <- vapply(reg, function(e) { - sprintf( - "%s|%s|%s", - or_q(e$package), - paste(sort(as.character(unlist(e$platforms))), collapse = ","), - or_q(e$versions) - ) -}, character(1L)) +keys <- vapply( + reg, + function(e) { + sprintf( + "%s|%s|%s", + or_q(e$package), + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + or_q(e$versions) + ) + }, + character(1L) +) dups <- keys[duplicated(keys)] if (length(dups) > 0L) { - errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups)))) + errs <- c( + errs, + sprintf("ambiguous duplicate entries: %s", toString(unique(dups))) + ) } if (length(errs) > 0L) { @@ -53,4 +75,8 @@ if (length(errs) > 0L) { cat(paste0(" - ", errs, "\n")) quit(status = 1L) } -cat(sprintf("Patch registry OK (%d %s).\n", length(reg), if (length(reg) == 1L) "entry" else "entries")) +cat(sprintf( + "Patch registry OK (%d %s).\n", + length(reg), + if (length(reg) == 1L) "entry" else "entries" +)) From af481f5129f1129f3937594c2335fdb4dc81dc37 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 15:17:16 +0000 Subject: [PATCH 064/125] ci(crow): add weekly patch-proposal + feedback-loop pipeline (#118) Adds the scheduled pipeline for the failure-triage proposer/tracker landed in #117 (issue #115, steps 3 + 4). ## `.crow/weekly-patch-proposals.yaml` A single (non-matrix) job -- the classifier groups over all of `single_builds`, so unlike the per-platform `weekly-audit-missing` it runs once globally. Weekly it: 1. `propose-patches.R --open-issue` -- posts/updates a Forgejo tracking issue with the auto-proposable registry entries (human reviews, trial-builds, opens the PR; nothing merges). 2. `proposal-tracking.R` -- logs the feedback-loop metrics (signature hit rate, proposed-vs-merged, retirement candidates). Low blast radius: clones read-only (`REPO_RO_TOKEN`); the only write is the tracking issue via `FORGEJO_TOKEN`. It builds nothing (no B2 keys, no bincraft). Triggers: `cron: weekly-patch-proposals` (register the cron in the crow UI) or manual with `task=weekly-patch-proposals`, matching the `archive-missed-packages` convention. Documented under "Scheduled run" in `local/patches/README.md`. `yamllint` and the other pre-commit hooks pass. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/118 --- .crow/weekly-patch-proposals.yaml | 57 +++++++++++++++++++++++++++++++ local/patches/README.md | 5 +++ 2 files changed, 62 insertions(+) create mode 100644 .crow/weekly-patch-proposals.yaml diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml new file mode 100644 index 0000000..8fa45df --- /dev/null +++ b/.crow/weekly-patch-proposals.yaml @@ -0,0 +1,57 @@ +# Weekly failure-triage proposals (issue #115, steps 3 + 4). +# Classifies the recorded `single_builds` failures and: +# 1. posts/updates a Forgejo tracking issue with the auto-proposable registry +# entries (human reviews, trial-builds, and opens the PR -- nothing merges), and +# 2. prints the feedback-loop metrics (signature hit rate, proposed-vs-merged, +# retirement candidates) to the run log. +# Global across platforms (the classifier groups over all of single_builds), so +# a single job -- no matrix. Clones read-only; the only write is the Forgejo +# issue via FORGEJO_TOKEN. +# +# Register the cron in the crow UI as `weekly-patch-proposals`, or run manually: +# woodpecker-cli pipeline create --var task=weekly-patch-proposals --branch=main 7 +when: + - event: manual + evaluate: 'task == "weekly-patch-proposals"' + - event: cron + cron: weekly-patch-proposals + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Propose registry patches + track feedback loop' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + R_VERSION: 4.5.3 + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue + - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 2000m + limits: + memory: 2Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/local/patches/README.md b/local/patches/README.md index 7dd769c..e875217 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -93,3 +93,8 @@ PGPASS=... Rscript local/proposal-tracking.R --json metrics.json ``` The pure metric/ledger helpers live in `local/proposal-tracking-lib.R` and are covered by `local/tests/test-proposal-tracking-lib.R`. + +### Scheduled run + +`.crow/weekly-patch-proposals.yaml` runs both steps weekly (register the `weekly-patch-proposals` cron in the crow UI): it posts/updates a Forgejo tracking issue with the auto-proposable entries and logs the feedback-loop metrics. +It clones read-only; the only write is the tracking issue. From 77508f8fc5f8d5f82ffcbc7121adb60593137cf6 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 18:17:28 +0200 Subject: [PATCH 065/125] ci: clean weekly-patch-proposals trigger --- .crow/weekly-patch-proposals.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 8fa45df..f956f3f 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -7,12 +7,8 @@ # Global across platforms (the classifier groups over all of single_builds), so # a single job -- no matrix. Clones read-only; the only write is the Forgejo # issue via FORGEJO_TOKEN. -# -# Register the cron in the crow UI as `weekly-patch-proposals`, or run manually: -# woodpecker-cli pipeline create --var task=weekly-patch-proposals --branch=main 7 when: - event: manual - evaluate: 'task == "weekly-patch-proposals"' - event: cron cron: weekly-patch-proposals From f1dd661213acdefdd7ee52cadb20bd0ddad9524b Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 17:11:54 +0000 Subject: [PATCH 066/125] fix(local): route multi-signature packages to human triage in the proposer (#119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem A live `propose-patches.R --open-issue` run aborted with: ``` Patch registry validation FAILED: - ambiguous duplicate entries: hmmTMB|ubuntu-2604|*, imt|ubuntu-2604|*, refundBayes|ubuntu-2604|* Error: Candidate registry failed validation; not writing or proposing. ``` Root cause: a package whose failing builds match **more than one** auto-proposable signature (e.g. some logs hit `tbb-stddef-removed`, others `rcppparallel-bundled-tbb`) landed in two separate signature groups, so the proposer emitted two candidate entries with the same `package|platforms|versions` key. The validator's ambiguous-duplicate check then failed the whole candidate set, aborting the run — so even the unambiguous candidates (e.g. `vacalibration`) never got proposed. ## Fix Add a pure, tested `dedupe_candidates()` that: - keeps a package's candidate only when it maps to a **single** signature (collapsing an accidental repeat under the same signature), and - routes a package that maps to **multiple** signatures (conflicting fix tiers -- makevars vs source patch) to **human triage** instead of guessing between them. The proposer prints the skipped ambiguous packages and proceeds with the clean candidates, so one ambiguous package no longer blocks the rest. This matches the issue #115 guardrail: don't auto-pick when the fix is ambiguous. ## Verification - New unit tests in `test-proposal-tracking-lib.R` cover the single-signature, multi-signature (conflict), duplicate-under-one-signature, and empty cases. - Reproduced the original failure end-to-end with a stubbed DB (`hmmTMB` ambiguous + `vacalibration` clean): `hmmTMB` is now listed under "Ambiguous -> human triage", `vacalibration` is proposed, and the candidate registry validates (`Patch registry OK`). - Full suite: 78 tests pass; all pre-commit hooks pass. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/119 --- local/proposal-tracking-lib.R | 31 +++++++++++++++++++++++ local/propose-patches.R | 18 ++++++++++++- local/tests/test-proposal-tracking-lib.R | 32 ++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 4e8a468..2edd9c8 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -6,6 +6,37 @@ # Kept free of DB/HTTP/clock so it can be sourced by the proposer, the tracker # entrypoint, and the unit tests. Timestamps are passed in by callers. +# Split proposal candidates into the ones safe to emit and the ambiguous ones. +# A candidate is a list with at least `package` and `signature`. A package that +# maps to more than one distinct auto-proposable signature is genuinely +# ambiguous (two conflicting fix tiers, e.g. makevars vs source patch): emitting +# both would create colliding registry entries, so those are routed to human +# triage instead of guessed at. Returns list(keep = ..., ambiguous = ...), where +# `ambiguous` is a named list of package -> the distinct signatures seen. +dedupe_candidates <- function(candidates) { + if (length(candidates) == 0L) { + return(list(keep = list(), ambiguous = list())) + } + pkgs <- vapply(candidates, function(c) as.character(c$package), character(1L)) + by_pkg <- split(candidates, pkgs) + keep <- list() + ambiguous <- list() + for (pkg in names(by_pkg)) { + cs <- by_pkg[[pkg]] + sigs <- unique(vapply( + cs, + function(c) as.character(c$signature), + character(1L) + )) + if (length(sigs) == 1L) { + keep[[length(keep) + 1L]] <- cs[[1L]] # one signature -> take the first + } else { + ambiguous[[pkg]] <- sigs + } + } + list(keep = keep, ambiguous = ambiguous) +} + # Stable identity of a ledger record: one proposal per (package, signature). ledger_key <- function(record) { paste0( diff --git a/local/propose-patches.R b/local/propose-patches.R index 20583ba..d43668a 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -135,9 +135,25 @@ for (r in report) { } } +# A package that maps to more than one auto-proposable signature is ambiguous +# (conflicting fix tiers) and would collide on the same registry key; route it +# to human triage instead of emitting both. +split_candidates <- dedupe_candidates(candidates) +candidates <- split_candidates$keep +if (length(split_candidates$ambiguous) > 0L) { + cat("\nAmbiguous (multiple signatures) -> human triage, not proposed:\n") + for (pkg in names(split_candidates$ambiguous)) { + cat(sprintf( + " %s: %s\n", + pkg, + toString(split_candidates$ambiguous[[pkg]]) + )) + } +} + if (length(candidates) == 0L) { cat( - "No auto-proposable candidates (nothing classified, safe, and unregistered).\n" + "\nNo auto-proposable candidates (nothing classified, safe, unregistered, and unambiguous).\n" ) q(status = 0) } diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index 55da4b6..a2fbe05 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -46,6 +46,38 @@ test_that("merge_ledger handles an empty/NULL starting ledger", { expect_length(merge_ledger(list(), new), 1L) }) +test_that("dedupe_candidates keeps single-signature pkgs, routes conflicts to triage", { + candidates <- list( + list(package = "StanHeaders", signature = "tbb-stddef-removed"), + list(package = "hmmTMB", signature = "tbb-stddef-removed"), + list(package = "hmmTMB", signature = "rcppparallel-bundled-tbb") # conflict + ) + out <- dedupe_candidates(candidates) + kept <- vapply(out$keep, function(c) c$package, character(1L)) + expect_identical(sort(kept), "StanHeaders") # hmmTMB dropped as ambiguous + expect_true("hmmTMB" %in% names(out$ambiguous)) + expect_setequal( + out$ambiguous$hmmTMB, + c("tbb-stddef-removed", "rcppparallel-bundled-tbb") + ) +}) + +test_that("dedupe_candidates collapses a package repeated under one signature", { + candidates <- list( + list(package = "rstan", signature = "tbb-stddef-removed"), + list(package = "rstan", signature = "tbb-stddef-removed") + ) + out <- dedupe_candidates(candidates) + expect_length(out$keep, 1L) + expect_length(out$ambiguous, 0L) +}) + +test_that("dedupe_candidates handles the empty list", { + out <- dedupe_candidates(list()) + expect_length(out$keep, 0L) + expect_length(out$ambiguous, 0L) +}) + test_that("signature_hit_rate splits addressed vs open per signature", { report <- build_triage_report(mk_failures(), registered_pkgs = "RcppParallel") hit <- signature_hit_rate(report, registered_pkgs = "RcppParallel") From 9bc49731591c5d506ecf894364f61e3853723742 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 14 Jul 2026 18:40:41 +0000 Subject: [PATCH 067/125] fix(local): stop proposing per-dependent patches for dependency-cascade failures (#121) ## Problem The scheduled `--open-issue` run (issue #120) posted **800+** proposed entries, all `rcppparallel-bundled-tbb` on `ubuntu-2604`, each pre-filling `RcppParallel/disable-tbb.patch` for an unrelated package (`ACEsimFit`, `AovBay`, `AdaptGauss`, ...). Root cause: those packages fail on ubuntu-2604 only because their **RcppParallel dependency** does not build there. bincraft records RcppParallel's TBB error text against each dependent, so every one of them matches the `rcppparallel-bundled-tbb` signature. Proposing a RcppParallel source patch for `ACEsimFit` is not just noise -- it is **broken**: the diff targets RcppParallel's `Makevars.in`, so it cannot apply to a dependent's source. ## Fix Add an optional `applies_to` field that pins a package-specific fix to the package it targets: - A signature whose fix is a curated per-package source patch (`rcppparallel-bundled-tbb` -> `RcppParallel`) is only ever proposed **for that package**. - Other packages matching the signature are downstream failures, reported as **"blocked on ``"** and never proposed a bogus entry. Fix the dependency once and the whole cascade clears. Generic levers (the `tbb-stddef-removed` makevars fix) are unaffected -- they carry no `applies_to`, so they still apply to any matching package. ## Changes - gate proposal generation in `build_triage_report()` on `applies_to`; add a `blocked_on` field to group records - surface blocked groups in both `failing-builds-report.R` and `propose-patches.R` instead of the misleading "already registered" note - document `applies_to` in the signature-table header ## Verification - New tests: RcppParallel dependents are blocked (no `proposed_entries`, `blocked_on = "RcppParallel"`), while RcppParallel *itself* is still proposed when it is the failing package. - Reproduced the avalanche end-to-end with a stubbed DB: 5 dependents now report "Blocked on RcppParallel: 5 package(s) fail because RcppParallel does not build" and **zero** candidates are emitted; a genuine `tbb-stddef` proposal in the same run is unaffected. - Full suite: 84 tests pass; all pre-commit hooks pass. Refs #120. Note: the underlying build problem (RcppParallel failing on ubuntu-2604 despite its registry entry) is real and separate -- this PR stops the classifier from spamming broken per-dependent proposals about it. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/121 --- local/failing-builds-classify.R | 31 ++++++++++++++++++- local/failing-builds-report.R | 7 +++++ local/propose-patches.R | 15 ++++++++++ local/tests/test-failing-builds-classify.R | 35 ++++++++++++++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index 30c168d..4ce30cf 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -15,6 +15,11 @@ # would require a brand-new source diff for a previously-unseen package stay # `auto = FALSE` -> classified, but always routed to human triage, per the # issue's guardrail against shipping autonomous novel source diffs. +# `applies_to` (optional) pins a package-specific fix to the package it targets: +# a signature whose registry patch belongs to one package (e.g. RcppParallel's +# disable-tbb patch) is only proposed for that package. Other packages matching +# the signature are downstream failures blocked on that dependency, not +# individually patchable, so they are reported but never proposed an entry. # # Seeded from the existing registry entries and known recurring failures; add # a row here as new signatures are confirmed. Order matters: the first match @@ -45,6 +50,14 @@ build_signatures <- function() { tier = "patch", confidence = "high", auto = TRUE, + # The fix is a RcppParallel-specific source patch, so it is only ever + # proposed for RcppParallel itself. The hundreds of packages that fail + # merely because their RcppParallel *dependency* did not build carry + # RcppParallel's error text and match this signature too -- they are + # blocked on RcppParallel, not individually patchable (applying this diff + # to their source is meaningless). `applies_to` pins the proposal so those + # downstream failures are never proposed a bogus per-package entry. + applies_to = "RcppParallel", fix = "apply the curated RcppParallel/disable-tbb.patch so the bundled TBB build is skipped and the TinyThread backend is used", example = "RcppParallel", registry = list( @@ -278,9 +291,23 @@ build_triage_report <- function( unregistered <- setdiff(pkgs, registered_pkgs) auto_proposable <- isTRUE(sig$auto) && sig$matched + # A signature whose fix is package-specific (`applies_to`) may only be + # proposed for that package. Everything else matching it is a downstream + # failure blocked on that package (e.g. RcppParallel dependents carrying + # RcppParallel's own error text) -- never propose those a bogus entry. + proposable <- unregistered + blocked_on <- NULL + if (!is.null(sig$applies_to)) { + proposable <- intersect(unregistered, sig$applies_to) + downstream <- setdiff(pkgs, sig$applies_to) + if (length(downstream) > 0L) { + blocked_on <- sig$applies_to + } + } + proposed <- list() if (auto_proposable) { - for (p in unregistered) { + for (p in proposable) { proposed[[p]] <- propose_registry_entry_list( sig, p, @@ -297,6 +324,8 @@ build_triage_report <- function( tier = sig$tier, confidence = sig$confidence, suggested_fix = sig$fix, + applies_to = sig$applies_to, + blocked_on = blocked_on, fingerprint = names(fp_tab)[[1L]], fingerprint_variants = length(fp_tab), build_count = nrow(g), diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R index 60f65ea..51f5bc7 100644 --- a/local/failing-builds-report.R +++ b/local/failing-builds-report.R @@ -194,6 +194,13 @@ for (r in report) { ) cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "") } + } else if (!is.null(r$blocked_on)) { + cat(sprintf( + " (blocked on %s -- these %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", + toString(r$blocked_on), + length(r$packages), + toString(r$blocked_on) + )) } else if (r$auto_proposable) { cat(" (all affected packages already have a registry entry)\n") } diff --git a/local/propose-patches.R b/local/propose-patches.R index d43668a..7ff195b 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -135,6 +135,21 @@ for (r in report) { } } +# Groups blocked on a dependency (e.g. RcppParallel dependents) are reported, +# not proposed: fixing the named dependency clears them all at once. +blocked <- Filter(function(r) !is.null(r$blocked_on), report) +if (length(blocked) > 0L) { + cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") + for (r in blocked) { + cat(sprintf( + " %s: %d package(s) fail because %s does not build\n", + toString(r$blocked_on), + length(r$packages), + toString(r$blocked_on) + )) + } +} + # A package that maps to more than one auto-proposable signature is ambiguous # (conflicting fix tiers) and would collide on the same registry key; route it # to human triage instead of emitting both. diff --git a/local/tests/test-failing-builds-classify.R b/local/tests/test-failing-builds-classify.R index 66cd26b..a365e14 100644 --- a/local/tests/test-failing-builds-classify.R +++ b/local/tests/test-failing-builds-classify.R @@ -85,3 +85,38 @@ test_that("propose_registry_entry fills a schema-valid entry for a known lever", # No template -> no proposal (unclassified path). expect_null(propose_registry_entry(classify_error("weird"), "x", "alpine")) }) + +test_that("RcppParallel dependents are blocked, not proposed a per-package patch", { + # Hundreds of packages fail on ubuntu-2604 only because their RcppParallel + # dependency fails to build, so they carry RcppParallel's TBB error text. + failures <- data.frame( + name = c("ACEsimFit", "AovBay", "AdaptGauss"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + grp <- Filter(function(g) g$signature == "rcppparallel-bundled-tbb", report)[[ + 1L + ]] + # No bogus per-dependent entries (the patch targets RcppParallel's source). + expect_null(grp$proposed_entries) + expect_identical(grp$blocked_on, "RcppParallel") +}) + +test_that("RcppParallel itself is still proposed when it is the failing package", { + failures <- data.frame( + name = "RcppParallel", + platform = "ubuntu-2604", + arch = "amd64", + error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + grp <- report[[1L]] + expect_identical(grp$signature, "rcppparallel-bundled-tbb") + expect_false(is.null(grp$proposed_entries)) + expect_true("RcppParallel" %in% names(grp$proposed_entries)) + expect_null(grp$blocked_on) # RcppParallel is the target, not a dependent +}) From 21a2fe9e6cfe93efcd653c461e5fb0fac9fc1d49 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 15 Jul 2026 07:46:29 +0000 Subject: [PATCH 068/125] feat(local): report unclassified and dependency-blocked failures for discovery (#122) ## Why Issue #120 (the auto-proposed-patches issue) only lists **auto-proposable** fixes -- currently just the TBB signatures. So a reasonable read of it was "TBB is our only build failure", when in fact three whole categories are simply not shown there: - **Unclassified failures** -- anything that doesn't match a seeded signature is routed to human triage and never appears (we've only seeded TBB and libuv signatures). - **Dependency-blocked failures** -- the ~800 RcppParallel dependents (post #121) are still failing; they only show as a log line. - Human-only signatures (libuv). These blind spots are exactly where the *next* signatures should come from, so they deserve the same visibility as the proposals. ## What Extend the feedback-loop tracker to surface the classifier's blind spots: - **`unclassified_summary()`** -- groups every unknown-signature failure by normalised fingerprint, ranked by build count, capped with an explicit `dropped_groups` count (no silent truncation), each with example packages + platforms. These are the candidates for new `build_signatures()` rules. - **`blocked_summary()`** -- lists each dependency (e.g. RcppParallel) and how many dependents wait on it. - `proposal-tracking.R` prints both sections, and a new **`--open-issue`** mode posts/updates a *"Unclassified build failures (needs signatures) (#115)"* Forgejo issue. - The weekly crow pipeline now runs the tracker with `--open-issue`, so it maintains a second tracking issue alongside the proposals one. Read-only on the DB; the only writes are the two issues. ## Verification - New tests cover `unclassified_summary` (ranking + both caps) and `blocked_summary`. - Tracker smoke with a stubbed DB (proposable + blocked + unclassified mix) prints the hit rate, `Blocked on a dependency: RcppParallel: 2 dependent(s)`, and `Unclassified failures ... [2 builds | 2 pkgs] ld: undefined reference ...`. - Full suite: 95 tests pass; all pre-commit hooks pass (air, prettier, markdownlint, yamllint, validate-patches). Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/122 --- .crow/weekly-patch-proposals.yaml | 15 +- local/patches/README.md | 12 +- local/proposal-tracking-lib.R | 44 ++++++ local/proposal-tracking.R | 179 ++++++++++++++++++++++- local/tests/test-proposal-tracking-lib.R | 43 ++++++ 5 files changed, 278 insertions(+), 15 deletions(-) diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index f956f3f..0284e93 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -1,12 +1,13 @@ # Weekly failure-triage proposals (issue #115, steps 3 + 4). # Classifies the recorded `single_builds` failures and: -# 1. posts/updates a Forgejo tracking issue with the auto-proposable registry -# entries (human reviews, trial-builds, and opens the PR -- nothing merges), and -# 2. prints the feedback-loop metrics (signature hit rate, proposed-vs-merged, -# retirement candidates) to the run log. +# 1. posts/updates a Forgejo issue with the auto-proposable registry entries +# (human reviews, trial-builds, and opens the PR -- nothing merges), +# 2. posts/updates a second Forgejo issue with the classifier's blind spots +# (unclassified failures = candidates for new signatures, and groups +# blocked on a dependency), and prints the feedback-loop metrics to the log. # Global across platforms (the classifier groups over all of single_builds), so -# a single job -- no matrix. Clones read-only; the only write is the Forgejo -# issue via FORGEJO_TOKEN. +# a single job -- no matrix. Clones read-only; the only writes are the two +# Forgejo issues via FORGEJO_TOKEN. when: - event: manual - event: cron @@ -36,7 +37,7 @@ steps: - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R + - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: kubernetes: resources: diff --git a/local/patches/README.md b/local/patches/README.md index e875217..992395f 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -86,15 +86,21 @@ Rscript local/trial-build-patch.R ### Feedback loop (step 4) `local/proposal-tracking.R` reports the signature hit rate, proposed-vs-merged status (a proposal counts as merged once its package appears in the registry), and retirement candidates (registry entries whose package no longer appears in any current failure, so the upstream cause was likely fixed). -It is read-only. +It also surfaces the classifier's **blind spots**: the unclassified failures (candidates for a new signature) and the groups blocked on a dependency build, so the unknown buckets get the same visibility as the proposals. +It is read-only on the DB, with an optional Forgejo issue as the only write. ```bash +# Print the metrics + blind spots. PGPASS=... Rscript local/proposal-tracking.R --json metrics.json +# Post/update a "needs signatures" tracking issue with the unclassified failures. +PGPASS=... FORGEJO_TOKEN=... Rscript local/proposal-tracking.R --open-issue ``` +The unclassified groups are the natural place to discover which new signatures are worth adding to `build_signatures()`. The pure metric/ledger helpers live in `local/proposal-tracking-lib.R` and are covered by `local/tests/test-proposal-tracking-lib.R`. ### Scheduled run -`.crow/weekly-patch-proposals.yaml` runs both steps weekly (register the `weekly-patch-proposals` cron in the crow UI): it posts/updates a Forgejo tracking issue with the auto-proposable entries and logs the feedback-loop metrics. -It clones read-only; the only write is the tracking issue. +`.crow/weekly-patch-proposals.yaml` runs both steps weekly (register the `weekly-patch-proposals` cron in the crow UI). +It posts/updates two Forgejo issues -- one with the auto-proposable entries, one with the unclassified/blocked failures -- and logs the feedback-loop metrics. +It clones read-only; the only writes are the two tracking issues. diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 2edd9c8..5efc87c 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -131,4 +131,48 @@ retirement_candidates <- function(registry_entries, failing_pkgs) { vapply(keep, function(e) as.character(e$package), character(1L)) } +# Discovery view: the failure groups the classifier could NOT auto-propose, so +# they get the same visibility as proposals. `report` is a build_triage_report +# result. Returns the unclassified groups (unknown signature -> candidates for a +# new signature) ranked by build count, capped to `max_groups`, each with up to +# `max_pkgs` example packages. `dropped_groups`/`packages_truncated` record any +# cap so nothing is silently hidden. +unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) { + un <- Filter(function(g) !isTRUE(g$matched), report) + un <- un[order(-vapply(un, function(g) g$build_count, integer(1L)))] + shown <- utils::head(un, max_groups) + groups <- lapply(shown, function(g) { + pkgs <- g$packages + list( + fingerprint = g$fingerprint, + build_count = g$build_count, + n_packages = length(pkgs), + packages = utils::head(pkgs, max_pkgs), + packages_truncated = length(pkgs) > max_pkgs, + platforms = g$platforms + ) + }) + list( + groups = groups, + total_groups = length(un), + dropped_groups = max(0L, length(un) - length(shown)), + total_builds = sum(vapply(un, function(g) g$build_count, integer(1L))) + ) +} + +# Groups blocked on a dependency (a package-specific fix pinned via `applies_to` +# whose dependents merely carry its error): report the dependency + how many +# dependents wait on it, so fixing it once is recognised as clearing the batch. +blocked_summary <- function(report, max_pkgs = 15L) { + bl <- Filter(function(g) !is.null(g$blocked_on), report) + lapply(bl, function(g) { + list( + blocked_on = g$blocked_on, + n_packages = length(g$packages), + packages = utils::head(g$packages, max_pkgs), + packages_truncated = length(g$packages) > max_pkgs + ) + }) +} + `%||%` <- function(a, b) if (is.null(a)) b else a diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R index 60bbd10..4d82a11 100644 --- a/local/proposal-tracking.R +++ b/local/proposal-tracking.R @@ -1,12 +1,16 @@ #!/usr/bin/env Rscript # Feedback loop for the failure classifier (issue #115, step 4): report the -# signature hit rate, proposed-vs-merged status, and registry entries that look -# retirable, so the rule set can improve and stale entries can be pruned. -# Read-only: queries `single_builds` and reads the registry + proposals ledger. +# signature hit rate, proposed-vs-merged status, registry entries that look +# retirable, and -- so the classifier's blind spots get the same visibility as +# its proposals -- the failures it could NOT classify (candidates for new +# signatures) plus the groups blocked on a dependency build. +# Read-only on the DB/registry; the only optional write is the Forgejo issue. # # Usage: -# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] +# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] [--open-issue] +# --open-issue post/update a Forgejo issue listing the unclassified and +# dependency-blocked failures (needs FORGEJO_TOKEN) options(error = function() { cat("ERROR:", geterrmessage(), "\n", file = stdout()) @@ -33,6 +37,7 @@ json_out <- local({ i <- match("--json", args) if (!is.na(i) && i < length(args)) args[[i + 1L]] else NA_character_ }) +do_issue <- "--open-issue" %in% args if (nchar(Sys.getenv("PGPASS")) == 0L) { stop("PGPASS env var is not set; a DB password is required.") @@ -119,12 +124,52 @@ if (length(retire) > 0L) { cat(" (none)\n") } +# --------------------------------------------------------------------------- +# Blind spots: failures the classifier could not auto-propose. +# --------------------------------------------------------------------------- +blocked <- blocked_summary(report) +unmatched <- unclassified_summary(report) + +cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") +if (length(blocked) > 0L) { + for (b in blocked) { + cat(sprintf(" %s: %d dependent(s) waiting\n", b$blocked_on, b$n_packages)) + } +} else { + cat(" (none)\n") +} + +cat(sprintf( + "\nUnclassified failures (candidates for new signatures): %d group(s), %d builds.\n", + unmatched$total_groups, + unmatched$total_builds +)) +cat(strrep("-", 60L), "\n", sep = "") +for (g in unmatched$groups) { + cat(sprintf( + " [%d builds | %d pkgs] %s\n e.g. %s%s\n", + g$build_count, + g$n_packages, + g$fingerprint, + toString(g$packages), + if (isTRUE(g$packages_truncated)) ", ..." else "" + )) +} +if (unmatched$dropped_groups > 0L) { + cat(sprintf( + " (+%d more unclassified group(s) not shown)\n", + unmatched$dropped_groups + )) +} + if (!is.na(json_out)) { jsonlite::write_json( list( signature_hit_rate = hit, proposed_vs_merged = pvm, - retirement_candidates = retire + retirement_candidates = retire, + blocked = blocked, + unclassified = unmatched ), json_out, auto_unbox = TRUE, @@ -133,3 +178,127 @@ if (!is.na(json_out)) { ) cat(sprintf("\nWrote metrics to %s\n", json_out)) } + +# --------------------------------------------------------------------------- +# Optionally publish the blind-spots to a Forgejo tracking issue. +# --------------------------------------------------------------------------- +if (do_issue) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-issue requires FORGEJO_TOKEN.") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + issue_title <- "Unclassified build failures (needs signatures) (#115)" + now <- format(Sys.time(), "%Y-%m-%d %H:%M:%S") + + body_lines <- c( + sprintf("_Generated %s from `single_builds` failures._", now), + "", + "Failures the classifier could **not** auto-propose a fix for.", + "Each unclassified group is a candidate for a new signature in `local/failing-builds-classify.R`; the blocked groups clear once the named dependency builds.", + "" + ) + + body_lines <- c(body_lines, "## Blocked on a dependency", "") + if (length(blocked) > 0L) { + for (b in blocked) { + body_lines <- c( + body_lines, + sprintf( + "- **%s**: %d dependent(s) waiting (e.g. %s%s)", + b$blocked_on, + b$n_packages, + toString(b$packages), + if (isTRUE(b$packages_truncated)) ", ..." else "" + ) + ) + } + } else { + body_lines <- c(body_lines, "_None._") + } + + body_lines <- c( + body_lines, + "", + sprintf( + "## Unclassified failures (%d groups, %d builds)", + unmatched$total_groups, + unmatched$total_builds + ), + "" + ) + if (length(unmatched$groups) > 0L) { + for (g in unmatched$groups) { + body_lines <- c( + body_lines, + sprintf( + "### %d builds / %d pkg(s)", + g$build_count, + g$n_packages + ), + "", + sprintf("Fingerprint: `%s`", g$fingerprint), + sprintf( + "Packages: %s%s", + toString(g$packages), + if (isTRUE(g$packages_truncated)) ", ..." else "" + ), + sprintf("Platforms: %s", toString(g$platforms)), + "" + ) + } + if (unmatched$dropped_groups > 0L) { + body_lines <- c( + body_lines, + sprintf("_(+%d more group(s) not shown.)_", unmatched$dropped_groups) + ) + } + } else { + body_lines <- c(body_lines, "_None -- every failure is classified._") + } + new_body <- paste(body_lines, collapse = "\n") + + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(issue_title, reserved = TRUE) + ) + existing <- httr2::request(search_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + existing, + function(x) identical(x$title, issue_title), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- existing[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("\nUpdated tracking issue #%d.\n", num)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/issues", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = issue_title, body = new_body)) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf("\nOpened tracking issue #%d.\n", created$number)) + } +} diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index a2fbe05..02a29f1 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -119,6 +119,49 @@ test_that("proposed_vs_merged marks a package merged once it is registered", { expect_identical(stan$status, "merged") }) +test_that("unclassified_summary ranks unknown groups and caps output", { + failures <- data.frame( + name = c("a", "b", "c", "d", "solo"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = c( + # 4 builds share one unknown fingerprint; 1 build a different unknown. + rep("mystery linker meltdown at stage 3", 4L), + "a totally different unknown boom" + ), + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + s <- unclassified_summary(report, max_groups = 30L, max_pkgs = 2L) + expect_identical(s$total_groups, 2L) + expect_identical(s$total_builds, 5L) + # Largest group first, and its example packages are capped at max_pkgs. + expect_identical(s$groups[[1L]]$build_count, 4L) + expect_length(s$groups[[1L]]$packages, 2L) + expect_true(s$groups[[1L]]$packages_truncated) + + # max_groups cap is reported, not silently dropped. + capped <- unclassified_summary(report, max_groups = 1L) + expect_length(capped$groups, 1L) + expect_identical(capped$dropped_groups, 1L) +}) + +test_that("blocked_summary lists each dependency and its dependent count", { + failures <- data.frame( + name = c("ACEsimFit", "AovBay", "AdaptGauss"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + b <- blocked_summary(report, max_pkgs = 2L) + expect_length(b, 1L) + expect_identical(b[[1L]]$blocked_on, "RcppParallel") + expect_identical(b[[1L]]$n_packages, 3L) + expect_true(b[[1L]]$packages_truncated) +}) + test_that("retirement_candidates flags entries whose package no longer fails", { entries <- list( list(package = "RcppParallel"), From 4bca17e4acae7abea58e478ac56f1725b90cb5dc Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 15 Jul 2026 08:28:15 +0000 Subject: [PATCH 069/125] feat(local): auto-apply registry patches with a build-env trial-build gate (#124) Closes the classifier loop (issue #115, step 3): turns the auto-proposable candidates into an actual PR, gated by a real trial build **in our own build-env images**. Chosen model (from the design discussion): **autonomous PR**, **PR-first with a CI trial-build gate**, **bounded top-N batch** per run. ## Creating the patch PR - `propose-patches.R` gains: - `--limit N` -- act on the top-N candidates by failure volume; the rest defer to the next run (logged). - `--open-pr` -- write the entries onto the reused `auto/registry-patch-proposals` branch, push (with `REPO_RW_TOKEN`), and open/update **one** PR via the Forgejo API (so re-runs update the same PR instead of piling up). - `.crow/auto-apply-patches.yaml` -- a single job that runs `--open-pr --limit` on a cron/manual trigger. Needs `FORGEJO_TOKEN` + a write-scoped `REPO_RW_TOKEN`. Novel source diffs and unknown signatures are still never proposed; nothing merges. ## The merge gate (our build-env images) - `.crow/trial-build-registry.yaml` -- matrixed over the real `OS/IMG` build-env matrix (alpine:3.24, redhat:8/9/10, ubuntu:jammy/noble/**resolute** for ubuntu-2604). Each platform runs `local/trial-build-registry.R`, which diffs the branch registry against `main` and trial-builds **only the entries the branch adds** that apply to that platform, inside `reg.devxy.io/rpkgs/build-env-*`. Green only if every new entry builds; a platform with no new entries is a fast no-op. Nothing is uploaded/archived/recorded. - The base-registry read **fails loud** if it can't read `registry.json` at `main`, rather than silently treating the base as empty and trial-building the whole registry. ## Notes / follow-up - The repo uses **no `pull_request` triggers**, so the gate runs manually or on a cron against the branch (`--var patch_branch=...`). Wiring it to fire automatically on the PR needs `event: pull_request` enabled on the Forgejo webhook -- a one-line addition once that's on. - Two new crons to register in the crow UI: `auto-apply-patches` and `trial-build-registry`. New secret needed: `REPO_RW_TOKEN` (write scope) for the push. ## Verification - New pure helpers `entry_applies_to_os()` / `new_registry_packages()` covered by tests (platform codename/family/wildcard matching; added-vs-unchanged entry detection; per-platform filtering). - `--limit` smoke (stubbed DB): top-2 by volume proposed, 3 deferred, candidate registry validates. - Full suite: 105 tests pass; all pre-commit hooks pass (air, prettier, markdownlint, yamllint, validate-patches). Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/124 --- .crow/auto-apply-patches.yaml | 65 ++++++++ .crow/trial-build-registry.yaml | 140 +++++++++++++++++ local/patches/README.md | 13 ++ local/proposal-tracking-lib.R | 33 ++++ local/propose-patches.R | 188 ++++++++++++++++++++++- local/tests/test-proposal-tracking-lib.R | 51 ++++++ local/trial-build-registry.R | 145 +++++++++++++++++ 7 files changed, 633 insertions(+), 2 deletions(-) create mode 100644 .crow/auto-apply-patches.yaml create mode 100644 .crow/trial-build-registry.yaml create mode 100644 local/trial-build-registry.R diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml new file mode 100644 index 0000000..15cc1ee --- /dev/null +++ b/.crow/auto-apply-patches.yaml @@ -0,0 +1,65 @@ +# Auto-apply registry patches (issue #115, step 3 automation). +# Classifies `single_builds` failures and, for the top-N auto-proposable +# candidates by failure volume, writes the registry entries onto the reused +# `auto/registry-patch-proposals` branch and opens/updates a single PR. +# Nothing merges: the `trial-build-registry` pipeline is the merge gate, and a +# human reviews the PR. Novel source diffs / unknown signatures are never +# proposed. Global across platforms, so a single job -- no matrix. +# +# Needs a write token (REPO_RW_TOKEN) to push and FORGEJO_TOKEN to open the PR. +# Register the `auto-apply-patches` cron in the crow UI, or run manually: +# woodpecker-cli pipeline create --var task=auto-apply-patches --branch=main 7 +variables: + patch_limit: + description: 'Max candidates to propose per run (top by failure volume).' + default: '10' + +when: + - event: manual + evaluate: 'task == "auto-apply-patches"' + - event: cron + cron: auto-apply-patches + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Auto-apply registry patches' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + REPO_RW_TOKEN: + from_secret: REPO_RW_TOKEN + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + GIT_USER: devxy-bot + GIT_EMAIL: bot@devxy.io + PATCH_LIMIT: ${patch_limit} + R_VERSION: 4.5.3 + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 2000m + limits: + memory: 2Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml new file mode 100644 index 0000000..1ac71e8 --- /dev/null +++ b/.crow/trial-build-registry.yaml @@ -0,0 +1,140 @@ +# Merge gate for the auto-patch PR (issue #115, step 3). +# For each platform, trial-builds every registry entry the auto-patch branch +# ADDS (vs main) in that platform's own `reg.devxy.io/rpkgs/build-env-*` image, +# with the registry applied. A row with no new entries for its platform is a +# fast no-op. The pipeline is green only if every new entry builds, so it gates +# the PR before merge. Nothing is uploaded/archived/recorded. +# +# The repo uses no `pull_request` triggers, so this runs manually against the +# branch (or on a cron); point it at the auto-patch branch via `patch_branch`: +# woodpecker-cli pipeline create --var task=trial-build-registry \ +# --var patch_branch=auto/registry-patch-proposals --branch=main 7 +variables: + patch_branch: + description: 'Branch whose new registry entries to trial-build.' + default: auto/registry-patch-proposals + +when: + - event: manual + evaluate: 'task == "trial-build-registry"' + - event: cron + cron: trial-build-registry + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + +steps: + - name: 'Trial-build new registry entries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + PLATFORM: ${OS} + ARCH: ${ARCH} + R_VERSION: ${R_VERSION} + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q --branch ${patch_branch} https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - git fetch -q origin main + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/Rscript local/trial-build-registry.R origin/main + backend_options: + kubernetes: + resources: + requests: + memory: 2Gi + cpu: 2000m + limits: + memory: 4Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/local/patches/README.md b/local/patches/README.md index 992395f..455a96c 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -81,6 +81,19 @@ Run the trial build inside the failing platform's build-env image; it uploads/ar Rscript local/trial-build-patch.R ``` +#### Autonomous PR + trial-build gate + +`--open-pr` closes the loop: it writes the top-N candidates (by failure volume) onto the reused `auto/registry-patch-proposals` branch and opens/updates a single PR. +`.crow/auto-apply-patches.yaml` runs this on a cron (needs `FORGEJO_TOKEN` and a write-scoped `REPO_RW_TOKEN`). + +```bash +# Bounded batch; opens/updates one PR. +PGPASS=... FORGEJO_TOKEN=... REPO_RW_TOKEN=... Rscript local/propose-patches.R --open-pr --limit 10 +``` + +The merge gate is `.crow/trial-build-registry.yaml`: matrixed over the build-env images, each platform trial-builds only the entries the branch **adds** (`local/trial-build-registry.R`, which diffs the registry against `main`) and is green only if every new entry builds. +The repo uses no `pull_request` triggers, so this gate runs manually or on a cron against the auto-patch branch (`--var patch_branch=...`); wiring it to fire on the PR needs `event: pull_request` enabled on the forge. + `--write` and `--open-issue` also append to `local/patches/proposals-log.json`, a ledger of what was proposed. ### Feedback loop (step 4) diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 5efc87c..80c4ab2 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -175,4 +175,37 @@ blocked_summary <- function(report, max_pkgs = 15L) { }) } +# Does a registry entry's `platforms` apply to a build on `os` (e.g. +# "ubuntu-2604")? Mirrors bincraft's token match: an entry applies if any of its +# platform tokens is "*", the OS codename, or the distro family ("ubuntu"). +entry_applies_to_os <- function(entry_platforms, os) { + toks <- as.character(unlist(entry_platforms)) + family <- sub("-.*$", "", os) # ubuntu-2604 -> ubuntu + any(toks %in% c("*", os, family)) +} + +# Registry entries present in `current` but not in `base` (matched on +# package|platforms|versions), optionally restricted to those that apply to a +# given `os`. Used by the trial-build gate to build only the entries a PR adds. +new_registry_packages <- function(current, base, os = NULL) { + key <- function(e) { + sprintf( + "%s|%s|%s", + e$package %||% "?", + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + e$versions %||% "?" + ) + } + base_keys <- vapply(base %||% list(), key, character(1L)) + added <- Filter(function(e) !(key(e) %in% base_keys), current %||% list()) + if (!is.null(os)) { + added <- Filter(function(e) entry_applies_to_os(e$platforms, os), added) + } + unique(vapply( + added, + function(e) as.character(e$package %||% ""), + character(1L) + )) +} + `%||%` <- function(a, b) if (is.null(a)) b else a diff --git a/local/propose-patches.R b/local/propose-patches.R index 7ff195b..fb45ea9 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -15,10 +15,17 @@ # # Usage: # PGPASS=... Rscript local/propose-patches.R [--platform P] [--arch A] [--min N] +# [--limit N] # (default) print candidates + validation, take no action # --write append candidates to local/patches/registry.json and # the proposals ledger (commit + open a PR yourself) # --open-issue post/update a Forgejo tracking issue (needs FORGEJO_TOKEN) +# --open-pr write the entries, push the `auto/registry-patch-proposals` +# branch, and open/update a PR autonomously (needs +# FORGEJO_TOKEN, and REPO_RW_TOKEN to push in CI). The +# `trial-build-registry` pipeline is the merge gate. +# --limit N only act on the top-N candidates by failure volume +# (bounded batch; the rest are picked up on the next run) # --json PATH also write the machine-readable candidate list to PATH options(error = function() { @@ -58,6 +65,8 @@ if (is.na(min_count)) { } do_write <- "--write" %in% args do_issue <- "--open-issue" %in% args +do_pr <- "--open-pr" %in% args +limit <- suppressWarnings(as.integer(opt_val("--limit", NA_character_))) if (nchar(Sys.getenv("PGPASS")) == 0L) { stop("PGPASS env var is not set; a DB password is required.") @@ -65,6 +74,8 @@ if (nchar(Sys.getenv("PGPASS")) == 0L) { registry_file <- file.path(script_dir, "patches", "registry.json") ledger_file <- file.path(script_dir, "patches", "proposals-log.json") +# Branch the autonomous PR reuses, so re-runs update one PR instead of piling up. +pr_branch <- "auto/registry-patch-proposals" # --------------------------------------------------------------------------- # Query failing builds (same shape as failing-builds-report.R) @@ -118,7 +129,8 @@ registered_pkgs <- unique(vapply( report <- build_triage_report(failures, registered_pkgs) report <- Filter(function(r) r$build_count >= min_count, report) -# Flatten auto-proposable groups into candidate records. +# Flatten auto-proposable groups into candidate records, carrying the group's +# build volume so we can prioritise the highest-impact fixes. candidates <- list() for (r in report) { if (is.null(r$proposed_entries)) { @@ -130,6 +142,7 @@ for (r in report) { signature = r$signature, confidence = r$confidence, tier = r$tier, + build_count = r$build_count, entry = r$proposed_entries[[pkg]] ) } @@ -173,6 +186,22 @@ if (length(candidates) == 0L) { q(status = 0) } +# Prioritise by failure volume, then apply --limit so one run tackles a bounded +# batch (the rest are picked up on the next run). +candidates <- candidates[order( + -vapply(candidates, function(c) c$build_count %||% 0L, integer(1L)) +)] +deferred <- 0L +if (!is.na(limit) && limit >= 0L && length(candidates) > limit) { + deferred <- length(candidates) - limit + candidates <- utils::head(candidates, limit) + cat(sprintf( + "\nLimiting to top %d candidate(s) by failure volume; %d deferred to a later run.\n", + limit, + deferred + )) +} + cat(sprintf( "\n%d candidate registry %s:\n", length(candidates), @@ -386,8 +415,163 @@ if (do_write) { cat(sprintf("Opened tracking issue #%d.\n", created$number)) } save_ledger(merge_ledger(load_ledger(), new_ledger_records)) +} else if (do_pr) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-pr requires FORGEJO_TOKEN (to open the PR).") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + + # Write the entries + ledger, then commit them onto the reused auto branch. + jsonlite::write_json( + merged, + registry_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) + + git <- function(...) { + st <- system2("git", c(...), stdout = TRUE, stderr = TRUE) + if (!identical(attr(st, "status"), NULL)) { + stop(sprintf( + "git %s failed:\n%s", + paste(..., collapse = " "), + paste(st, collapse = "\n") + )) + } + invisible(st) + } + git("config", "user.name", Sys.getenv("GIT_USER", "devxy-bot")) + git( + "config", + "user.email", + Sys.getenv("GIT_EMAIL", "bot@devxy.io") + ) + git("checkout", "-B", pr_branch) + git( + "add", + file.path(script_dir, "patches", "registry.json"), + ledger_file + ) + git( + "commit", + "-m", + sprintf( + "feat(patches): auto-propose %d registry %s from classified failures", + length(candidate_entries), + if (length(candidate_entries) == 1L) "entry" else "entries" + ) + ) + # Push with a write token when provided (CI); otherwise rely on origin creds. + rw_token <- Sys.getenv("REPO_RW_TOKEN") + push_target <- if (nchar(rw_token) > 0L) { + sprintf( + "https://%s:%s@git.devxy.io/%s.git", + Sys.getenv("GIT_USER", "devxy-bot"), + rw_token, + repo + ) + } else { + "origin" + } + git("push", "-f", push_target, sprintf("HEAD:refs/heads/%s", pr_branch)) + + pr_title <- sprintf( + "feat(patches): auto-proposed registry patches (%s)", + now + ) + body_lines <- c( + sprintf( + "_Auto-generated %s by `local/propose-patches.R --open-pr` from classified `single_builds` failures._", + now + ), + "", + sprintf( + "Adds %d known-lever registry %s (top by failure volume%s).", + length(candidate_entries), + if (length(candidate_entries) == 1L) "entry" else "entries", + if (deferred > 0L) { + sprintf("; %d deferred to a later run", deferred) + } else { + "" + } + ), + "", + "**Merge gate:** the `trial-build-registry` pipeline builds each new entry in its target build-env image; merge only once it is green.", + "Novel source diffs and unknown signatures are never auto-proposed.", + "", + "| package | signature | platforms |", + "| --- | --- | --- |" + ) + for (c in candidates) { + body_lines <- c( + body_lines, + sprintf( + "| %s | %s | %s |", + c$package, + c$signature, + toString(as.character(c$entry$platforms)) + ) + ) + } + new_body <- paste(body_lines, collapse = "\n") + + # One PR per reused branch: update if open, else create. + pulls_url <- sprintf( + "%s/repos/%s/pulls?state=open&limit=50", + forgejo_base, + repo + ) + open_pulls <- httr2::request(pulls_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + open_pulls, + function(p) identical(p$head$ref, pr_branch), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- open_pulls[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/pulls/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = pr_title, body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("\nUpdated auto-patch PR #%d (branch %s).\n", num, pr_branch)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/pulls", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list( + title = pr_title, + head = pr_branch, + base = "main", + body = new_body + )) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf( + "\nOpened auto-patch PR #%d (branch %s).\n", + created$number, + pr_branch + )) + } } else { cat( - "\nDry run: no changes made. Re-run with --write or --open-issue to act.\n" + "\nDry run: no changes made. Re-run with --write, --open-issue, or --open-pr to act.\n" ) } diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index 02a29f1..05cd0e9 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -162,6 +162,57 @@ test_that("blocked_summary lists each dependency and its dependent count", { expect_true(b[[1L]]$packages_truncated) }) +test_that("entry_applies_to_os matches codename, family, and wildcard", { + expect_true(entry_applies_to_os(list("ubuntu-2604"), "ubuntu-2604")) + expect_true(entry_applies_to_os(list("ubuntu"), "ubuntu-2604")) # family + expect_true(entry_applies_to_os(list("*"), "ubuntu-2604")) + expect_true(entry_applies_to_os(list("alpine", "ubuntu-2604"), "ubuntu-2604")) + expect_false(entry_applies_to_os(list("alpine-324"), "ubuntu-2604")) + expect_false(entry_applies_to_os(list("ubuntu-2404"), "ubuntu-2604")) # other codename +}) + +test_that("new_registry_packages returns only added entries for the platform", { + base <- list( + list( + package = "RcppParallel", + platforms = list("alpine", "ubuntu-2604"), + versions = "*" + ) + ) + current <- list( + base[[1L]], # unchanged -> not "new" + list(package = "BFpack", platforms = list("ubuntu-2604"), versions = "*"), + list( + package = "someAlpinePkg", + platforms = list("alpine-324"), + versions = "*" + ) + ) + # For ubuntu-2604: only the newly-added BFpack (RcppParallel is unchanged, + # someAlpinePkg does not apply to this OS). + expect_identical( + new_registry_packages(current, base, os = "ubuntu-2604"), + "BFpack" + ) + # For alpine-324: the alpine package is new and applies. + expect_identical( + new_registry_packages(current, base, os = "alpine-324"), + "someAlpinePkg" + ) + # Without an OS filter, both additions are returned. + expect_setequal( + new_registry_packages(current, base), + c("BFpack", "someAlpinePkg") + ) + # A changed platform set on the same package counts as a new entry. + widened <- list(list( + package = "RcppParallel", + platforms = list("*"), + versions = "*" + )) + expect_identical(new_registry_packages(widened, base), "RcppParallel") +}) + test_that("retirement_candidates flags entries whose package no longer fails", { entries <- list( list(package = "RcppParallel"), diff --git a/local/trial-build-registry.R b/local/trial-build-registry.R new file mode 100644 index 0000000..f649b62 --- /dev/null +++ b/local/trial-build-registry.R @@ -0,0 +1,145 @@ +#!/usr/bin/env Rscript + +# Merge gate for the auto-patch PR (issue #115, step 3): for every registry +# entry the PR ADDS that applies to this platform, trial-build the package with +# the registry applied, in this platform's own build-env image. Nothing is +# uploaded, archived, or written to the metadata DB. +# +# Exit 0 only if every new entry's package builds; exit 1 if any fails, so it +# gates the PR. A platform with no new entries is a fast no-op. +# +# Usage (inside a build-env image): +# PLATFORM=ubuntu-2604 Rscript local/trial-build-registry.R [base_ref] +# base_ref git ref to diff the registry against (default: origin/main) + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(jsonlite, quietly = TRUE) + library(bincraft, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +args <- commandArgs(trailingOnly = TRUE) +base_ref <- if (length(args) >= 1L) { + args[[1L]] +} else { + Sys.getenv("BASE_REF", "origin/main") +} +os <- Sys.getenv("PLATFORM", "") +if (!nzchar(os)) { + stop("PLATFORM env var is not set (e.g. ubuntu-2604).") +} + +patches_dir <- file.path(script_dir, "patches") +registry_file <- file.path(patches_dir, "registry.json") +current <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +# Read the registry at base_ref. Fail loud if the ref or file can't be read: +# silently treating the base as empty would trial-build the WHOLE registry +# instead of just the entries the branch adds. +registry_rel <- "local/patches/registry.json" +ref_ok <- suppressWarnings(system2( + "git", + c("rev-parse", "--verify", "--quiet", sprintf("%s^{commit}", base_ref)), + stdout = TRUE, + stderr = FALSE +)) +if (!is.null(attr(ref_ok, "status"))) { + stop(sprintf("base ref %s does not resolve to a commit.", base_ref)) +} +in_base <- suppressWarnings(system2( + "git", + c("ls-tree", base_ref, "--", registry_rel), + stdout = TRUE, + stderr = FALSE +)) +file_in_base <- length(in_base) > 0L && any(nzchar(in_base)) +base_json <- suppressWarnings(system2( + "git", + c("show", sprintf("%s:%s", base_ref, registry_rel)), + stdout = TRUE, + stderr = FALSE +)) +show_ok <- is.null(attr(base_json, "status")) +if (file_in_base && !show_ok) { + stop(sprintf( + "could not read %s at %s; refusing to build the whole registry.", + registry_rel, + base_ref + )) +} +base <- if (show_ok && length(base_json) > 0L) { + jsonlite::fromJSON(paste(base_json, collapse = "\n"), simplifyVector = FALSE) +} else { + list() # file genuinely absent at base -> every entry is new +} + +pkgs <- new_registry_packages(current, base, os = os) +if (length(pkgs) == 0L) { + cat(sprintf( + "No new registry entries apply to %s; nothing to trial-build.\n", + os + )) + q(status = 0) +} + +cat(sprintf( + "Trial-building %d new registry %s on %s (vs %s):\n %s\n", + length(pkgs), + if (length(pkgs) == 1L) "entry" else "entries", + os, + base_ref, + toString(pkgs) +)) + +results <- vapply( + pkgs, + function(pkg) { + cat(sprintf("\n=== trial build: %s ===\n", pkg)) + tryCatch( + { + bincraft::build_binary_package( + pkg, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ) + TRUE + }, + error = function(e) { + cat(sprintf("FAILED %s: %s\n", pkg, conditionMessage(e))) + FALSE + } + ) + }, + logical(1L) +) + +failed <- pkgs[!results] +cat(sprintf( + "\n%d/%d passed on %s.%s\n", + sum(results), + length(results), + os, + if (length(failed) > 0L) sprintf(" Failed: %s", toString(failed)) else "" +)) +if (length(failed) > 0L) { + q(status = 1) +} +q(status = 0) From 1bca6004fd7a219560e468aa7e7de66bc2aa532b Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 15 Jul 2026 14:50:19 +0000 Subject: [PATCH 070/125] chore(local): reuse FORGEJO_TOKEN for the auto-patch push (#125) Follow-up to #124 (merged): the auto-patch pipeline required a separate write-scoped `REPO_RW_TOKEN` to push the branch. Reuse the existing `FORGEJO_TOKEN` instead. - `propose-patches.R --open-pr` now pushes `auto/registry-patch-proposals` over HTTPS with `FORGEJO_TOKEN` (the same token used for the PR API); the read-only `origin` clone URL can't push, so it builds an authenticated URL explicitly. - Drop `REPO_RW_TOKEN` from `.crow/auto-apply-patches.yaml` and the docs. No new secret needed: the pipeline's secrets are now `PGPASS`, `REPO_RO_TOKEN`, and `FORGEJO_TOKEN` (all existing). `FORGEJO_TOKEN` must have repository write scope for the push to succeed. 105 tests pass; all pre-commit hooks pass. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/125 --- .crow/auto-apply-patches.yaml | 7 +++---- local/patches/README.md | 5 +++-- local/propose-patches.R | 25 +++++++++++-------------- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index 15cc1ee..2ac6886 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -6,8 +6,9 @@ # human reviews the PR. Novel source diffs / unknown signatures are never # proposed. Global across platforms, so a single job -- no matrix. # -# Needs a write token (REPO_RW_TOKEN) to push and FORGEJO_TOKEN to open the PR. -# Register the `auto-apply-patches` cron in the crow UI, or run manually: +# FORGEJO_TOKEN is used for both the branch push and opening the PR (no separate +# write-scoped secret needed). Register the `auto-apply-patches` cron in the crow +# UI, or run manually: # woodpecker-cli pipeline create --var task=auto-apply-patches --branch=main 7 variables: patch_limit: @@ -34,8 +35,6 @@ steps: from_secret: PGPASS REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN - REPO_RW_TOKEN: - from_secret: REPO_RW_TOKEN FORGEJO_TOKEN: from_secret: FORGEJO_TOKEN GIT_USER: devxy-bot diff --git a/local/patches/README.md b/local/patches/README.md index 455a96c..ce60c9a 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -84,11 +84,12 @@ Rscript local/trial-build-patch.R #### Autonomous PR + trial-build gate `--open-pr` closes the loop: it writes the top-N candidates (by failure volume) onto the reused `auto/registry-patch-proposals` branch and opens/updates a single PR. -`.crow/auto-apply-patches.yaml` runs this on a cron (needs `FORGEJO_TOKEN` and a write-scoped `REPO_RW_TOKEN`). +`.crow/auto-apply-patches.yaml` runs this on a cron. +`FORGEJO_TOKEN` is used for both the branch push and the PR (no separate write-scoped secret). ```bash # Bounded batch; opens/updates one PR. -PGPASS=... FORGEJO_TOKEN=... REPO_RW_TOKEN=... Rscript local/propose-patches.R --open-pr --limit 10 +PGPASS=... FORGEJO_TOKEN=... Rscript local/propose-patches.R --open-pr --limit 10 ``` The merge gate is `.crow/trial-build-registry.yaml`: matrixed over the build-env images, each platform trial-builds only the entries the branch **adds** (`local/trial-build-registry.R`, which diffs the registry against `main`) and is green only if every new entry builds. diff --git a/local/propose-patches.R b/local/propose-patches.R index fb45ea9..11085f0 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -21,8 +21,8 @@ # the proposals ledger (commit + open a PR yourself) # --open-issue post/update a Forgejo tracking issue (needs FORGEJO_TOKEN) # --open-pr write the entries, push the `auto/registry-patch-proposals` -# branch, and open/update a PR autonomously (needs -# FORGEJO_TOKEN, and REPO_RW_TOKEN to push in CI). The +# branch, and open/update a PR autonomously. Uses +# FORGEJO_TOKEN for both the push and the PR API. The # `trial-build-registry` pipeline is the merge gate. # --limit N only act on the top-N candidates by failure volume # (bounded batch; the rest are picked up on the next run) @@ -466,18 +466,15 @@ if (do_write) { if (length(candidate_entries) == 1L) "entry" else "entries" ) ) - # Push with a write token when provided (CI); otherwise rely on origin creds. - rw_token <- Sys.getenv("REPO_RW_TOKEN") - push_target <- if (nchar(rw_token) > 0L) { - sprintf( - "https://%s:%s@git.devxy.io/%s.git", - Sys.getenv("GIT_USER", "devxy-bot"), - rw_token, - repo - ) - } else { - "origin" - } + # Push over HTTPS with FORGEJO_TOKEN (same token used for the PR API), so no + # separate write-scoped secret is needed. The read-only `origin` clone URL + # can't push, so build an authenticated URL explicitly. + push_target <- sprintf( + "https://%s:%s@git.devxy.io/%s.git", + Sys.getenv("GIT_REMOTE_USER", "pat-s"), + forgejo_token, + repo + ) git("push", "-f", push_target, sprintf("HEAD:refs/heads/%s", pr_branch)) pr_title <- sprintf( From ed1485c542d4768e75537cf3728423b1c5ee564b Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 15 Jul 2026 16:51:15 +0200 Subject: [PATCH 071/125] ci: remove evaluate conditions --- .crow/auto-apply-patches.yaml | 1 - .crow/trial-build-registry.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index 2ac6886..b9c8697 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -17,7 +17,6 @@ variables: when: - event: manual - evaluate: 'task == "auto-apply-patches"' - event: cron cron: auto-apply-patches diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 1ac71e8..9871238 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -16,7 +16,6 @@ variables: when: - event: manual - evaluate: 'task == "trial-build-registry"' - event: cron cron: trial-build-registry From ff9f5f5177529553d89a289e8682681289b46be9 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 15 Jul 2026 15:43:02 +0000 Subject: [PATCH 072/125] fix(local): shell-quote git args in the auto-patch push and trial-build gate (#126) ## Problem The first real `auto-apply-patches` run classified, limited to the top-10, and validated the candidate registry cleanly, then died at the push step: ``` Validating candidate registry: Patch registry OK (13 entries). sh: syntax error: unexpected "(" Error in git("commit", "-m", ...): git commit -m feat(patches): auto-propose 10 registry entries ... failed ``` Root cause: R's `system2()` with captured output (`stdout=TRUE`) runs the command through `/bin/sh`, and the arguments were passed **unquoted**. The commit message `feat(patches): ...` contains `()`, which the shell tried to interpret. The same class of bug affects the `^{commit}` and `ref:path` git refs in the trial-build gate. ## Fix `shQuote()` every git argument: - `propose-patches.R` -- the `git()` helper used by `--open-pr` (commit, push, checkout). - `trial-build-registry.R` -- the base-ref reads (`rev-parse ... ^{commit}`, `ls-tree`, `show ref:path`). ## Verification - Reproduced against a real git repo: the unquoted call fails with status 2 (the same `unexpected "("`); the `shQuote`d call commits successfully, and `rev-parse HEAD^{commit}` resolves. - 105 tests pass; all pre-commit hooks pass. Everything else in that run was correct: 842,659 failing builds classified, RcppParallel's 895 dependents correctly reported as blocked (not proposed), top-10 tbb-stddef candidates selected, 69 deferred, registry validated. Only the shell quoting was broken. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/126 --- local/propose-patches.R | 4 +++- local/trial-build-registry.R | 13 ++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/local/propose-patches.R b/local/propose-patches.R index 11085f0..7008251 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -435,7 +435,9 @@ if (do_write) { save_ledger(merge_ledger(load_ledger(), new_ledger_records)) git <- function(...) { - st <- system2("git", c(...), stdout = TRUE, stderr = TRUE) + # system2() with captured output runs via /bin/sh, so shell-quote every arg + # (commit messages contain "()", refs contain "^{}", etc.). + st <- system2("git", shQuote(c(...)), stdout = TRUE, stderr = TRUE) if (!identical(attr(st, "status"), NULL)) { stop(sprintf( "git %s failed:\n%s", diff --git a/local/trial-build-registry.R b/local/trial-build-registry.R index f649b62..0749bf7 100644 --- a/local/trial-build-registry.R +++ b/local/trial-build-registry.R @@ -52,9 +52,16 @@ current <- if (file.exists(registry_file)) { # silently treating the base as empty would trial-build the WHOLE registry # instead of just the entries the branch adds. registry_rel <- "local/patches/registry.json" +# system2() with captured output runs via /bin/sh, so shell-quote the git args +# (refs contain "^{}" and ":" that the shell would otherwise mangle). ref_ok <- suppressWarnings(system2( "git", - c("rev-parse", "--verify", "--quiet", sprintf("%s^{commit}", base_ref)), + shQuote(c( + "rev-parse", + "--verify", + "--quiet", + sprintf("%s^{commit}", base_ref) + )), stdout = TRUE, stderr = FALSE )) @@ -63,14 +70,14 @@ if (!is.null(attr(ref_ok, "status"))) { } in_base <- suppressWarnings(system2( "git", - c("ls-tree", base_ref, "--", registry_rel), + shQuote(c("ls-tree", base_ref, "--", registry_rel)), stdout = TRUE, stderr = FALSE )) file_in_base <- length(in_base) > 0L && any(nzchar(in_base)) base_json <- suppressWarnings(system2( "git", - c("show", sprintf("%s:%s", base_ref, registry_rel)), + shQuote(c("show", sprintf("%s:%s", base_ref, registry_rel))), stdout = TRUE, stderr = FALSE )) From 875b0861226d55b1762a1f5a181940b5e9be43d4 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 16 Jul 2026 08:23:44 +0000 Subject: [PATCH 073/125] ci(crow): enable verbose patched builds in the trial-build gate (#129) Companion to bincraft codefloe #65. Sets `BINCRAFT_VERBOSE_PATCH_BUILD=TRUE` in the trial-build gate so a failed isolated patched build prints the real compiler error instead of `System command 'R' failed`. Harmless on bincraft versions without the flag (unknown env var is ignored). Takes effect once bincraft #65 is released and the build-env images pick it up. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/129 --- .crow/trial-build-registry.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 9871238..602837f 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -116,6 +116,10 @@ steps: ARCH: ${ARCH} R_VERSION: ${R_VERSION} R_LIBS_USER: /mnt/cache/R-pkgs + # Surface the real compiler error when an isolated patched build fails, + # instead of bincraft's opaque "System command 'R' failed" (needs bincraft + # with BINCRAFT_VERBOSE_PATCH_BUILD support; harmless on older versions). + BINCRAFT_VERBOSE_PATCH_BUILD: 'TRUE' commands: - git clone -q --branch ${patch_branch} https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git fetch -q origin main From f9d399fac0e60f748eaa2c65d23f2c34c437d3a1 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 16 Jul 2026 08:23:54 +0000 Subject: [PATCH 074/125] feat(local): detect dependency-cascade failures generally, not just RcppParallel (#128) ## Why (from the #127 trial-build gate) The gate did its job: 0/3 passed, merge blocked. The log showed *why* -- BFpack, BayesERtools, GMLTM all fail while building their shared dependency **`rstan`**, not in their own code: ``` Failed to build source package rstan. .../StanHeaders/include/stan/math/prim/core/init_threadpool_tbb.hpp:9:10: fatal error: tbb/tbb_stddef.h: No such file or directory ``` So the per-package `-DTBB_INTERFACE_NEW` makevars entries the classifier proposed are useless for these packages -- they're blocked on `rstan` (which already has a registry entry). This is the **same dependency cascade** the RcppParallel `applies_to` guard catches, but `tbb-stddef-removed` is a generic signature with no such pin, so ~73 Stan packages kept getting proposed. ## What Generalise cascade detection beyond the RcppParallel special case: - `failing_dependency(error_text, package)` -- when the log names a **different** package as the one that failed to compile (`Failed to build source package X`, `compilation failed for package 'X'`, `dependency 'X' ... not available`), that package is the real cause. - `build_triage_report()` now blocks any package whose **every** failing build is such a cascade: reported as `blocked_on` that dependency, never proposed a bogus per-package entry. A package that fails in its **own** compilation is still proposed. - The `applies_to` (RcppParallel) and data-driven (rstan) cases are unified into one `blocked_packages` / `blocked_on` model; the report, proposer, and `blocked_summary` count the actually-blocked packages, and the blocked note shows even when a group also has genuine proposals. ## Effect Next auto-apply run will stop proposing the rstan-blocked Stan packages (and any future dependency cascade) and surface them as "blocked on rstan" instead. Fixing `rstan` once clears the whole cluster. ## Verification - New tests: `failing_dependency` (cascade vs own-compile vs none), and an end-to-end split where BFpack/GMLTM (blocked on rstan) are not proposed while an own-compile package still is. - Full suite: 112 tests pass; all pre-commit hooks pass. Refs #120, #127. (Separate follow-ups: fixing rstan's build itself, and quieting the gate's metadata-DB retry storm -- both root-caused to bincraft.) Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/128 --- local/failing-builds-classify.R | 77 +++++++++++++++++++--- local/failing-builds-report.R | 12 ++-- local/proposal-tracking-lib.R | 19 ++++-- local/propose-patches.R | 7 +- local/tests/test-failing-builds-classify.R | 43 ++++++++++++ 5 files changed, 135 insertions(+), 23 deletions(-) diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index 4ce30cf..716fab2 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -155,6 +155,44 @@ fingerprint_error <- function(error_text, package = NULL, max_chars = 200L) { fp } +# --------------------------------------------------------------------------- +# Dependency-cascade detection +# --------------------------------------------------------------------------- +# If a build's error_text shows the failure was actually in a DIFFERENT package +# (a dependency that would not compile), return that dependency's name; +# otherwise NA. Used to avoid proposing a per-package fix for a package that +# only fails because a shared dependency does not build (e.g. the ~73 Stan +# packages that fail while building `rstan`). Generalises the RcppParallel +# `applies_to` guard to any dependency named in the log. +failing_dependency <- function(error_text, package) { + if (length(error_text) == 0L || is.na(error_text) || !nzchar(error_text)) { + return(NA_character_) + } + x <- as.character(error_text) + # optional opening quote before the package name: apostrophe, double-quote, + # backtick, or curly quotes -- written as \u escapes so the pattern stays + # valid UTF-8 regardless of source encoding. + q <- "[\u0027\u0022\u0060\u2018\u2019]?" + name <- "([A-Za-z][A-Za-z0-9._]+)" + # Markers R/pak emit naming the package that actually failed to compile. + pats <- c( + paste0("compilation failed for package ", q, name), + paste0("Failed to build source package ", q, name), + paste0("Error in building package ", q, name), + paste0("dependenc(?:y|ies) ", q, name, q, "?[^\\n]*not available") + ) + deps <- character(0L) + for (p in pats) { + hits <- regmatches(x, gregexpr(p, x, perl = TRUE))[[1L]] + if (length(hits) > 0L) { + deps <- c(deps, sub(p, "\\1", hits, perl = TRUE)) + } + } + deps <- sub("[._]+$", "", deps) # drop a trailing sentence period (e.g. "rstan.") + deps <- setdiff(unique(deps), package) # a package failing on its OWN code is not a cascade + if (length(deps) == 0L) NA_character_ else deps[[1L]] +} + # --------------------------------------------------------------------------- # Classification # --------------------------------------------------------------------------- @@ -291,18 +329,38 @@ build_triage_report <- function( unregistered <- setdiff(pkgs, registered_pkgs) auto_proposable <- isTRUE(sig$auto) && sig$matched - # A signature whose fix is package-specific (`applies_to`) may only be - # proposed for that package. Everything else matching it is a downstream - # failure blocked on that package (e.g. RcppParallel dependents carrying - # RcppParallel's own error text) -- never propose those a bogus entry. + # Decide, per package, whether it is genuinely fixable or merely blocked on + # a dependency (so a per-package entry would be useless). Two blocking modes: + # 1. `applies_to`: a package-specific patch (e.g. RcppParallel's) is only + # valid for its own package; other matches are downstream of it. + # 2. data-driven cascade: the package's error_text shows a *different* + # package failed to compile (e.g. the ~73 Stan packages blocked on rstan). + # `blocked_map` maps a blocked package -> the dependency it waits on. + g$blocked_dep <- vapply( + seq_len(nrow(g)), + function(i) failing_dependency(g$error_text[[i]], g$name[[i]]), + character(1L) + ) proposable <- unregistered - blocked_on <- NULL + blocked_map <- list() if (!is.null(sig$applies_to)) { - proposable <- intersect(unregistered, sig$applies_to) - downstream <- setdiff(pkgs, sig$applies_to) - if (length(downstream) > 0L) { - blocked_on <- sig$applies_to + for (p in setdiff(pkgs, sig$applies_to)) { + blocked_map[[p]] <- sig$applies_to } + proposable <- intersect(proposable, sig$applies_to) + } + for (p in proposable) { + deps <- g$blocked_dep[g$name == p] + if (!any(is.na(deps))) { + # every failing build of p is a cascade -> blocked, not fixable here + blocked_map[[p]] <- unique(deps) + } + } + proposable <- setdiff(proposable, names(blocked_map)) + blocked_packages <- names(blocked_map) + blocked_on <- unique(unlist(blocked_map, use.names = FALSE)) + if (length(blocked_on) == 0L) { + blocked_on <- NULL } proposed <- list() @@ -326,6 +384,7 @@ build_triage_report <- function( suggested_fix = sig$fix, applies_to = sig$applies_to, blocked_on = blocked_on, + blocked_packages = blocked_packages, fingerprint = names(fp_tab)[[1L]], fingerprint_variants = length(fp_tab), build_count = nrow(g), diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R index 51f5bc7..ce7b683 100644 --- a/local/failing-builds-report.R +++ b/local/failing-builds-report.R @@ -194,15 +194,17 @@ for (r in report) { ) cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "") } - } else if (!is.null(r$blocked_on)) { + } else if (r$auto_proposable && length(r$blocked_packages) == 0L) { + cat(" (all affected packages already have a registry entry)\n") + } + # Blocked packages are shown even when the group also has proposals. + if (length(r$blocked_packages) > 0L) { cat(sprintf( - " (blocked on %s -- these %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", + " (blocked on %s -- %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", toString(r$blocked_on), - length(r$packages), + length(r$blocked_packages), toString(r$blocked_on) )) - } else if (r$auto_proposable) { - cat(" (all affected packages already have a registry entry)\n") } } diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 80c4ab2..9568d78 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -160,17 +160,22 @@ unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) { ) } -# Groups blocked on a dependency (a package-specific fix pinned via `applies_to` -# whose dependents merely carry its error): report the dependency + how many -# dependents wait on it, so fixing it once is recognised as clearing the batch. +# Groups with packages blocked on a dependency (a package-specific fix pinned +# via `applies_to`, or a data-driven cascade where the package fails building a +# dependency): report the dependency + how many dependents wait on it, so fixing +# it once is recognised as clearing the batch. blocked_summary <- function(report, max_pkgs = 15L) { - bl <- Filter(function(g) !is.null(g$blocked_on), report) + bl <- Filter( + function(g) length(g$blocked_packages %||% character(0L)) > 0L, + report + ) lapply(bl, function(g) { + pkgs <- g$blocked_packages list( blocked_on = g$blocked_on, - n_packages = length(g$packages), - packages = utils::head(g$packages, max_pkgs), - packages_truncated = length(g$packages) > max_pkgs + n_packages = length(pkgs), + packages = utils::head(pkgs, max_pkgs), + packages_truncated = length(pkgs) > max_pkgs ) }) } diff --git a/local/propose-patches.R b/local/propose-patches.R index 7008251..cfce338 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -150,14 +150,17 @@ for (r in report) { # Groups blocked on a dependency (e.g. RcppParallel dependents) are reported, # not proposed: fixing the named dependency clears them all at once. -blocked <- Filter(function(r) !is.null(r$blocked_on), report) +blocked <- Filter( + function(r) length(r$blocked_packages) > 0L, + report +) if (length(blocked) > 0L) { cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") for (r in blocked) { cat(sprintf( " %s: %d package(s) fail because %s does not build\n", toString(r$blocked_on), - length(r$packages), + length(r$blocked_packages), toString(r$blocked_on) )) } diff --git a/local/tests/test-failing-builds-classify.R b/local/tests/test-failing-builds-classify.R index a365e14..d4df9f5 100644 --- a/local/tests/test-failing-builds-classify.R +++ b/local/tests/test-failing-builds-classify.R @@ -105,6 +105,49 @@ test_that("RcppParallel dependents are blocked, not proposed a per-package patch expect_identical(grp$blocked_on, "RcppParallel") }) +test_that("failing_dependency names the dependency that actually failed", { + # A leaf package (BFpack) that fails building its rstan dependency. + txt <- paste( + "Error in installing dependencies for package BFpack with tag 1.6.1", + "Failed to build source package rstan.", + "ERROR: compilation failed for package ‘rstan’", + sep = "\n" + ) + expect_identical(failing_dependency(txt, "BFpack"), "rstan") + # A package failing in its OWN compilation is not a cascade. + own <- "ERROR: compilation failed for package ‘BFpack’" + expect_true(is.na(failing_dependency(own, "BFpack"))) + expect_true(is.na(failing_dependency(NA_character_, "x"))) + expect_true(is.na(failing_dependency("some unrelated error", "x"))) +}) + +test_that("Stan packages blocked on rstan are not proposed a per-package entry", { + # BFpack/GMLTM fail building rstan; the tbb error is in rstan's compile. + cascade <- paste( + "Failed to build source package rstan.", + "fatal error: tbb/tbb_stddef.h: No such file or directory", + sep = "\n" + ) + failures <- data.frame( + name = c("BFpack", "GMLTM", "someOwnPkg"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = c( + cascade, + cascade, + # someOwnPkg fails in its OWN compile on the same header -> fixable. + "someOwnPkg.cpp: fatal error: tbb/tbb_stddef.h: No such file or directory" + ), + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + grp <- Filter(function(g) g$signature == "tbb-stddef-removed", report)[[1L]] + # Only the own-compile package is proposed; the rstan cascades are blocked. + expect_identical(names(grp$proposed_entries), "someOwnPkg") + expect_setequal(grp$blocked_packages, c("BFpack", "GMLTM")) + expect_identical(grp$blocked_on, "rstan") +}) + test_that("RcppParallel itself is still proposed when it is the failing package", { failures <- data.frame( name = "RcppParallel", From 118a92889f323f01363c7eed9573e203666ee8a0 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 16 Jul 2026 08:24:03 +0000 Subject: [PATCH 075/125] fix(local): make the trial-build gate detect non-throwing build failures (#130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Critical: the gate was false-green The latest run printed **`3/3 passed on alpine-324`** while all three builds actually **failed** (their `rstan` dependency won't compile). A false-green gate would let broken registry entries merge — worse than no gate. Root cause: `bincraft::build_binary_package()` catches build failures internally and **returns `"error"`** for the failed tag rather than throwing. The gate's `tryCatch` only treated a *thrown* exception as failure, so every non-throwing failure looked like a pass. Fix: inspect the return value. A tag passes only if the flattened result is non-empty and contains no `"error"` sentinel; a thrown error still counts as failure. Verified the verdict against `error`/`skipped`/`TRUE`/`list(success=TRUE)`/`NULL`/mixed inputs. With this, the current rstan-blocked entries will correctly show **0/3 (red)** — which is the right answer until rstan builds. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/130 --- local/trial-build-registry.R | 39 +++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/local/trial-build-registry.R b/local/trial-build-registry.R index 0749bf7..9708408 100644 --- a/local/trial-build-registry.R +++ b/local/trial-build-registry.R @@ -117,23 +117,34 @@ results <- vapply( pkgs, function(pkg) { cat(sprintf("\n=== trial build: %s ===\n", pkg)) - tryCatch( - { - bincraft::build_binary_package( - pkg, - tag_limit = 1L, - patches = patches_dir, - archive = FALSE, - upload = FALSE, - store_build_metadata = FALSE - ) - TRUE - }, + # bincraft::build_binary_package() catches build failures internally and + # RETURNS "error" for the failed tag rather than throwing, so a green gate + # must inspect the return value -- checking only for a thrown exception + # reports a broken build as passing. + res <- tryCatch( + bincraft::build_binary_package( + pkg, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ), error = function(e) { - cat(sprintf("FAILED %s: %s\n", pkg, conditionMessage(e))) - FALSE + cat(sprintf("FAILED %s (threw): %s\n", pkg, conditionMessage(e))) + "error" } ) + flat <- as.character(unlist(res)) + ok <- length(flat) > 0L && !("error" %in% flat) + if (!ok) { + cat(sprintf( + "FAILED %s: build did not succeed (result: %s)\n", + pkg, + if (length(flat) > 0L) toString(flat) else "" + )) + } + ok }, logical(1L) ) From 6c03f278ecac4d3df58d1d8674259eee45dfe6fd Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 16 Jul 2026 21:24:03 +0000 Subject: [PATCH 076/125] feat(local): aggregate blocked-on-dependency reporting by dependency (#131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why With cascade detection (#128) live, the latest `auto-apply-patches` run did exactly the right thing — **proposed nothing** (`No auto-proposable candidates`) because every failure is a dependency cascade, and it surfaced the ~30 root-cause dependencies to fix. But the "Blocked on a dependency" list printed **one line per fingerprint group**, so the same dependency repeated (rstan ×4, lpsymphony ×4, salso ×2, BH ×2, GO.db ×2, RcppCWB ×2, …), burying the priority. ## What Aggregate blocked packages across all groups **by the dependency they wait on**: - Expose `blocked_map` (package → dependency) from `build_triage_report()`. - Add `blocked_by_dependency()` — dedupes dependents (a package in two groups counts once) and ranks dependencies by how many distinct dependents they block. - Proposer and tracker (log + issue) now print one line per dependency, sorted by impact. Replaces the per-group `blocked_summary`. ## Result (same data, aggregated) ``` Blocked on a dependency (3 dependencies block 6 dependents; fix the dependency, not each dependent): RcppParallel 3 dependent(s) rstan 2 dependent(s) sf 1 dependent(s) ``` So the real run becomes a crisp, ranked worklist: RcppParallel (894), sf (128), rstan (~96), Rfast (33), clarabel/DescTools (26), Rglpk (22), xgboost (18), … ## Verified New test covers cross-group aggregation, dedup (a dependent in two groups counted once), the example cap, and ranking. 112 tests pass; hooks pass. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/131 --- local/failing-builds-classify.R | 1 + local/proposal-tracking-lib.R | 38 ++++++++++++++++-------- local/proposal-tracking.R | 8 ++--- local/propose-patches.R | 27 ++++++++--------- local/tests/test-proposal-tracking-lib.R | 33 ++++++++++++++------ 5 files changed, 68 insertions(+), 39 deletions(-) diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index 716fab2..73922b3 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -385,6 +385,7 @@ build_triage_report <- function( applies_to = sig$applies_to, blocked_on = blocked_on, blocked_packages = blocked_packages, + blocked_map = if (length(blocked_map) > 0L) blocked_map else NULL, fingerprint = names(fp_tab)[[1L]], fingerprint_variants = length(fp_tab), build_count = nrow(g), diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 9568d78..28f65db 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -160,24 +160,38 @@ unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) { ) } -# Groups with packages blocked on a dependency (a package-specific fix pinned -# via `applies_to`, or a data-driven cascade where the package fails building a -# dependency): report the dependency + how many dependents wait on it, so fixing -# it once is recognised as clearing the batch. -blocked_summary <- function(report, max_pkgs = 15L) { - bl <- Filter( - function(g) length(g$blocked_packages %||% character(0L)) > 0L, - report - ) - lapply(bl, function(g) { - pkgs <- g$blocked_packages +# Aggregate blocked packages across ALL groups by the dependency they wait on, +# so one dependency (RcppParallel, rstan, sf, ...) is a single line -- deduped +# and ranked by how many distinct dependents it blocks -- instead of repeating +# once per fingerprint group. Reads each group's `blocked_map` (package -> the +# dependency it is blocked on). Returns records sorted by dependent count desc, +# each with up to `max_pkgs` example dependents. +blocked_by_dependency <- function(report, max_pkgs = 15L) { + acc <- list() # dependency -> character vector of dependent packages + for (g in report) { + bm <- g$blocked_map + if (is.null(bm) || length(bm) == 0L) { + next + } + for (pkg in names(bm)) { + for (dep in as.character(unlist(bm[[pkg]]))) { + acc[[dep]] <- unique(c(acc[[dep]], pkg)) + } + } + } + if (length(acc) == 0L) { + return(list()) + } + out <- lapply(names(acc), function(dep) { + pkgs <- acc[[dep]] list( - blocked_on = g$blocked_on, + dependency = dep, n_packages = length(pkgs), packages = utils::head(pkgs, max_pkgs), packages_truncated = length(pkgs) > max_pkgs ) }) + out[order(-vapply(out, function(x) x$n_packages, integer(1L)))] } # Does a registry entry's `platforms` apply to a build on `os` (e.g. diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R index 4d82a11..f8e1284 100644 --- a/local/proposal-tracking.R +++ b/local/proposal-tracking.R @@ -127,13 +127,13 @@ if (length(retire) > 0L) { # --------------------------------------------------------------------------- # Blind spots: failures the classifier could not auto-propose. # --------------------------------------------------------------------------- -blocked <- blocked_summary(report) +blocked <- blocked_by_dependency(report) unmatched <- unclassified_summary(report) cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") if (length(blocked) > 0L) { for (b in blocked) { - cat(sprintf(" %s: %d dependent(s) waiting\n", b$blocked_on, b$n_packages)) + cat(sprintf(" %-20s %5d dependent(s)\n", b$dependency, b$n_packages)) } } else { cat(" (none)\n") @@ -207,8 +207,8 @@ if (do_issue) { body_lines <- c( body_lines, sprintf( - "- **%s**: %d dependent(s) waiting (e.g. %s%s)", - b$blocked_on, + "- **%s**: %d dependent(s) (e.g. %s%s)", + b$dependency, b$n_packages, toString(b$packages), if (isTRUE(b$packages_truncated)) ", ..." else "" diff --git a/local/propose-patches.R b/local/propose-patches.R index cfce338..360df9f 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -148,21 +148,20 @@ for (r in report) { } } -# Groups blocked on a dependency (e.g. RcppParallel dependents) are reported, -# not proposed: fixing the named dependency clears them all at once. -blocked <- Filter( - function(r) length(r$blocked_packages) > 0L, - report -) +# Packages blocked on a dependency are reported (aggregated by dependency, +# ranked by impact), not proposed: fixing the named dependency clears the batch. +blocked <- blocked_by_dependency(report) if (length(blocked) > 0L) { - cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") - for (r in blocked) { - cat(sprintf( - " %s: %d package(s) fail because %s does not build\n", - toString(r$blocked_on), - length(r$blocked_packages), - toString(r$blocked_on) - )) + n_blocked_pkgs <- length(unique(unlist( + lapply(report, function(r) r$blocked_packages) + ))) + cat(sprintf( + "\nBlocked on a dependency (%d dependencies block %d dependents; fix the dependency, not each dependent):\n", + length(blocked), + n_blocked_pkgs + )) + for (b in blocked) { + cat(sprintf(" %-20s %5d dependent(s)\n", b$dependency, b$n_packages)) } } diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index 05cd0e9..869d2b3 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -146,20 +146,35 @@ test_that("unclassified_summary ranks unknown groups and caps output", { expect_identical(capped$dropped_groups, 1L) }) -test_that("blocked_summary lists each dependency and its dependent count", { +test_that("blocked_by_dependency aggregates across groups, deduped and ranked", { + # RcppParallel dependents split across platforms/fingerprints -> separate + # groups, but one aggregated line; a dependent seen twice is counted once. failures <- data.frame( - name = c("ACEsimFit", "AovBay", "AdaptGauss"), - platform = "ubuntu-2604", + name = c("ACEsimFit", "AovBay", "AdaptGauss", "ACEsimFit", "loner"), + platform = c( + "ubuntu-2604", + "ubuntu-2604", + "alpine-324", + "alpine-324", + "ubuntu-2604" + ), arch = "amd64", - error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + error_text = c( + rep("Error: USE_TBB=Linux is not supported on this toolchain", 4L), + "Failed to build source package rstan.\nfatal error: tbb/tbb_stddef.h" + ), stringsAsFactors = FALSE ) report <- build_triage_report(failures, registered_pkgs = character(0L)) - b <- blocked_summary(report, max_pkgs = 2L) - expect_length(b, 1L) - expect_identical(b[[1L]]$blocked_on, "RcppParallel") - expect_identical(b[[1L]]$n_packages, 3L) - expect_true(b[[1L]]$packages_truncated) + agg <- blocked_by_dependency(report, max_pkgs = 2L) + deps <- vapply(agg, function(b) b$dependency, character(1L)) + expect_true("RcppParallel" %in% deps && "rstan" %in% deps) + rcpp <- Filter(function(b) b$dependency == "RcppParallel", agg)[[1L]] + # ACEsimFit appears in two groups -> counted once (3 distinct dependents). + expect_identical(rcpp$n_packages, 3L) + expect_true(rcpp$packages_truncated) # capped at max_pkgs = 2 + # Ranked by dependent count: RcppParallel (3) before rstan (1). + expect_identical(deps[[1L]], "RcppParallel") }) test_that("entry_applies_to_os matches codename, family, and wildcard", { From 13fcad6dc4711c05813132b505c215a986f1b1df Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 17 Jul 2026 10:27:58 +0000 Subject: [PATCH 077/125] fix(crow): make the trial-build gate no-op when the auto-patch branch is absent (#132) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The trial pipeline errored: ``` git clone -q --branch auto/registry-patch-proposals ... fatal: Remote branch auto/registry-patch-proposals not found in upstream origin ``` I deleted the stale `auto/registry-patch-proposals` branch during cleanup, and — now that cascade detection (#128) makes the proposer correctly find **no candidates** — the proposer exits before ever recreating it. So the branch is legitimately absent, and the gate's hard `git clone --branch` fails. ## Fix "Branch absent" = "no pending auto-patch proposals" = **nothing to verify**, which should be a clean no-op, not a failure. Clone `main`, then fetch + checkout the patch branch only if it exists; otherwise log and `exit 0`. A present branch is verified exactly as before (checkout its tip, diff registry vs `main`). This also means the gate now runs `main`'s `trial-build-registry.R` (with the #130 fix) rather than a possibly-stale copy on the branch. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/132 --- .crow/trial-build-registry.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 602837f..3ec0417 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -121,8 +121,13 @@ steps: # with BINCRAFT_VERBOSE_PATCH_BUILD support; harmless on older versions). BINCRAFT_VERBOSE_PATCH_BUILD: 'TRUE' commands: - - git clone -q --branch ${patch_branch} https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + # Clone main, then check out the auto-patch branch if it exists. When the + # proposer had no candidates it never (re)creates that branch, so a missing + # branch means "nothing to verify" -- no-op cleanly instead of failing the + # clone. + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git fetch -q origin main + - 'if git fetch -q origin ${patch_branch}; then git checkout -q FETCH_HEAD; else echo "No ${patch_branch} branch; no pending auto-patch proposals to verify."; exit 0; fi' - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R From a30c6f953273ec6001ed7f1834f4f4f8e7427448 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 17 Jul 2026 11:52:07 +0000 Subject: [PATCH 078/125] fix(crow): probe the auto-patch branch with git ls-remote to avoid a scary fatal (#133) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #132. The no-op path already worked (the `else` branch runs and `exit 0`s), but using `git fetch` as the *existence check* printed `fatal: couldn't find remote ref auto/registry-patch-proposals` to stderr — which looks like a failure even though the step succeeds. Probe with `git ls-remote --exit-code --heads origin ${patch_branch}` instead: verified against the live repo it exits 0 for an existing branch and 2 for a missing one, **silently** (no `fatal`). Only fetch + checkout when the branch actually exists. Net: same behavior, clean log — a run with no pending auto-patch branch prints just `No ... branch; no pending auto-patch proposals to verify.` and exits 0. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/133 --- .crow/trial-build-registry.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 3ec0417..cb8df8b 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -127,7 +127,7 @@ steps: # clone. - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git fetch -q origin main - - 'if git fetch -q origin ${patch_branch}; then git checkout -q FETCH_HEAD; else echo "No ${patch_branch} branch; no pending auto-patch proposals to verify."; exit 0; fi' + - 'if git ls-remote --exit-code --heads origin ${patch_branch} >/dev/null 2>&1; then git fetch -q origin ${patch_branch} && git checkout -q FETCH_HEAD; else echo "No ${patch_branch} branch; no pending auto-patch proposals to verify."; exit 0; fi' - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R From b4b7d53a832fc8d041849169fc94077679c88c1c Mon Sep 17 00:00:00 2001 From: pat-s Date: Sat, 18 Jul 2026 08:37:04 +0000 Subject: [PATCH 079/125] fix(local): make trial-build-patch.R detect non-throwing build failures (#134) `local/trial-build-patch.R` -- the single-package acceptance gate the proposer prints for humans to run (`PGPASS=... Rscript local/trial-build-patch.R `) -- reported **`Trial build OK`** for rstan even though rstan.so failed to load (`symbol not found: tbb::detail::r1::observe`). Same false-green as the registry gate before #130: `build_binary_package()` catches failures internally and returns `"error"` rather than throwing, so a `tryCatch` that only treats a thrown exception as failure passes a broken build. Now inspects the return value (mirrors trial-build-registry.R). Found while verifying the rstan/oneTBB unblock. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/134 --- local/trial-build-patch.R | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/local/trial-build-patch.R b/local/trial-build-patch.R index 4e8030a..2229074 100644 --- a/local/trial-build-patch.R +++ b/local/trial-build-patch.R @@ -40,23 +40,26 @@ cat(sprintf( patches_dir )) -ok <- tryCatch( - { - bincraft::build_binary_package( - package, - tag_limit = 1L, - patches = patches_dir, - archive = FALSE, - upload = FALSE, - store_build_metadata = FALSE - ) - TRUE - }, +# build_binary_package() catches build failures internally and RETURNS "error" +# for the failed tag rather than throwing (bincraft >= v4.4.7), so inspect the +# return value -- checking only for a thrown exception reports a broken build as +# passing (false green). +res <- tryCatch( + bincraft::build_binary_package( + package, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ), error = function(e) { - cat(sprintf("Trial build FAILED: %s\n", conditionMessage(e))) - FALSE + cat(sprintf("Trial build FAILED (threw): %s\n", conditionMessage(e))) + "error" } ) +flat <- as.character(unlist(res)) +ok <- length(flat) > 0L && !("error" %in% flat) if (ok) { cat(sprintf("Trial build OK: %s builds with the proposed patch.\n", package)) From 6df945d1472beb6fd3f0372a31085af38225b119 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 20 Jul 2026 09:56:21 +0000 Subject: [PATCH 080/125] fix(patches): build RcppParallel against system oneTBB instead of disabling TBB (#135) ## What Replace RcppParallel's `disable-tbb` registry patch with `system-tbb`: build RcppParallel against the system oneTBB instead of stripping TBB entirely. ## Why `disable-tbb` skipped the bundled Intel TBB build by forcing the TinyThread backend, which also removed RcppParallel's TBB linkage. That broke every dependent that links TBB through `RcppParallelLibs()` -- `rstan` and the whole Stan cluster -- with `symbol not found: tbb::detail::r1::observe`. This is the root cause behind the large "blocked on RcppParallel" / rstan clusters (issues #115, #120): the packages were not individually broken, they were all waiting on one TBB-linkage regression. ## Change `system-tbb.patch` leaves `USE_TBB` unset (so the bundled build is still skipped on musl and g++ 15) but keeps the TBB backend and links the system oneTBB now shipped in the build-env images: ``` PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=1 -DTBB_SUPPRESS_DEPRECATED_MESSAGES=1 -DTBB_INTERFACE_NEW PKG_LIBS += -ltbb -ltbbmalloc ``` Scoped to `alpine` + `ubuntu-2604`, the only platforms where the bundled build fails; redhat and older ubuntus keep the bundled TBB. ## Requires The companion image change that exports `TBB_INC`/`TBB_LIB` so `RcppParallelLibs()` hands the system-TBB flags to dependents: build-env-images PR #12. Both must ship together. ## Verified In `build-env-alpine:3.24` with the new images: - RcppParallel builds against oneTBB 2022, no ABI errors, patch applies cleanly to the target clone. - `RcppParallelLibs()` returns `-L/usr/lib -Wl,-rpath,/usr/lib -ltbb -ltbbmalloc`. - rstan links (`-ltbb -ltbbmalloc`) and loads with no missing symbol -- `* DONE (rstan)`, trial build exit 0. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/135 --- local/patches/RcppParallel/disable-tbb.patch | 16 ---------------- local/patches/RcppParallel/system-tbb.patch | 19 +++++++++++++++++++ local/patches/registry.json | 17 ++++++++++++----- 3 files changed, 31 insertions(+), 21 deletions(-) delete mode 100644 local/patches/RcppParallel/disable-tbb.patch create mode 100644 local/patches/RcppParallel/system-tbb.patch diff --git a/local/patches/RcppParallel/disable-tbb.patch b/local/patches/RcppParallel/disable-tbb.patch deleted file mode 100644 index 0fe8ad9..0000000 --- a/local/patches/RcppParallel/disable-tbb.patch +++ /dev/null @@ -1,16 +0,0 @@ -diff --git a/src/Makevars.in b/src/Makevars.in -index be8445f..faee771 100644 ---- a/src/Makevars.in -+++ b/src/Makevars.in -@@ -60,7 +60,10 @@ else - endif - - ifeq ($(UNAME), Linux) -- USE_TBB=Linux -+ # bincraft patch: the bundled Intel TBB build hangs/fails on musl (Alpine) -+ # and newer toolchains (g++ 15). Skip it (leave USE_TBB unset) and force the -+ # TinyThread backend so RcppParallel still builds. -+ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=0 - endif - - ifeq ($(UNAME), SunOS) diff --git a/local/patches/RcppParallel/system-tbb.patch b/local/patches/RcppParallel/system-tbb.patch new file mode 100644 index 0000000..263a593 --- /dev/null +++ b/local/patches/RcppParallel/system-tbb.patch @@ -0,0 +1,19 @@ +diff --git a/src/Makevars.in b/src/Makevars.in +index be8445f..7cb6c9e 100644 +--- a/src/Makevars.in ++++ b/src/Makevars.in +@@ -60,7 +60,13 @@ else + endif + + ifeq ($(UNAME), Linux) +- USE_TBB=Linux ++ # bincraft patch: link the system oneTBB (installed in the build-env images) ++ # instead of building the bundled Intel TBB, which fails on musl (Alpine) and ++ # newer toolchains (g++ 15). Leaving USE_TBB unset skips the bundled build; ++ # -DRCPP_PARALLEL_USE_TBB=1 keeps the TBB backend so dependents (rstan, ...) ++ # link TBB, and -ltbb/-ltbbmalloc pull the system library from default paths. ++ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=1 -DTBB_SUPPRESS_DEPRECATED_MESSAGES=1 -DTBB_INTERFACE_NEW ++ PKG_LIBS += -ltbb -ltbbmalloc + endif + + ifeq ($(UNAME), SunOS) diff --git a/local/patches/registry.json b/local/patches/registry.json index e53f1c9..c446fda 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -2,17 +2,22 @@ { "package": "RcppParallel", "versions": "*", - "platforms": ["alpine", "ubuntu-2604"], + "platforms": [ + "alpine", + "ubuntu-2604" + ], "env": {}, "configure_args": [], "makevars": {}, - "patch": "RcppParallel/disable-tbb.patch", - "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); patch unsets USE_TBB and forces -DRCPP_PARALLEL_USE_TBB=0 so RcppParallel skips the bundled build and uses the TinyThread backend" + "patch": "RcppParallel/system-tbb.patch", + "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); link the system oneTBB (now in the build-env images) instead, keeping the TBB backend so dependents (rstan, ...) link TBB" }, { "package": "fs", "versions": "*", - "platforms": ["*"], + "platforms": [ + "*" + ], "env": {}, "configure_args": [], "makevars": {}, @@ -22,7 +27,9 @@ { "package": "rstan", "versions": "*", - "platforms": ["*"], + "platforms": [ + "*" + ], "env": {}, "configure_args": [], "makevars": { From 61e1c0188d79f8dd03b5a523ef5e0a1542a469ef Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 20 Jul 2026 15:24:17 +0200 Subject: [PATCH 081/125] chore: adjust weekly audit runs --- .crow/weekly-audit-missing.yaml | 36 ++++++++++++++++++++----------- .crow/weekly-rebuild-missing.yaml | 26 +++++++++++++++++++--- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 6ff71e3..d66a37b 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -11,12 +11,12 @@ variables: description: "Manual run target: a specific -, or 'all' for every os/arch." options: - all - - alpine-321-amd64 - - alpine-321-arm64 - alpine-322-amd64 - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -27,6 +27,8 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 default: all when: @@ -42,27 +44,27 @@ labels: matrix: include: - - OS: alpine-321 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: alpine-321 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 @@ -106,6 +108,14 @@ matrix: ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute steps: - name: 'Audit missing binaries' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 934c518..53f178f 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -16,6 +16,8 @@ variables: - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -26,6 +28,8 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 default: all when: @@ -44,16 +48,24 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 @@ -97,6 +109,14 @@ matrix: ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute steps: - name: 'Rebuild missing binaries' From 3cc323d99e4de7ad3261f926a9da28d54ed3ff2f Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 20 Jul 2026 20:59:47 +0000 Subject: [PATCH 082/125] fix(patches): widen RcppParallel system-tbb scope to all oneTBB-2021 platforms (#137) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Widen the RcppParallel `system-tbb` registry patch scope from `[alpine, ubuntu-2604]` to `[alpine, ubuntu, redhat-10]`. ## Why #135 fixed the TBB cluster on alpine + ubuntu-2604 only. A per-image audit showed the problem is broader: RcppParallel's bundled Intel TBB build fails on **every** build image with a modern toolchain (alpine/musl, ubuntu jammy/noble/resolute g++ 11/13/15, redhat-10 GCC 14), and only redhat-8/9 (older toolchains + pre-oneTBB system TBB) tolerate the bundled build. The system-TBB fix applies wherever the image ships **oneTBB 2021+** (has the `tbb::detail::r1` namespace rstan needs): | platform | system TBB | in scope? | |---|---|---| | alpine (all) | oneTBB 2022 | ✅ | | ubuntu 2204/2404/2604 | oneTBB 2021.5 / 2021.11 / 2022 | ✅ (via `ubuntu` family token) | | redhat-10 | oneTBB 2021.11 | ✅ | | redhat-8 | TBB 2018 (classic) | ❌ pre-oneTBB → bundled | | redhat-9 | TBB 2020.3 (classic) | ❌ pre-oneTBB, bundled works → bundled | ## Requires The matching build-env-images env (`TBB_INC`/`TBB_LIB`) extended to all ubuntu + el10: build-env-images #13. Ship together. ## Verified RcppParallel + rstan build and link (no `symbol not found`) on alpine, redhat-10, ubuntu noble and ubuntu resolute; RcppParallel builds on ubuntu jammy. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/137 --- local/patches/registry.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/local/patches/registry.json b/local/patches/registry.json index c446fda..b444986 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -4,13 +4,14 @@ "versions": "*", "platforms": [ "alpine", - "ubuntu-2604" + "ubuntu", + "redhat-10" ], "env": {}, "configure_args": [], "makevars": {}, "patch": "RcppParallel/system-tbb.patch", - "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); link the system oneTBB (now in the build-env images) instead, keeping the TBB backend so dependents (rstan, ...) link TBB" + "reason": "RcppParallel bundles an old Intel TBB whose build fails on musl and modern toolchains (g++ 14/15); link the system oneTBB (2021+) shipped in the build-env images instead, keeping the TBB backend so dependents (rstan, ...) link TBB. Scoped to platforms with oneTBB 2021+ (alpine, all ubuntu, el10); el8/el9 ship pre-oneTBB TBB and are left on the bundled build." }, { "package": "fs", From 71f3f0c6011576c751fda00e64fffd059e26ab6e Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 21 Jul 2026 08:51:23 +0000 Subject: [PATCH 083/125] fix(audit): replace arch subsection in place instead of appending duplicates (#138) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem A fresh `weekly-audit-missing` run followed by `weekly-rebuild-missing` reported that almost every package "already exists in the remote bucket", even though the audit had just flagged them as missing. Root cause: the audit's Forgejo-issue update matched the existing `### ` subsection by its **bare** header (`### arm64`) while writing headers with a ` (N missing, M to rebuild)` suffix. The equality test never matched, so every run **appended** a new block instead of replacing the old one. Issue #63 had accumulated 38 arch subsections under `## alpine-323` where 2 are expected (75 total across the alpine platforms; body ~77k lines). `fetch-rebuild-packages-from-issue.R` reads the **first** matching block, which was the oldest snapshot. So the rebuild kept re-checking a months-old list (180 packages, mostly already built), while the genuinely-missing packages, ~1921 for alpine-323/arm64 in the freshest block, were never fed to the rebuild and the backlog grew silently. ## Changes - **`local/weekly-missing-binaries-audit.R`**: match arch subsections by prefix (`^### ( |$)`) and remove **all** blocks for that arch before writing one fresh block. Accumulation now self-heals on every run. - **`local/dedupe-audit-issue.R`** (new): one-off cleanup that collapses each `## platform` section to the freshest block per arch across the three OS-family issues. Supports `DRY_RUN=1`. - **`local/fetch-rebuild-packages-from-issue.R`**: prefer the audit's freshly-written RDS (overwritten each run, immune to issue-body drift), falling back to issue parsing when absent. ## Validation Simulated the dedupe logic against the live #63 body: **75 → 8** arch subsections, body 77k → 28k lines, and the kept alpine-323/arm64 block correctly resolves to `GARCH.X (3.0)` (the stale first block held `2.0`). ## Follow-ups (not in this PR) - `alpine-321` is audited but has no row in the rebuild matrix (7,230 missing, never rebuilt). - An `alpine-324` section exists in the issue but is in neither matrix. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/138 --- local/dedupe-audit-issue.R | 150 ++++++++++++++++++++++ local/fetch-rebuild-packages-from-issue.R | 31 ++++- local/weekly-missing-binaries-audit.R | 54 ++++---- 3 files changed, 211 insertions(+), 24 deletions(-) create mode 100644 local/dedupe-audit-issue.R diff --git a/local/dedupe-audit-issue.R b/local/dedupe-audit-issue.R new file mode 100644 index 0000000..c78015a --- /dev/null +++ b/local/dedupe-audit-issue.R @@ -0,0 +1,150 @@ +# One-off maintenance: collapse the duplicate arch subsections that accumulated +# in the "Missing package binaries for latest version ()" issues. +# +# A bug in weekly-missing-binaries-audit.R matched the existing "### " +# subsection by its bare header while writing headers with a +# " (N missing, M to rebuild)" suffix, so every audit run appended a fresh block +# instead of replacing it. This script rewrites each "## " section to +# keep only the *last* (freshest) block per arch. The audit fix prevents further +# accumulation; this cleans up what is already there. +# +# Env: FORGEJO_TOKEN (required). DRY_RUN=1 to preview counts without patching. + +library(httr2, quietly = TRUE) + +forgejo_base <- "https://git.devxy.io/api/v1" +repo <- "devxy/build-cran-binaries" +token <- Sys.getenv("FORGEJO_TOKEN") +dry_run <- nchar(Sys.getenv("DRY_RUN")) > 0 + +if (nchar(token) == 0) { + stop("FORGEJO_TOKEN env var is not set") +} + +issue_titles <- c( + "Missing package binaries for latest version (Alpine)", + "Missing package binaries for latest version (Ubuntu)", + "Missing package binaries for latest version (Red Hat)" +) + +# Collapse one "## " block: keep only the last block per arch, +# emitted in order of first appearance. `pl[1]` is the "## " header. +dedupe_platform <- function(pl) { + sub_hdr <- which(grepl("^### ", pl)) + if (length(sub_hdr) == 0) { + return(pl) + } + preamble <- pl[seq_len(sub_hdr[1] - 1)] + sub_end <- c(sub_hdr[-1] - 1, length(pl)) + blocks <- lapply(seq_along(sub_hdr), function(k) { + pl[seq(sub_hdr[k], sub_end[k])] + }) + arches <- vapply( + blocks, + function(b) sub("^### (\\S+).*", "\\1", b[1]), + character(1) + ) + # Index of the last block for each arch, kept in first-appearance order. + last_idx <- vapply( + unique(arches), + function(a) max(which(arches == a)), + integer(1) + ) + keep <- sort(last_idx) + out <- preamble + for (i in keep) { + out <- c(out, blocks[[i]]) + } + out +} + +process_issue <- function(title) { + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(title, reserved = TRUE) + ) + resp <- request(search_url) |> + req_headers(Authorization = paste("token", token)) |> + req_perform() + issues <- resp_body_json(resp, simplifyVector = FALSE) + match_idx <- which(vapply(issues, function(x) x$title, character(1)) == title) + if (length(match_idx) == 0) { + cat(sprintf("[skip] No issue found: %s\n", title)) + return(invisible()) + } + + issue_number <- issues[[match_idx[1]]]$number + body <- issues[[match_idx[1]]]$body + if (is.null(body) || nchar(body) == 0) { + cat(sprintf("[skip] Empty body: #%d %s\n", issue_number, title)) + return(invisible()) + } + + lines <- strsplit(body, "\n", fixed = TRUE)[[1]] + before <- sum(grepl("^### ", lines)) + + # Split off the "## Excluded packages" footer so it is preserved verbatim. + excl_idx <- which(lines == "## Excluded packages") + footer <- character(0) + if (length(excl_idx) > 0) { + pre_dash <- which(lines == "---" & seq_along(lines) < excl_idx[1]) + cut <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_idx[1] + footer <- lines[seq(cut, length(lines))] + lines <- lines[seq_len(cut - 1)] + } + + # Platform headers ("## "); everything before the first is preamble. + plat_idx <- which(grepl("^## ", lines)) + if (length(plat_idx) == 0) { + cat(sprintf("[skip] No platform sections: #%d %s\n", issue_number, title)) + return(invisible()) + } + top <- lines[seq_len(plat_idx[1] - 1)] + plat_end <- c(plat_idx[-1] - 1, length(lines)) + + new_lines <- top + for (j in seq_along(plat_idx)) { + pl <- lines[seq(plat_idx[j], plat_end[j])] + new_lines <- c(new_lines, dedupe_platform(pl)) + } + if (length(footer) > 0) { + new_lines <- c(new_lines, footer) + } + + after <- sum(grepl("^### ", new_lines)) + cat(sprintf( + "#%d %s: %d -> %d arch subsections%s\n", + issue_number, + title, + before, + after, + if (dry_run) " (dry run, not patched)" else "" + )) + + if (dry_run) { + return(invisible()) + } + + patch_url <- sprintf( + "%s/repos/%s/issues/%d", + forgejo_base, + repo, + issue_number + ) + request(patch_url) |> + req_headers( + Authorization = paste("token", token), + `Content-Type` = "application/json" + ) |> + req_body_json(list(body = paste(new_lines, collapse = "\n"))) |> + req_method("PATCH") |> + req_perform() + cat(sprintf(" patched #%d\n", issue_number)) +} + +for (t in issue_titles) { + process_issue(t) +} +cat("Done.\n") diff --git a/local/fetch-rebuild-packages-from-issue.R b/local/fetch-rebuild-packages-from-issue.R index 26e544d..9d8043c 100644 --- a/local/fetch-rebuild-packages-from-issue.R +++ b/local/fetch-rebuild-packages-from-issue.R @@ -4,7 +4,6 @@ forgejo_base <- "https://git.devxy.io/api/v1" repo <- "devxy/build-cran-binaries" platform <- Sys.getenv("PLATFORM") arch <- Sys.getenv("ARCH") -token <- Sys.getenv("FORGEJO_TOKEN") output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") if (nchar(platform) == 0) { @@ -13,6 +12,36 @@ if (nchar(platform) == 0) { if (nchar(arch) == 0) { stop("ARCH env var is not set") } + +# Prefer the audit's freshly-written RDS. The audit overwrites it each run +# (saveRDS), so unlike the Forgejo issue body it is never subject to the +# duplicate-subsection accumulation bug. Fall back to parsing the issue when the +# RDS is absent (e.g. a fresh runner with no shared cache). +rds_file <- file.path( + Sys.getenv("REBUILD_PKG_RDS_DIR", "/mnt/cache/packages"), + sprintf("weekly_rebuild_%s_%s.rds", platform, arch) +) +if (file.exists(rds_file)) { + pkgs <- tryCatch(as.character(readRDS(rds_file)), error = function(e) NULL) + if (!is.null(pkgs) && length(pkgs) > 0) { + cat(sprintf( + "Using audit RDS %s: %d rebuildable packages for %s/%s\n", + rds_file, + length(pkgs), + platform, + arch + )) + writeLines(pkgs, output_file) + cat(sprintf("Wrote package list to %s\n", output_file)) + q("no") + } + cat(sprintf( + "RDS %s present but empty/unreadable -- falling back to issue\n", + rds_file + )) +} + +token <- Sys.getenv("FORGEJO_TOKEN") if (nchar(token) == 0) { stop("FORGEJO_TOKEN env var is not set") } diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index 6b32037..7b0b65d 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -328,7 +328,6 @@ if (nchar(forgejo_token) == 0) { } plat_header <- sprintf("## %s", platform) - arch_header <- sprintf("### %s", arch) plat_idx <- which(lines == plat_header) @@ -359,31 +358,40 @@ if (nchar(forgejo_token) == 0) { } plat_lines <- lines[seq(pi, plat_end)] - arch_local_idx <- which(plat_lines == arch_header) - if (length(arch_local_idx) == 0) { - # Append arch subsection at end of platform block - lines <- c( - lines[seq_len(plat_end)], - "", - arch_lines, - lines[seq(plat_end + 1, length(lines))] - ) + # Arch subsection headers within the platform block (### arm64 / ### amd64). + # Match by prefix: headers carry a " (N missing, M to rebuild)" suffix, so + # exact-equality matching never found the existing block and silently + # appended a duplicate on every run. Remove *all* blocks for this arch + # (collapsing any previously accumulated duplicates), then write one fresh + # block, so the issue holds a single current subsection per arch. + sub_hdr <- which(grepl("^### ", plat_lines)) + arch_re <- sprintf("^### %s( |$)", arch) + + if (length(sub_hdr) == 0) { + # No arch subsections yet -- append after the platform header/preamble. + new_plat_lines <- c(plat_lines, "", arch_lines) } else { - ai <- pi + arch_local_idx[1] - 1 # absolute line index - - # End of arch subsection - next_arch <- which( - grepl("^### |^## |^---", lines) & seq_along(lines) > ai - ) - arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end - - lines <- c( - lines[seq_len(ai - 1)], - arch_lines, - lines[seq(arch_end + 1, length(lines))] - ) + preamble <- plat_lines[seq_len(sub_hdr[1] - 1)] + # Each subsection runs from its ### header to the line before the next + # ### header (#### known-failures stays inside its own block). + sub_end <- c(sub_hdr[-1] - 1, length(plat_lines)) + kept <- character(0) + for (k in seq_along(sub_hdr)) { + block <- plat_lines[seq(sub_hdr[k], sub_end[k])] + if (!grepl(arch_re, block[1])) { + kept <- c(kept, block) + } + } + new_plat_lines <- c(preamble, kept, "", arch_lines) } + + tail_lines <- if (plat_end < length(lines)) { + lines[seq(plat_end + 1, length(lines))] + } else { + character(0) + } + lines <- c(lines[seq_len(pi - 1)], new_plat_lines, tail_lines) } # Rebuild excluded footer From 736564a4ff39c7f46b0c72a6bc742062deca9694 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 24 Jul 2026 11:18:28 +0200 Subject: [PATCH 084/125] fix: prevent renovate bumps for fixed matrix assignments --- .crow/process-updates.yaml | 8 ++++---- .crow/trial-build-registry.yaml | 8 ++++---- renovate.json | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 3da41fd..59b1c26 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -47,25 +47,25 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 OS_ID: alpine322 PROCESS_NEW: "FALSE" - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 OS_ID: alpine322 PROCESS_NEW: "FALSE" - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 OS_ID: alpine323 PROCESS_NEW: "FALSE" - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 OS_ID: alpine323 PROCESS_NEW: "FALSE" - OS: alpine-324 diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index cb8df8b..73856a1 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -29,19 +29,19 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/renovate.json b/renovate.json index 53c7306..8c8e208 100644 --- a/renovate.json +++ b/renovate.json @@ -1,7 +1,7 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": ["local>devxy/renovate-config"], - "ignorePaths": ["docker/**"], + "ignorePaths": ["docker/**", ".crow/process-updates.yaml", ".crow/build-all-versions.yaml", ".crow/weekly-rebuild-missing.yaml"], "customManagers": [ { "customType": "regex", From 36c963eac9b13eb525a4aa2a14f5dea2494cfe28 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 28 Jul 2026 00:32:39 +0000 Subject: [PATCH 085/125] chore(deps): update pre-commit hook davidanson/markdownlint-cli2 to v0.23.2 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0f4422..df8af14 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.23.0 + rev: v0.23.2 hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier From adcc74ba4fec1c02d51778c36532cd72058f74ce Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 28 Jul 2026 00:32:55 +0000 Subject: [PATCH 086/125] chore(deps): update pre-commit hook rbubley/mirrors-prettier to v3.9.6 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df8af14..cee748e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.4 + rev: v3.9.6 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit From c7293aefa1bde9ed214eec9702f63c25e4244f2f Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 28 Jul 2026 02:33:47 +0000 Subject: [PATCH 087/125] chore(deps): update terraform bunnynet to v0.15.2 --- .terraform.lock.hcl | 70 ++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 6ef10fa..b18d5af 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.15.1" + version = "0.15.2" constraints = "~> 0.15" hashes = [ - "h1:/2NUpbtjkc+w6n5V3kGP0rSzGjN0K2Wdfe2K+CZdmhU=", - "h1:1TOrpmCR0aT5xX1sjt70zqlYkMJnVe6r0nx2B0DS/mE=", - "h1:4uC8r8ILr+vZJ230uGqIUaWQ7uORCrIzoEuYrAq2JuE=", - "h1:6b+2osJUfwcaYZ6mathLPf/58sr/4XkLXQrcSufnkMk=", - "h1:CPldfjf79QS8mIP+GWoS0FVhrFlyjvN2+39zfyP76Ik=", - "h1:EvXICHyGIKpoYlDeKHOPzfmpvdRdhgsOMcR2nb6+tKY=", - "h1:GZLq+nDxS1CyBH0ELGTSQj9X7ozemJ1jpPA4KwbtR+c=", - "h1:ISNFqL745IQgZ6yMLy8ofV8ixbYqZYa9JKdi2W3pmNk=", - "h1:IrNrEuvFd0nYDGQefwmT8d1CSJb9e8LN5w9vw1ODp7E=", - "h1:Ms79slY9bZ94+n4cwIHvI9+/cvbucwo6S1+z5KAiznw=", - "h1:NWA9XSEBcpSkgwwIvl6tHrxGQY3uYhqNS4Vnb9RLyLQ=", - "h1:UjvxxxggicLtiE3yTe1Gx0oLUTeZpWmgIfXuHzWHn1c=", - "h1:VgJjo14DGkU4Jwo4D3GT4/5sq1tdjiZscKS5l3cb890=", - "h1:dBu3AW5YNLIvbBIMNk3wUHKw4TW+BDbj34a+mCqYhWE=", - "h1:i5oGD06nQ3JRsBIa2u4wCej+ETgp970CFl75dOKkHno=", - "h1:mTqR+vD1AWPx+mu7S0/pzBy71z7WOrjH7arOP77PXh4=", - "h1:nmTM61G8vYjpofeEqspMORpsNvTGCNZySGfjdXardL0=", - "zh:0f9bf5aaa47164a4d6ae4433d5e285a9456a5053401b2bad4ed68622f574ddee", - "zh:3039bee421fb8855a919f449fc731d145254371f5f0c39cc4660f3aeed6a8b10", - "zh:36664b08186e0c194747b18dee24ed97327c6e704133d4cf0df27abef1652f86", - "zh:3c7eae99d8c5ff65dfb99c8b9c1980147282d68971e1ef1ff0f126a6bff59d8f", - "zh:5293cc21abf54f4e5745437ca2d40d206aae323b2e1d41cf45dcc63a8868cbb3", - "zh:5994e5145e616e7e881010717e4c7def2945eb6d933f62db4ec3167732ccac84", - "zh:7994db9ed3fdb6cbf21f2154ea962cf82f04e425988bedc6657de2497d6cb6c3", + "h1:1OktRVcHWTvR/KKK4UWVm49JKibjyKOu0Epew+TawZI=", + "h1:4L1bPF0zW5z7Y6BEuPLXWP1RiOr/V8G5dhaRStV1yQM=", + "h1:4fpJGk4TRnWa0ZsNxBLWLlFgOb5WLyohpJ61Lsk5D0Y=", + "h1:7JTOjvrbATIKRCI+NKXCTMMSmUHYnKgsCsbKV+peAFM=", + "h1:8NDV2elfSxmixLUDqGpCSkKVm1pk7hgToP/L5bmEb4g=", + "h1:CYoaY53mSTjxq8nTMXuu7ET53q4Qe1+M6SfH9UaxE80=", + "h1:JbepdZF6DL97FUxGUI0Fi4sH9txTgzYk5q89Tzltifo=", + "h1:T6u0dqstAgR3L+8oPlNfcmOHVYZjIJ3UNvgdUyucpNA=", + "h1:U0Benusw0c0fJSdFvICBxuBxV1nAVOplY+AfLuda6ds=", + "h1:XZkiPAERPz4PKOMqohfMsA1qlnGhb2++R7rbMYAiDpo=", + "h1:Znc6swnxXdPeMarUgImUtbmw2iP83xVWUb4ShJC+F+U=", + "h1:aTJTUfP8tC71u1qgf4fqKv7RWd5dDTSpRQvD2Co2tWY=", + "h1:fozSGXAjeabAda2lqjXxqZ4haDE/QQAR1T6wV/t/NNw=", + "h1:pVBNrDoiSd5Pmf0lNa0uVFMl2u447kT0iXhDBT67mXk=", + "h1:rLnEQPCr3YbxzUWgmBITbyXzNaraTJGaW0mNEANzOPQ=", + "h1:wNSoVvpO1Y9vFTx3XbWYW5WSb2Go6EoRyci6l/kgBPE=", + "h1:x2gE2pPOUypQL5nCIak13MMqmGIlozwVpaaXfdZakSw=", + "zh:03acea3774550e25e7bf214ea3a8a9163b68d2009eb613c71d7f57c333cd8416", + "zh:13acba069e86d6a5d4d176952950c77fd8bd81629ed6b861349e5f74b0e78cb8", + "zh:275b9567ea05d5a5f0fb4d846032ed01482220a290aeb5d28e0bfa159779e3db", + "zh:444b008de5cbc7275f6d4230219a464e7b97daeea938223d693a3161c1158caf", + "zh:533b4cea6b5aa2351335453f45406eadb6d4485d5c339a0d5d5e0d84df21c22b", + "zh:55f0323631a34d1aa96178d08ec7eb7ec317306886daa00fd3a7dca61b00315b", + "zh:6e077b28e89f4364bbddf4dd783fc657c13df7d1a385f04629ba8deb1eb0a332", + "zh:6e10e279e7c1d44d3fe287c033d10068b5194621f6da03f28fa7b91fa4f2ef19", + "zh:74c1d79356f2c213ef19c7b478162c4f77ab8b840438b83701088554bd4090a1", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:93aa863e536ba9376ccf9e614e9edc9b214a2ce8c4316d416a8e249b436f52d2", - "zh:ab5cb4baeda57559686a0ccf0e09158aa64624ee6ba0ef32b769f13b11a43068", - "zh:ae9388b62eede8fd9272407bf75f8241a965bd489d45ec9dd3f9fac696d500e1", - "zh:caa5befd16960e2f69c7ec483e228e5ff43ab0979c17f1b874c9ffaa1c7c0e43", - "zh:cddd3e1067defa06a4e4ad5cb3940c7943e29c42417de57236aa7d3e2aeaae13", - "zh:cdfa44d591d0805116159556947904d70f534c6816188c45cf3a7544d2722ac9", - "zh:d607e9f1f3e09f13404f219e1893e3b3c77aece4afb54e999f934c021f41f576", - "zh:d8a397aca95125c6a0c0c78d2ded5843b9204effa9f5cf7419f017b500ad9228", - "zh:f5499eaff0d221725ad209d27d87c5b46d5c554caad7dc42947c760377abe3b0", - "zh:fc5f5cf433abc83e5169fa222992ec521c0c802075970251e3dd2c1d50c4f5c1", + "zh:9ebabdd167d1fd13d357517007027ad34acb7db130fdc75fd3b2652560c0399e", + "zh:a185124d2b93dff80203074c0f4a9dea8f21965c3cbc0592929b4ae58692b045", + "zh:a46483a7cd69699488c82406519769361b65eb59c39ab1bfd42f4d84d1bd30fd", + "zh:b29e2807adf9a0965f33246c59eb04bc2ac089fe44f2e439ece7c53bdee033a1", + "zh:b550a4c5eb7b17ca85b2f76c4b10bf0523b9c22f6104fe9fd5dce7b3c3bbbf37", + "zh:b9a9685c9fa99674ddbb11633bb9648152f8fbf92881deae1dd28ebca95b150e", + "zh:e412569cf41ebb5106d034f148df3968fafcf75de804713a20bee7d3d4e74c36", + "zh:fb639b074ee65c08590b971866e10e61aa85c7937a7b7f941899ac8cbcd9157d", ] } From 463054625afaa404a8f5a93e362957785836efab Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 28 Jul 2026 02:34:05 +0000 Subject: [PATCH 088/125] chore(deps): update pre-commit hook posit-dev/air-pre-commit to v0.11.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cee748e..140ae88 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit - rev: 0.10.0 + rev: 0.11.0 hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker From 8d0c4b7ccdca01863be1c6976d154a0074c94a36 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 28 Jul 2026 06:57:08 +0000 Subject: [PATCH 089/125] fix(patches): widen RcppParallel system-tbb scope to the whole redhat family (#139) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends #137. el8/el9 were excluded because they ship only classic Intel TBB. build-env-images now vendors oneTBB 2021 into `/usr/local` on el8/el9, so every platform has oneTBB 2021+. Widen the scope `redhat-10` → `redhat` (family token covers el8/el9/el10) and fix the reason text (no longer 'left on bundled'). **Requires the el8/el9 images rebuilt with the vendored oneTBB (build-env-images PR).** Verified: patched RcppParallel builds + links the vendored oneTBB on both el8 (g++ 8.5) and el9 (g++ 11.5). Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/139 --- local/patches/registry.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/local/patches/registry.json b/local/patches/registry.json index b444986..99b80e3 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -5,13 +5,13 @@ "platforms": [ "alpine", "ubuntu", - "redhat-10" + "redhat" ], "env": {}, "configure_args": [], "makevars": {}, "patch": "RcppParallel/system-tbb.patch", - "reason": "RcppParallel bundles an old Intel TBB whose build fails on musl and modern toolchains (g++ 14/15); link the system oneTBB (2021+) shipped in the build-env images instead, keeping the TBB backend so dependents (rstan, ...) link TBB. Scoped to platforms with oneTBB 2021+ (alpine, all ubuntu, el10); el8/el9 ship pre-oneTBB TBB and are left on the bundled build." + "reason": "RcppParallel bundles an old Intel TBB whose build fails on musl and modern toolchains (g++ 8-15 + modern make); link the system oneTBB (2021+) instead, keeping the TBB backend so dependents (rstan, ...) link TBB. All build-env platforms now provide oneTBB 2021+: native on alpine/ubuntu/el10, built from source into /usr/local on el8/el9 (their stock TBB is classic 2018/2020, too old)." }, { "package": "fs", From 86707e3de4f69e912c2e060386f08dbcd4f25cb9 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Fri, 31 Jul 2026 00:32:20 +0000 Subject: [PATCH 090/125] chore(deps): update terraform bunnynet to ~> 0.16 --- .terraform.lock.hcl | 72 ++++++++++++++++++++++----------------------- provider.tf | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index b18d5af..fdf41e1 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.15.2" - constraints = "~> 0.15" + version = "0.16.0" + constraints = "~> 0.16" hashes = [ - "h1:1OktRVcHWTvR/KKK4UWVm49JKibjyKOu0Epew+TawZI=", - "h1:4L1bPF0zW5z7Y6BEuPLXWP1RiOr/V8G5dhaRStV1yQM=", - "h1:4fpJGk4TRnWa0ZsNxBLWLlFgOb5WLyohpJ61Lsk5D0Y=", - "h1:7JTOjvrbATIKRCI+NKXCTMMSmUHYnKgsCsbKV+peAFM=", - "h1:8NDV2elfSxmixLUDqGpCSkKVm1pk7hgToP/L5bmEb4g=", - "h1:CYoaY53mSTjxq8nTMXuu7ET53q4Qe1+M6SfH9UaxE80=", - "h1:JbepdZF6DL97FUxGUI0Fi4sH9txTgzYk5q89Tzltifo=", - "h1:T6u0dqstAgR3L+8oPlNfcmOHVYZjIJ3UNvgdUyucpNA=", - "h1:U0Benusw0c0fJSdFvICBxuBxV1nAVOplY+AfLuda6ds=", - "h1:XZkiPAERPz4PKOMqohfMsA1qlnGhb2++R7rbMYAiDpo=", - "h1:Znc6swnxXdPeMarUgImUtbmw2iP83xVWUb4ShJC+F+U=", - "h1:aTJTUfP8tC71u1qgf4fqKv7RWd5dDTSpRQvD2Co2tWY=", - "h1:fozSGXAjeabAda2lqjXxqZ4haDE/QQAR1T6wV/t/NNw=", - "h1:pVBNrDoiSd5Pmf0lNa0uVFMl2u447kT0iXhDBT67mXk=", - "h1:rLnEQPCr3YbxzUWgmBITbyXzNaraTJGaW0mNEANzOPQ=", - "h1:wNSoVvpO1Y9vFTx3XbWYW5WSb2Go6EoRyci6l/kgBPE=", - "h1:x2gE2pPOUypQL5nCIak13MMqmGIlozwVpaaXfdZakSw=", - "zh:03acea3774550e25e7bf214ea3a8a9163b68d2009eb613c71d7f57c333cd8416", - "zh:13acba069e86d6a5d4d176952950c77fd8bd81629ed6b861349e5f74b0e78cb8", - "zh:275b9567ea05d5a5f0fb4d846032ed01482220a290aeb5d28e0bfa159779e3db", - "zh:444b008de5cbc7275f6d4230219a464e7b97daeea938223d693a3161c1158caf", - "zh:533b4cea6b5aa2351335453f45406eadb6d4485d5c339a0d5d5e0d84df21c22b", - "zh:55f0323631a34d1aa96178d08ec7eb7ec317306886daa00fd3a7dca61b00315b", - "zh:6e077b28e89f4364bbddf4dd783fc657c13df7d1a385f04629ba8deb1eb0a332", - "zh:6e10e279e7c1d44d3fe287c033d10068b5194621f6da03f28fa7b91fa4f2ef19", - "zh:74c1d79356f2c213ef19c7b478162c4f77ab8b840438b83701088554bd4090a1", + "h1:+7y/HM0f4/cm7dpL5RulSZ3k5EDokVaIutfuanc1hhA=", + "h1:/4n/gUEzVa7LArNrNbZa4RyGC3+MRCCYTuqRqF36Z7U=", + "h1:4cSbrSalUxXnG0RVwrQG51NNleZNFU9GJxWCLK1M9WM=", + "h1:AlidPvq+LqsveQVhc9JYa7OKf4mcR+DjCzMomoBcmkw=", + "h1:Beny1tSl5O/jM7sSP7qYv+NEVTTyS/BGtSxvMFsII/o=", + "h1:E50kVGCtxWYoJgOLtdat1JFc9KCIeMByk65XSIpuaB0=", + "h1:ErNz5hzwf70NouW7a1DAknWI/0/BFUEX53hQ4BnUxWg=", + "h1:GTB8ugjpBEsRZpdVmKJw70IF2B1/fd1Gwven6XKghpk=", + "h1:JEEDgEc2gwjwfvnDwWO2ZHaf9T5WHtb7A6ABx6rQqhE=", + "h1:OVd3EgjJDuyMBFQ4XZYrs09NNEtFw4jGAbwyeos2H3M=", + "h1:U/9QeqZ+Q2MQ7eNfD1eWomLFlbYGwpliHcD/UTFe/D0=", + "h1:VPUe+m5Fx31TAR7udJPGGzk3EpCNizf0TmnKplOJyRM=", + "h1:YkJwUFpUemqi6bwRIEJW4xSpI7CAEXirqhyjl6Rmm68=", + "h1:a/15E8lijUcqvhUGxtqKmJT6jYGmQW9z7yBoOjusDm8=", + "h1:hii960NFcNpr0eZ/dcbnYOf2oZuo2IZl+z1IMTwh5YA=", + "h1:mq/KWNt6PPZLdNpO1ekzrn6EfWz6l8sTHG74dgy07nI=", + "h1:vso6Er2XV/MsUTMtXIlz0AhIFx0TZnK73YNSZfA/9ZY=", + "zh:1f581ed8bc8b676f0127af7c018912ca82972e9348370d9f4dbe5b44005bb0ec", + "zh:1fa5b9c01ffbc7fdc507bbcffc177a1f1deb9f56f3d27183c6d92a11ea3208f8", + "zh:42b6524a9658df7093199b65eb210b3246a9f9822c8778a77e2d00a222441c03", + "zh:4c4f17fe2ae86087626dfd1055b0faea6a23fe3a12aabb2a7de3b9b2df6e3e41", + "zh:5346bb67d7154a8bc8c887b4b86bef95b3e92857d4304f55afea6d56ed152932", + "zh:591df2c3d4552f6f613415c900023d43e73e00e90312b7b1a666fe567219120f", + "zh:7156849aa03348c7556b746d3d9956293678de4a9e30c80480358dcdacb2a7fa", + "zh:75dee86262352f061ad072a5b873d3b3eabbb124ab0f20a803cfd44a87a31067", + "zh:7c33954aff1878bec78a50742897fae78dd2fbd19abeaeaca0c560ccc8f78810", + "zh:7d9cf36633dd991b41891d1c91b45b5c803852a902705453d6be4513524a574c", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9ebabdd167d1fd13d357517007027ad34acb7db130fdc75fd3b2652560c0399e", - "zh:a185124d2b93dff80203074c0f4a9dea8f21965c3cbc0592929b4ae58692b045", - "zh:a46483a7cd69699488c82406519769361b65eb59c39ab1bfd42f4d84d1bd30fd", - "zh:b29e2807adf9a0965f33246c59eb04bc2ac089fe44f2e439ece7c53bdee033a1", - "zh:b550a4c5eb7b17ca85b2f76c4b10bf0523b9c22f6104fe9fd5dce7b3c3bbbf37", - "zh:b9a9685c9fa99674ddbb11633bb9648152f8fbf92881deae1dd28ebca95b150e", - "zh:e412569cf41ebb5106d034f148df3968fafcf75de804713a20bee7d3d4e74c36", - "zh:fb639b074ee65c08590b971866e10e61aa85c7937a7b7f941899ac8cbcd9157d", + "zh:9552ed1be3059b31544632d6293c2bfcd2474983e116c42702a6c408deb8e44d", + "zh:b3cb1f2bde0fd591d18109119f6235ff41c7ea6441777a356a64c348ee4ccdf7", + "zh:b5c1748403bf43274eba2574219cfa082351c8b1c1537a768acebef5cc5b2c0a", + "zh:c1b032e33aee40945770bbdf6002f777ee7b4ca541c00f4cf5eb23c09d32ef54", + "zh:d469680df9845fb506be27b2c838d39ccd8e98e26540b99936775f954299c35c", + "zh:dd59fa97fea4b10b901a6a749ebff81895f54126a2f08b4cd8f5baf6cfbbd854", + "zh:fb142d8ea02909587a4858374fe33d8609347150f4d32b7e15423320ffbd07b1", ] } diff --git a/provider.tf b/provider.tf index b267fcc..c22ce2d 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.15" + version = "~> 0.16" } } } From 98371fb9c585d9e30439c97ad08bec3d61993aff Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 31 Jul 2026 06:41:46 +0000 Subject: [PATCH 091/125] fix(patches): replace the RcppParallel system-TBB patch with a link-order fix (#145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Every build logs: ``` ! Patch for RcppParallel 6.2.0 did not apply cleanly; skipping patched build. ``` RcppParallel 6.x rewrote its build system. `src/Makevars.in` no longer contains the `USE_TBB=Linux` block that `system-tbb.patch` edited (it is now a short `@VAR@` template driven by `tools/config/configure.R`), so the patch can never apply again. The workaround it implemented is obsolete too. 6.x bundles **oneTBB 2022** and builds it with cmake, which works on musl and with g++ 8-15, so the 5.x reason for linking a system TBB is gone. Linking one is now harmful: `install.libs.R` symlinks the system libraries into `RcppParallel/lib`, so the published binary depends on a TBB the consumer does not have — the same failure mode as `fs` and libuv. What is still broken upstream is the **link order**. `configure.R` names the TBB directory with `-Wl,-L`, and gcc expands its own search dirs into `-L` options ahead of anything forwarded verbatim with `-Wl,`: ``` $ gcc -v -o t t.c -Wl,-L/usr/local/lib64 -ltbb -L/usr/lib/gcc/x86_64-redhat-linux/8 -L/lib/../lib64 -L/usr/lib/../lib64 ... -L/usr/local/lib64 ``` So on any build host with a distro TBB installed, `-ltbb` resolves to that library and `RcppParallel.so` records *its* SONAME (`libtbb.so.12`, or `libtbb.so.2` for the classic Intel TBB on el8/el9) instead of the bundled `libtbb.so`. ## Changes - Replace `RcppParallel/system-tbb.patch` with `RcppParallel/bundled-tbb-link-order.patch`, which changes the three `-Wl,-L` occurrences in `tools/config/configure.R` to plain `-L`. - Scope the RcppParallel entry to `>=6.0.0` and widen `platforms` to `*`. - Drop the `rstan` entry: its `-DTBB_INTERFACE_NEW` is already emitted by `RcppParallel::CxxFlags()` once the bundled oneTBB is used, and its `-I/usr/local/include` was an el8/el9 path applied on every platform. - `.pre-commit-config.yaml`: exclude `local/patches/*.patch` from `trailing-whitespace`, `end-of-file-fixer` and `editorconfig-checker`. They rewrite blank context lines (` ` -> ``) in every diff in the registry; `git apply` happens to tolerate it today, but a patch with meaningful trailing whitespace would be silently corrupted. The excludes are per-hook so `validate patch registry` still runs. - README: the RcppParallel example described the 5.x problem. ## Verification Built in the published images: | image | NEEDED | RPATH | loads with system libtbb removed | | --- | --- | --- | --- | | `build-env-alpine:3.24` | `libtbb.so` | `$ORIGIN/../lib` | yes | | `build-env-redhat:8` | `libtbb.so`, `libtbbmalloc.so` | `$ORIGIN/../lib` | yes | | `build-env-ubuntu:noble` | `libtbb.so` | `$ORIGIN/../lib` | yes | Without the patch the same builds record `libtbb.so.12` (alpine, ubuntu) or `libtbb.so.2` (el8, the classic 2018 TBB) and fall back to the system library at load time. rstan 2.32.7 compiles and loads against the patched RcppParallel on ubuntu noble with no makevars override and with the system libtbb moved away. On el8 it also compiles; loading it there is blocked by an unrelated image bug (see below). ## Behaviour change Requires the paired build-env-images PR (drops `TBB_INC`/`TBB_LIB`) and an image rebuild — with those env vars set, configure still takes the system-TBB branch. After that, RcppParallel binaries ship their own oneTBB and are self-contained. ## Also found, not fixed here - The `uvr lock failed ... GLIBC_2.29 not found` errors in the same log are the `-gnu` uvr artifact on el8's glibc 2.28; fixed in build-env-images. - `build-env-redhat:8` R 4.5.3/4.6.0 cannot load `stats` (`libRlapack.so: undefined symbol: dgemmtr_`): the el8 R RPM symlinks `libRblas.so` to openblas 0.3.15, which predates that symbol. R 4.4.3 and el9 are fine. Belongs in the R RPM build. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/145 --- .pre-commit-config.yaml | 9 ++++ README.md | 2 +- .../RcppParallel/bundled-tbb-link-order.patch | 46 +++++++++++++++++++ local/patches/RcppParallel/system-tbb.patch | 19 -------- local/patches/registry.json | 30 ++---------- 5 files changed, 61 insertions(+), 45 deletions(-) create mode 100644 local/patches/RcppParallel/bundled-tbb-link-order.patch delete mode 100644 local/patches/RcppParallel/system-tbb.patch diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 140ae88..80650cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,12 +7,20 @@ exclude: | benchmark/| docker/reprex/ ) +# The `^local/patches/.*\.patch$` excludes below keep unified diffs byte-exact: +# a context line for a blank line is a single space, and stripping it (or +# appending a newline) makes `git apply` reject the patch, which surfaces as +# "patch did not apply cleanly" at build time rather than as a lint failure +# here. The exclusions are per-hook, not global, so `validate patch registry` +# still runs when a patch changes. repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: end-of-file-fixer + exclude: ^local/patches/.*\.patch$ - id: trailing-whitespace + exclude: ^local/patches/.*\.patch$ args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 @@ -31,6 +39,7 @@ repos: rev: v3.8.0 hooks: - id: editorconfig-checker + exclude: ^local/patches/.*\.patch$ - repo: https://github.com/adrienverge/yamllint.git rev: v1.38.0 hooks: diff --git a/README.md b/README.md index 28541a1..538df34 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ For every package+tag combination: ## Patching packages Some CRAN packages fail to compile on specific platforms due to compiler- or OS-specific issues unrelated to the package itself. -The canonical example is `RcppParallel`, whose bundled TBB sources fail on musl (Alpine) and newer compiler/OS combinations. +The canonical example is `RcppParallel`, whose bundled TBB is linked in a way that lets a system TBB on the build host shadow it, so the published binary depends on a library the consumer does not have. Because such packages are often transitive dependencies of many others, a single failure cascades: all dependents fail even though nothing is wrong with the dependent itself. To address this, frequently-failing packages can be "patched" before they are installed — whether as a direct build target or a transitive dependency pulled in by `pak`. diff --git a/local/patches/RcppParallel/bundled-tbb-link-order.patch b/local/patches/RcppParallel/bundled-tbb-link-order.patch new file mode 100644 index 0000000..45114a8 --- /dev/null +++ b/local/patches/RcppParallel/bundled-tbb-link-order.patch @@ -0,0 +1,46 @@ +diff --git a/tools/config/configure.R b/tools/config/configure.R +index 6293fe1..2f84337 100644 +--- a/tools/config/configure.R ++++ b/tools/config/configure.R +@@ -186,12 +186,23 @@ define( + ) + + # set PKG_LIBS ++# ++# bincraft patch: the library directories below are passed as plain '-L', not ++# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) ++# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', ++# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build ++# host with a distro TBB installed, '-ltbb' resolves to that library and ++# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the ++# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads ++# the system TBB rather than the copy shipped in RcppParallel/lib, and fails ++# outright on a machine that has no system TBB. gcc places a plain '-L' before ++# its built-in dirs, so the intended library is found first. + pkgLibs <- if (!is.na(tbbLib)) { + + # a TBB supplied via TBB_LIB / TBB_ROOT. an rpath is meaningless on Windows, + # where the loader has no equivalent -- see R/zzz.R for how we resolve there + c( +- "-Wl,-L\"$(TBB_LIB)\"", ++ "-L\"$(TBB_LIB)\"", + if (.Platform$OS.type != "windows") + sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), + "-l$(TBB_NAME)", +@@ -201,14 +212,14 @@ pkgLibs <- if (!is.na(tbbLib)) { + } else if (R.version$os == "emscripten") { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)" + ) + + } else { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" + ) diff --git a/local/patches/RcppParallel/system-tbb.patch b/local/patches/RcppParallel/system-tbb.patch deleted file mode 100644 index 263a593..0000000 --- a/local/patches/RcppParallel/system-tbb.patch +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/src/Makevars.in b/src/Makevars.in -index be8445f..7cb6c9e 100644 ---- a/src/Makevars.in -+++ b/src/Makevars.in -@@ -60,7 +60,13 @@ else - endif - - ifeq ($(UNAME), Linux) -- USE_TBB=Linux -+ # bincraft patch: link the system oneTBB (installed in the build-env images) -+ # instead of building the bundled Intel TBB, which fails on musl (Alpine) and -+ # newer toolchains (g++ 15). Leaving USE_TBB unset skips the bundled build; -+ # -DRCPP_PARALLEL_USE_TBB=1 keeps the TBB backend so dependents (rstan, ...) -+ # link TBB, and -ltbb/-ltbbmalloc pull the system library from default paths. -+ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=1 -DTBB_SUPPRESS_DEPRECATED_MESSAGES=1 -DTBB_INTERFACE_NEW -+ PKG_LIBS += -ltbb -ltbbmalloc - endif - - ifeq ($(UNAME), SunOS) diff --git a/local/patches/registry.json b/local/patches/registry.json index 99b80e3..fe0d387 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -1,42 +1,22 @@ [ { "package": "RcppParallel", - "versions": "*", - "platforms": [ - "alpine", - "ubuntu", - "redhat" - ], + "versions": ">=6.0.0", + "platforms": ["*"], "env": {}, "configure_args": [], "makevars": {}, - "patch": "RcppParallel/system-tbb.patch", - "reason": "RcppParallel bundles an old Intel TBB whose build fails on musl and modern toolchains (g++ 8-15 + modern make); link the system oneTBB (2021+) instead, keeping the TBB backend so dependents (rstan, ...) link TBB. All build-env platforms now provide oneTBB 2021+: native on alpine/ubuntu/el10, built from source into /usr/local on el8/el9 (their stock TBB is classic 2018/2020, too old)." + "patch": "RcppParallel/bundled-tbb-link-order.patch", + "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake, which works on musl and g++ 8-15, so the system-TBB workaround needed for 5.x is gone. What remains broken is the link order: configure.R passes the TBB directory as '-Wl,-L', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,'. On a build host with a distro TBB installed, '-ltbb' therefore resolves to the system library and RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the classic Intel TBB on el8/el9) instead of the bundled 'libtbb.so'. The published binary then loads the system TBB rather than the copy in RcppParallel/lib and fails to dyn.load on a consumer machine without one, the same way fs did with libuv. Passing a plain '-L' puts the bundled build dir ahead of gcc's defaults; verified on alpine 3.24, el8 and ubuntu noble to produce NEEDED libtbb.so + RPATH $ORIGIN/../lib, loading with every system libtbb removed." }, { "package": "fs", "versions": "*", - "platforms": [ - "*" - ], + "platforms": ["*"], "env": {}, "configure_args": [], "makevars": {}, "patch": "fs/force-vendored-libuv.patch", "reason": "fs 2.x configure links system libuv whenever pkg-config finds libuv-devel (installed as a build-time sysreq), producing an fs.so with NEEDED libuv.so.1. That binary fails to dyn.load on consumer machines lacking runtime libuv, because install.packages/renv do not install SystemRequirements (only pak does, and only in the build container). The patch short-circuits configure to copy src/Makevars.vendor and build the bundled static libuv (needs cmake) so the binary is self-contained on every platform. An env/pkg-config override was tried first but the rebuilt binary still linked libuv.so.1, so a source patch is used instead." - }, - { - "package": "rstan", - "versions": "*", - "platforms": [ - "*" - ], - "env": {}, - "configure_args": [], - "makevars": { - "CPPFLAGS": "-DTBB_INTERFACE_NEW -I/usr/local/include" - }, - "patch": null, - "reason": "StanHeaders' init_threadpool_tbb.hpp unconditionally includes the legacy (removed in oneTBB 2021+) for version detection, breaking compilation of Module.cpp against the bundled oneTBB. Pre-defining TBB_INTERFACE_NEW skips that include and selects the modern tbb/global_control.h + tbb/task_arena.h path that the bundled TBB provides (-I/usr/local/include preserves the default CPPFLAGS the override replaces)" } ] From fe2289af56e4724200cd1ebc41e06e67f5c9cb71 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 31 Jul 2026 09:48:41 +0000 Subject: [PATCH 092/125] fix(patches): force RcppParallel to use its bundled oneTBB (#146) ## Why `RcppParallel` binaries built on ubuntu 22.04 still record `NEEDED libtbb.so.12` and cannot `dyn.load` without the distro TBB, although #145 landed the link-order patch and build-env-images dropped `TBB_INC`/`TBB_LIB` the same morning. The link-order fix does not survive a build image that exports those variables, and the published images still do: `.crow/images.yaml` in build-env-images triggers on `cron`/`manual` only, so every image in the registry is still the 2026-07-21 build. With `TBB_LIB` set, `configure.R` never reaches the bundled-oneTBB branch at all, `install.libs.R` fills `RcppParallel/lib` with absolute symlinks into the image's library dir, and the plain `-L` from #145 just points at the system TBB instead of the bundled one. Nothing on the pipeline side can undo this: R reads `~/.Renviron` **after** the process environment, so neither a Crow `environment:` block nor the registry `env` field wins. ``` # the image, not the Containerfile docker run --rm reg.devxy.io/rpkgs/build-env-ubuntu:jammy sh -lc 'grep TBB ~/.Renviron' TBB_INC=/usr/include/oneapi TBB_LIB=/usr/lib/x86_64-linux-gnu ``` Rather than couple correctness to image freshness for a third round-trip, the patch now forces the bundled TBB itself. ## Changes - `local/patches/RcppParallel/bundled-tbb-link-order.patch` -> `local/patches/RcppParallel/force-bundled-tbb.patch`. Keeps the `-Wl,-L` -> `-L` link-order hunk (the bundled branch has the same problem when a distro libtbb is installed) and adds an opt-out, `BINCRAFT_ALLOW_SYSTEM_TBB=TRUE`, for the ambient `TBB_ROOT`/`TBB_LIB`/`TBB_INC`. - All four read sites have to move together; patching only `configure.R` builds the bundled TBB and then **segfaults R on the post-install load test**: - `tools/config/configure.R`: branch selection. - `src/install.libs.R`: `R CMD INSTALL` runs it outside the `tbb` rule in `src/Makevars` that passes the configured values, so at install time it still sees the image environment. - `R/tbb.R` `tbbLibraryPath()`: run time. `.onLoad()` would `dyn.load` the system TBB into the global scope next to the bundled one that `RcppParallel.so` needs - two copies of the same symbols, hence the segfault. - `R/tbb.R` `tbbCxxFlags()` / `tbbLdFlags()`: otherwise dependents such as `rstan` get the system TBB handed back to them. - `local/patches/registry.json`: new patch path and a `reason` describing both failure modes. ## Verification Built through `bincraft:::prepare_patched_repo()` on the **current, unfixed** images (all three still export `TBB_INC`/`TBB_LIB`), with bincraft v5.0.1, i.e. what CI deploys: | image | `NEEDED` | rpath | `RcppParallel/lib` | load, all system `libtbb*` moved away | | --- | --- | --- | --- | --- | | `build-env-ubuntu:jammy` | `libtbb.so` | `$ORIGIN/../lib` | real `libtbb.so.2`, 5.0 MB | OK, 12 threads | | `build-env-redhat:9` | `libtbb.so` | `$ORIGIN/../lib` | real `libtbb.so.2`, 5.1 MB | OK, 12 threads | | `build-env-alpine:3.24` | `libtbb.so` | `$ORIGIN/../lib` | real `libtbb.so.2`, 4.7 MB | OK, 12 threads | For contrast, the same jammy build without this change: ``` NEEDED libtbb.so.12 RUNPATH /usr/lib/x86_64-linux-gnu:$ORIGIN/../lib lib/ libtbb.so.12.5 -> /usr/lib/x86_64-linux-gnu/libtbb.so.12.5 (dangling off the image) load libtbb.so.12: cannot open shared object file: No such file or directory ``` With `TBB_LIB` still exported, `RcppParallel::tbbLibraryPath()` resolves to the package's own `lib`, and `CxxFlags()` emits the package's own `include` plus `-DTBB_INTERFACE_NEW` - not `/usr/include/oneapi`. ## Behaviour change Published `RcppParallel` binaries carry their own oneTBB on every platform, whichever image version CI pulls. The patch content changed, so the patched-binary cache key changes with it and no stale entry is reused. The build-env-images fix is still worth rolling out (those images also carry the broken el8 `uvr` and uvr 0.4.1), but RcppParallel no longer waits on it. ## Follow-up, not in this PR The binaries already in B2 are the broken ones; they need a rebuild, and a Bunny `/purge` does not evict Perma-Cache. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/146 --- .../RcppParallel/bundled-tbb-link-order.patch | 46 ------ .../RcppParallel/force-bundled-tbb.patch | 154 ++++++++++++++++++ local/patches/registry.json | 4 +- 3 files changed, 156 insertions(+), 48 deletions(-) delete mode 100644 local/patches/RcppParallel/bundled-tbb-link-order.patch create mode 100644 local/patches/RcppParallel/force-bundled-tbb.patch diff --git a/local/patches/RcppParallel/bundled-tbb-link-order.patch b/local/patches/RcppParallel/bundled-tbb-link-order.patch deleted file mode 100644 index 45114a8..0000000 --- a/local/patches/RcppParallel/bundled-tbb-link-order.patch +++ /dev/null @@ -1,46 +0,0 @@ -diff --git a/tools/config/configure.R b/tools/config/configure.R -index 6293fe1..2f84337 100644 ---- a/tools/config/configure.R -+++ b/tools/config/configure.R -@@ -186,12 +186,23 @@ define( - ) - - # set PKG_LIBS -+# -+# bincraft patch: the library directories below are passed as plain '-L', not -+# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) -+# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', -+# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build -+# host with a distro TBB installed, '-ltbb' resolves to that library and -+# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the -+# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads -+# the system TBB rather than the copy shipped in RcppParallel/lib, and fails -+# outright on a machine that has no system TBB. gcc places a plain '-L' before -+# its built-in dirs, so the intended library is found first. - pkgLibs <- if (!is.na(tbbLib)) { - - # a TBB supplied via TBB_LIB / TBB_ROOT. an rpath is meaningless on Windows, - # where the loader has no equivalent -- see R/zzz.R for how we resolve there - c( -- "-Wl,-L\"$(TBB_LIB)\"", -+ "-L\"$(TBB_LIB)\"", - if (.Platform$OS.type != "windows") - sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), - "-l$(TBB_NAME)", -@@ -201,14 +212,14 @@ pkgLibs <- if (!is.na(tbbLib)) { - } else if (R.version$os == "emscripten") { - - c( -- "-Wl,-Ltbb/build/lib_release", -+ "-Ltbb/build/lib_release", - "-l$(TBB_NAME)" - ) - - } else { - - c( -- "-Wl,-Ltbb/build/lib_release", -+ "-Ltbb/build/lib_release", - "-l$(TBB_NAME)", - "-l$(TBB_MALLOC_NAME)" - ) diff --git a/local/patches/RcppParallel/force-bundled-tbb.patch b/local/patches/RcppParallel/force-bundled-tbb.patch new file mode 100644 index 0000000..c46a05e --- /dev/null +++ b/local/patches/RcppParallel/force-bundled-tbb.patch @@ -0,0 +1,154 @@ +diff --git a/R/aaa.R b/R/aaa.R +index 568a2aa..bfdab4c 100644 +--- a/R/aaa.R ++++ b/R/aaa.R +@@ -5,4 +5,21 @@ TBB_LIB <- "" + TBB_INC <- "" + + TBB_NAME <- "tbb" +-TBB_MALLOC_NAME <- "tbbmalloc" +\ No newline at end of file ++TBB_MALLOC_NAME <- "tbbmalloc" ++ ++# bincraft patch: our build images (and plenty of user environments) export ++# TBB_ROOT / TBB_LIB / TBB_INC. A binary we publish always carries its own ++# oneTBB in RcppParallel/lib -- see the companion changes in ++# tools/config/configure.R and src/install.libs.R -- so honouring those ++# variables at run time is actively harmful: .onLoad() would dyn.load() a ++# second, unrelated TBB into the process next to the bundled one (two copies ++# of the same symbols in the global scope, which segfaults R on load), and ++# RcppParallelLibs() / CxxFlags() would hand that system TBB to dependents ++# such as rstan, putting NEEDED libtbb.so.12 back into their binaries. Read ++# the variables only when explicitly opted back in. ++bincraftGetenv <- function(name, unset = "") { ++ if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") ++ Sys.getenv(name, unset = unset) ++ else ++ unset ++} +\ No newline at end of file +diff --git a/R/tbb.R b/R/tbb.R +index 6f6a745..e407986 100644 +--- a/R/tbb.R ++++ b/R/tbb.R +@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { + sysname <- Sys.info()[["sysname"]] + + # find root for TBB install +- tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) ++ tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) + if (is.null(name)) + return(tbbRoot) + +@@ -58,7 +58,7 @@ tbbCxxFlags <- function() { + flags <- c("-DRCPP_PARALLEL_USE_TBB=1") + + # if TBB_INC is set, apply those library paths +- tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) ++ tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) + if (!file.exists(tbbInc)) { + tbbInc <- system.file("include", package = "RcppParallel") + } +@@ -117,7 +117,7 @@ tbbLdFlags <- function() { + } + + # shortcut if TBB_LIB defined +- tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) ++ tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) + if (nzchar(tbbLib)) { + if (R.version$os == "emscripten") { + fmt <- "-L%1$s -l%2$s" +diff --git a/src/install.libs.R b/src/install.libs.R +index 3b3cfda..c0e6f3e 100644 +--- a/src/install.libs.R ++++ b/src/install.libs.R +@@ -477,6 +477,18 @@ prependFlags <- function(prependFlags, toFlags) { + tbbLib <- Sys.getenv("TBB_LIB") + tbbInc <- Sys.getenv("TBB_INC") + ++# bincraft patch: the companion change in tools/config/configure.R stops an ++# ambient TBB_LIB / TBB_INC from selecting a system TBB, but this script is ++# also run directly by `R CMD INSTALL` (not only through the `tbb` rule in ++# src/Makevars, which passes the configured values), so at install time it ++# still sees the image's environment and would symlink the system libraries ++# into RcppParallel/lib. Drop them here for the same reason, under the same ++# opt-out. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbLib <- "" ++ tbbInc <- "" ++} ++ + args <- commandArgs(trailingOnly = TRUE) + if (identical(args, "build")) { + if (nzchar(tbbLib) && nzchar(tbbInc)) { +diff --git a/tools/config/configure.R b/tools/config/configure.R +index 6293fe1..eae4aaa 100644 +--- a/tools/config/configure.R ++++ b/tools/config/configure.R +@@ -40,6 +40,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) + tbbLib <- Sys.getenv("TBB_LIB", unset = NA) + tbbInc <- Sys.getenv("TBB_INC", unset = NA) + ++# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Several of ++# our build images export these (a leftover from RcppParallel 5.x, whose ++# bundled Intel TBB would not build on musl or with modern g++), and any of ++# them switches the branches below to a system TBB. The published binary then ++# records NEEDED libtbb.so.12 (or libtbb.so.2 for the classic Intel TBB) and ++# gets a RcppParallel/lib full of absolute symlinks into the image's library ++# dir, so it cannot dyn.load on a consumer machine without that exact TBB. ++# 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, ++# so the bundled copy is always the right choice here; forcing it in the ++# package rather than relying on the image environment keeps the binary ++# correct whichever image version CI happens to pull. Set ++# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbRoot <- NA ++ tbbLib <- NA ++ tbbInc <- NA ++} ++ + tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") + tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") + +@@ -186,12 +204,23 @@ define( + ) + + # set PKG_LIBS ++# ++# bincraft patch: the library directories below are passed as plain '-L', not ++# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) ++# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', ++# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build ++# host with a distro TBB installed, '-ltbb' resolves to that library and ++# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the ++# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads ++# the system TBB rather than the copy shipped in RcppParallel/lib, and fails ++# outright on a machine that has no system TBB. gcc places a plain '-L' before ++# its built-in dirs, so the intended library is found first. + pkgLibs <- if (!is.na(tbbLib)) { + + # a TBB supplied via TBB_LIB / TBB_ROOT. an rpath is meaningless on Windows, + # where the loader has no equivalent -- see R/zzz.R for how we resolve there + c( +- "-Wl,-L\"$(TBB_LIB)\"", ++ "-L\"$(TBB_LIB)\"", + if (.Platform$OS.type != "windows") + sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), + "-l$(TBB_NAME)", +@@ -201,14 +230,14 @@ pkgLibs <- if (!is.na(tbbLib)) { + } else if (R.version$os == "emscripten") { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)" + ) + + } else { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" + ) diff --git a/local/patches/registry.json b/local/patches/registry.json index fe0d387..a7e29dd 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -6,8 +6,8 @@ "env": {}, "configure_args": [], "makevars": {}, - "patch": "RcppParallel/bundled-tbb-link-order.patch", - "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake, which works on musl and g++ 8-15, so the system-TBB workaround needed for 5.x is gone. What remains broken is the link order: configure.R passes the TBB directory as '-Wl,-L', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,'. On a build host with a distro TBB installed, '-ltbb' therefore resolves to the system library and RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the classic Intel TBB on el8/el9) instead of the bundled 'libtbb.so'. The published binary then loads the system TBB rather than the copy in RcppParallel/lib and fails to dyn.load on a consumer machine without one, the same way fs did with libuv. Passing a plain '-L' puts the bundled build dir ahead of gcc's defaults; verified on alpine 3.24, el8 and ubuntu noble to produce NEEDED libtbb.so + RPATH $ORIGIN/../lib, loading with every system libtbb removed." + "patch": "RcppParallel/force-bundled-tbb.patch", + "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, so the system-TBB workaround needed for 5.x is gone, but two things still steer the build back to a system TBB. (1) Ambient TBB_ROOT/TBB_LIB/TBB_INC: several build images still export these (build-env-images dropped them, but the images are rebuilt only by cron/manual runs, so a stale image keeps them), and R reads ~/.Renviron *after* the process environment, so no pipeline-side env override can undo it. configure.R then takes the system-TBB branch, install.libs.R symlinks the image's libraries into RcppParallel/lib as absolute paths, and the binary records NEEDED libtbb.so.12 (libtbb.so.2 for the classic Intel TBB on el8/el9) -- it cannot dyn.load on a consumer machine without that exact TBB. Both files are patched to ignore those variables (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE); install.libs.R needs it separately because R CMD INSTALL runs it outside the src/Makevars rule that passes the configured values. (2) Link order: the bundled branch passes its build dir as '-Wl,-Ltbb/build/lib_release', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,', so '-ltbb' would still resolve to a distro TBB when one is installed; a plain '-L' puts the bundled dir first. Verified on the current build-env-ubuntu:jammy image (which still exports TBB_INC/TBB_LIB): NEEDED libtbb.so, RUNPATH $ORIGIN/../lib, real libtbb.so.2 in RcppParallel/lib, and the package loads with every system libtbb moved away." }, { "package": "fs", From ee15c50f53996229451a44cfe12b2222853728e2 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 31 Jul 2026 12:19:26 +0000 Subject: [PATCH 093/125] refactor: migrate package installation from pak to uvr (#147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `bincraft` dropped pak in favour of uvr (5.0.x, "Dependencies and their system requirements are now installed with `uvr` instead of pak during `build_binary_package()`"), so the pipelines, helper scripts and images in this repo move with it. ## Approach uvr is project-scoped in a way pak is not: `uvr add` refuses to run outside a project and always installs into `.uvr/library/`, and only `uvr sync` honours `--library`. So there is no one-line `pak::pak(...)` equivalent. `local/uvr-install.sh` encapsulates the dance — bootstrap a pinned uvr, mint a throwaway project under `TMPDIR`, `uvr add --no-install`, then `uvr sync --library `. Keeping the project outside the checkout also keeps `uvr init`'s `.Rprofile` from hijacking `.libPaths()` for every other R call in the pipeline. This matches what bincraft itself does (`uvr sync --install-system-deps --library `), and bincraft requires `uvr` on `PATH`, which the bootstrap provides: every pipeline that calls `bincraft::` runs `install-bincraft.R` (and therefore the bootstrap) first. ## Changes | File | Change | | --- | --- | | `local/uvr-install.sh` | **New.** The single replacement for `pak::pak(...)`. Bootstraps uvr `v0.4.4`, resolves the R interpreter from `UVR_R_BIN`/`R_VERSION`/`PATH`, pins the manifest to that R's exact version, and syncs into `UVR_TARGET_LIB`/`R_LIBS_USER`. | | `local/install-bincraft.R` | Installs `forgejo::codefloe.com/rpkgs/bincraft@` instead of a `git::` URL; keeps the `git ls-remote` tag resolution. Exports `UVR_R_BIN`/`UVR_TARGET_LIB` from `R.home()`/`.libPaths()[1]` so the per-R-minor passes target their own R and library. | | `.crow/auto-apply-patches.yaml`, `.crow/weekly-patch-proposals.yaml`, `.crow/weekly-audit-missing.yaml`, `.crow/weekly-rebuild-missing.yaml`, `.crow/build-all-versions-install-deps.yaml` | `pak::pak(...)` → `UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh ...`. The explicit `UVR_R_BIN` matters in `build-all-versions-install-deps.yaml`, which has no `R_VERSION` in its step environment. | | `.crow/build-all-versions.yaml`, `.crow/process-updates.yaml`, `.crow/weekly-rebuild-missing.yaml` | `R_PKG_CACHE_DIR` → `UVR_CACHE_DIR` + `UVR_PACKAGES_DIR` on the same `/mnt/cache` volume, preserving the amd64-off/arm64-on split. Drops the `rm -rf .../pkgcache/_metadata/...` cleanup. | | `local/r-minor-helpers.R`, `local/build-all.R`, `local/tests/test-trim-pkgcache.R` | Removes `trim_pkgcache_metadata()`, its every-25-packages call and its tests. uvr's cache does not mint a fresh ~70 MB snapshot per `PACKAGES` change. | | `.crow/build-all-versions-install-deps.yaml` | Drops `pak::sysreqs_db_update()`; uvr resolves sysreqs from its vendored `r-system-requirements` rules via `--install-system-deps`. | | `docker/Containerfile-shiny-app` | Bootstraps uvr and drives both dependency installs through one uvr project with `UVR_LIBRARY` pointed at the image's R library. | | `docker/build-one.Dockerfile` | Ships `uvr-install.sh` at `/work/local/` so `install-bincraft.R` finds it. | | `docker/reprex/alpine.sh` | Replaces `pak::local_install_deps()` with DESCRIPTION parsing + `uvr add`. | | `local/test-package-loading.R` | Installs via the helper instead of `pak::pkg_install()`. | | `README.md` | Documents uvr for sysreq inference, archived-version installs and cache clearing. | | `renovate.json` | Tracks the `UVR_PIN` in `uvr-install.sh` via `github-releases`. | ## Behaviour notes - **`weekly-audit-missing` still takes bincraft from the default branch**, not the latest release tag, matching what the `git::` pak call did. Called out in a comment rather than silently changed. - **The uvr pin is repo-wide.** bincraft resolves `uvr` from `PATH` and pins no version of its own, so `UVR_PIN` in `uvr-install.sh` governs the whole pipeline. - **Persistent caches now also benefit bincraft**, which reads `UVR_CACHE_DIR`/`UVR_PACKAGES_DIR` from the inherited pipeline environment. - **`uvr sync` will not prune the shared library.** Pruning is disabled whenever `--library` is passed (`do_prune = prune && library_override.is_none()`), so `/mnt/cache/R-pkgs` keeps bincraft and its dependencies. The wipe-on-ABI-mismatch path is *not* similarly guarded, which is why the helper pins the manifest to the active R's exact version. - **`plans/` and `specs/` are untouched** — they are dated records of decisions made in June 2026 and describe bincraft's then-pak-based internals; rewriting them would misstate history. ## Verification `shellcheck`, all pre-commit hooks (on this commit's file range) and the `local/tests/` suite (100 assertions) pass. Not yet exercised in CI: the build-env images do not ship `uvr`, so the per-step `curl install.sh` bootstrap is untested against a real image. Worth a manual `build-all-versions-install-deps` run before merging. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/147 --- .crow/auto-apply-patches.yaml | 2 +- .crow/build-all-versions-install-deps.yaml | 33 ++++---- .crow/build-all-versions.yaml | 64 ++++++++------- .crow/process-updates.yaml | 46 +++++------ .crow/weekly-audit-missing.yaml | 3 +- .crow/weekly-patch-proposals.yaml | 2 +- .crow/weekly-rebuild-missing.yaml | 7 +- README.md | 19 ++--- build-all-versions-install-deps.yaml | 10 +-- docker/Containerfile-shiny-app | 22 +++++- docker/build-one.Dockerfile | 3 + docker/reprex/alpine.sh | 24 ++++-- local/build-all.R | 31 ++++---- local/install-bincraft.R | 47 +++++++++-- local/packages-to-build.R | 15 ++-- local/r-minor-helpers.R | 39 ---------- local/test-package-loading.R | 32 ++++---- local/tests/test-trim-pkgcache.R | 68 ---------------- local/uvr-install.sh | 90 ++++++++++++++++++++++ renovate.json | 14 +++- 20 files changed, 326 insertions(+), 245 deletions(-) delete mode 100644 local/tests/test-trim-pkgcache.R create mode 100755 local/uvr-install.sh diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index b9c8697..0076353 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -45,7 +45,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT backend_options: kubernetes: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index e0cca86..448cff3 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -10,22 +10,22 @@ variables: - arm64 default: amd64 OS: - description: "Base OS image name." + description: 'Base OS image name.' options: - alpine - redhat - ubuntu default: alpine OS_VERSION: - description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' options: - - "3.24" - - "8" - - "9" - - "10" - - "jammy" - - "noble" - default: "3.24" + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -66,23 +66,24 @@ steps: from_secret: B2_S3_SECRET_KEY PGPASS: from_secret: PGPASS - R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs + # Keep uvr's downloads and extracted-package entries on the persistent + # volume instead of the container-local ~/.uvr default. + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages CCACHE_DIR: /mnt/cache/ccache volumes: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: # one-time full wipe to fix corrupted .so files from previous failed builds # - rm -rf /mnt/cache/R-pkgs - # Clear churny pkgcache metadata left by a prior crashed run (the "patched" - # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). - # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. - - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 3932811..9ab888a 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -11,25 +11,25 @@ variables: - arm64 default: amd64 OS: - description: "Base OS image name." + description: 'Base OS image name.' options: - alpine - redhat - ubuntu default: alpine OS_VERSION: - description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' options: - - "3.22" - - "3.23" - - "3.24" - - "8" - - "9" - - "10" - - "jammy" - - "noble" - - "resolute" - default: "3.24" + - '3.22' + - '3.23' + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + - 'resolute' + default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -47,38 +47,49 @@ labels: platform: linux/${ARCH} group: rpkgs-${ARCH} +# Empty UVR_CACHE_DIR/UVR_PACKAGES_DIR fall back to uvr's container-local +# ~/.uvr defaults; amd64 deliberately does not persist them (as with the +# pkgcache dir it replaces), arm64 does. matrix: include: - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 4 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 4 @@ -107,8 +118,10 @@ steps: from_secret: GITHUB_PAT # normal env vars GIT_USER: pat-s - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR} + # set the location of uvr's caches, which persist the R package + # dependencies needed to install the packages themselves + UVR_CACHE_DIR: ${UVR_CACHE_DIR} + UVR_PACKAGES_DIR: ${UVR_PACKAGES_DIR} R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache NCPUS: 2 @@ -116,12 +129,7 @@ steps: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - # Clear churny pkgcache metadata left by a prior crashed run (the "patched" - # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). - # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. Within-run - # growth is bounded separately by trim_pkgcache_metadata() in build-all.R. - - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # The primary pass must not rely on build-all-versions-install-deps having # run on *this* agent: depends_on only orders the steps, but the cache # volume is per-agent, so a job landing on an agent where install-deps did diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 59b1c26..1e4833c 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -49,109 +49,109 @@ matrix: R_VERSION: 4.5.3 IMG: alpine:3.22 OS_ID: alpine322 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.23 OS_ID: alpine322 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.23 OS_ID: alpine323 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.23 OS_ID: alpine323 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine324 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine324 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-8 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-9 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-9 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-10 ARCH: amd64 R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-10 ARCH: arm64 R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2204 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2204 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2404 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2404 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2604 ARCH: amd64 R_VERSION: 4.5.3 IMG: ubuntu:resolute OS_ID: resolute - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2604 ARCH: arm64 R_VERSION: 4.5.3 IMG: ubuntu:resolute OS_ID: resolute - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' steps: - name: 'Processing Updates' @@ -182,8 +182,10 @@ steps: NTFY_AUTH: TRUE NTFY_PASSWORD: from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + # set the location of uvr's caches, which persist the R package + # dependencies needed to install the packages themselves + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache @@ -194,10 +196,10 @@ steps: INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index d66a37b..4efe409 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -142,7 +142,8 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' backend_options: docker: diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 0284e93..027d41c 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -35,7 +35,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 53f178f..03a13f9 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -141,7 +141,8 @@ steps: FORGEJO_TOKEN: from_secret: FORGEJO_TOKEN GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache @@ -150,12 +151,12 @@ steps: NCPUS: 2 commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: diff --git a/README.md b/README.md index 538df34..989069e 100644 --- a/README.md +++ b/README.md @@ -112,9 +112,9 @@ Processing updates with an existing database file takes around 5 minutes. ### Inferring System Dependencies -R package dependencies and their system dependencies are installed through {pak}. -{pak} allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. -Additionally, it automatically infers package dependencies using JSON rules from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). +R package dependencies and their system dependencies are installed through [uvr](https://github.com/nbafrank/uvr). +`uvr` allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. +Additionally, it automatically infers package dependencies using JSON rules vendored from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). Not all R packages specify required system dependencies in their DESCRIPTION file, and not all listed dependencies have existing rules in `rstudio/r-system-requirements`. For Alpine, no rules existed until recently, establishing a foundation for semi-automated package installation on Alpine Linux. @@ -152,12 +152,13 @@ A Shiny dashboard providing a search functionality of the database and grouped s Is supported by writing `Meta/archive.rds` during each package index update, listing all available archived packages. -### `pak::pak(package@version)` +### `uvr add package@version` -`pak` searches for `Archive/` and can install all versions it finds. +Clients that resolve archived versions search for `Archive/` and can install all versions they find. +This holds for `uvr` as well as for older `pak`-based clients. Ensure to use a clean cache if other repositories have been used previously. -If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. +If in doubt or when testing, call `uvr cache clean`. ## Lessons Learned @@ -180,7 +181,7 @@ If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. ## URL Composition and Platform Identifiers -Platform identifiers have been aligned with those used in to ensure proper recognition by the automatic syslib dependency installer of `pak`, specifically via the environment variable `PKG_SYSREQS_PLATFORM`: +Platform identifiers have been aligned with those used in , which `uvr` vendors and uses for its automatic syslib dependency installer (`uvr sync --install-system-deps`): - redhat-9 - redhat-8 @@ -237,8 +238,8 @@ internal error 1 in memDecompress Solution: ```sh -rm -rf /mnt/cache/R-pkgs/pak /mnt/cache/pkgcache/ /root/.cache/R/ -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +uvr cache clean +rm -rf /mnt/cache/uvr /root/.cache/R/ ```
diff --git a/build-all-versions-install-deps.yaml b/build-all-versions-install-deps.yaml index 926d550..dd02e8a 100644 --- a/build-all-versions-install-deps.yaml +++ b/build-all-versions-install-deps.yaml @@ -30,8 +30,9 @@ # from_secret: HETZNER_S3_SECRET_KEY_K3S # # normal env vars # GIT_USER: pat-s -# # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves -# # R_PKG_CACHE_DIR: /mnt/cache/pkgcache +# # set the location of uvr's caches, which persist the R package dependencies needed to install the packages themselves +# # UVR_CACHE_DIR: /mnt/cache/uvr/cache +# # UVR_PACKAGES_DIR: /mnt/cache/uvr/packages # R_LIBS_USER: /mnt/cache/R-pkgs # CCACHE_DIR: /mnt/cache/ccache # volumes: @@ -39,9 +40,8 @@ # commands: # - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # - mkdir -p /mnt/cache/R-pkgs -# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/pkgcache -# - /opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' -# - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git", dependencies = TRUE)' +# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/uvr +# - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft # - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine322', 'amd64'); saveRDS(pkgs, '/mnt/cache/pkgs_amd64.rds')" # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine323', 'arm64'); saveRDS(pkgs, '/mnt/cache/pkgs_arm64.rds')" diff --git a/docker/Containerfile-shiny-app b/docker/Containerfile-shiny-app index 7c7994f..4b0bd6c 100644 --- a/docker/Containerfile-shiny-app +++ b/docker/Containerfile-shiny-app @@ -1,7 +1,15 @@ FROM devxygmbh/r-alpine:4.4-3.20 AS build # ARG GITHUB_PAT -RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +RUN apk add --no-cache curl ca-certificates \ + && curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh \ + | UVR_INSTALL_DIR=/usr/local/bin sh + +# One uvr project drives both dependency installs below. UVR_LIBRARY points the +# syncs at the image's R library instead of the project-local .uvr/library/, so +# `Rscript app.R` finds the packages without a uvr-aware .Rprofile. +ENV UVR_LIBRARY=/usr/lib/R/library +RUN mkdir -p /uvr && cd /uvr && uvr init --here COPY --link ./DESCRIPTION . COPY --link ./R ./R @@ -11,13 +19,19 @@ COPY --link ./shiny/app.R /app/app.R RUN ls -la -# install R package deps -RUN R -q -e 'pak::pak()' +# install R package deps. uvr has no equivalent of bare `pak::pak()`, which +# reads the DESCRIPTION in the working directory, so extract the dependency +# names and hand them to `uvr add`. +RUN R -q --no-echo -e "d <- read.dcf('DESCRIPTION'); f <- intersect(colnames(d), c('Depends', 'Imports', 'LinkingTo')); p <- trimws(sub('[(].*', '', unlist(strsplit(paste(d[, f], collapse = ','), ',')))); writeLines(setdiff(p[nzchar(p)], c('R', rownames(installed.packages()))), '/tmp/deps.txt')" \ + && cd /uvr && xargs -r uvr add --no-install < /tmp/deps.txt \ + && uvr sync --install-system-deps RUN R CMD INSTALL --no-docs --without-keep.source . # install shiny app deps -RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); pak::pak(pkgs)" +RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); writeLines(pkgs, '/tmp/app-deps.txt')" \ + && cd /uvr && xargs -r uvr add --no-install < /tmp/app-deps.txt \ + && uvr sync --install-system-deps ENV PGPASS="" diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 5bc107a..8e61ebd 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -17,7 +17,10 @@ ARG CACHEBUST WORKDIR /work COPY build-one.R /work/build-one.R # Resolve and install the latest bincraft release dynamically (no hardcoded pin). +# uvr-install.sh lands under /work/local/ because install-bincraft.R looks for it +# there when the working directory is not a repo checkout. COPY install-bincraft.R /work/install-bincraft.R +COPY uvr-install.sh /work/local/uvr-install.sh # Ship the patch registry so build-one.R's `patches = "local/patches"` resolves # (build context is `local/`, CWD is /work). COPY patches /work/local/patches diff --git a/docker/reprex/alpine.sh b/docker/reprex/alpine.sh index 94806e5..851a9cc 100644 --- a/docker/reprex/alpine.sh +++ b/docker/reprex/alpine.sh @@ -1,10 +1,14 @@ docker run --rm -it --platform linux/arm64 alpine sh -apk add --no-cache R R-dev g++ +apk add --no-cache R R-dev g++ curl ca-certificates -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | UVR_INSTALL_DIR=/usr/local/bin sh -R -q -e 'pak::pak(c("gert", "purrr"))' +# `uvr add` always writes to .uvr/library/; only `uvr sync` honours UVR_LIBRARY, +# so add without installing and let the sync place the packages. +export UVR_LIBRARY=/usr/lib/R/library +mkdir -p /uvr && cd /uvr && uvr init --here +uvr add --no-install gert purrr && uvr sync --install-system-deps R @@ -22,14 +26,20 @@ unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE) tag <- all_tags$name package_name <- rep(package_name, length(tag)) +# uvr has no `pak::local_install_deps()`; read the DESCRIPTION of the checkout +# and `uvr add` the dependency names instead. purrr::walk2(package_name[1], tag, \(x, y) { print(y) system("git config --global advice.detachedHead false") + src <- sprintf("/tmp/%s_%s", x[1], y) system2("git", args = c( "clone", "-q", sprintf("--branch=%s", tail(y, 1)), - sprintf("https://github.com/cran/%s", x[1]), sprintf("/tmp/%s_%s", x[1], y) + sprintf("https://github.com/cran/%s", x[1]), src )) - pak::local_install_deps(sprintf("/tmp/%s_%s", x[1], y)) + d <- read.dcf(file.path(src, "DESCRIPTION")) + f <- intersect(colnames(d), c("Depends", "Imports", "LinkingTo")) + deps <- trimws(sub("[(].*", "", unlist(strsplit(paste(d[, f], collapse = ","), ",")))) + deps <- setdiff(deps[nzchar(deps)], "R") + system2("uvr", c("add", "--no-install", deps)) + system2("uvr", c("sync", "--install-system-deps")) }) - - diff --git a/local/build-all.R b/local/build-all.R index 1df1f83..37fc593 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -27,7 +27,9 @@ package_cache_files <- c( "/mnt/cache/packages/s3_cache.rds" ) if (!all(file.exists(package_cache_files))) { - message("Package snapshot missing from cache; recomputing via packages-to-build.R") + message( + "Package snapshot missing from cache; recomputing via packages-to-build.R" + ) dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { tmp <- paste0(path, ".tmp.", Sys.getpid()) @@ -36,7 +38,10 @@ if (!all(file.exists(package_cache_files))) { } source(file.path("local", "packages-to-build.R")) save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds") - save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds") + save_rds_atomic( + pkgs[r_minor_sensitive == TRUE], + "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" + ) message("Package snapshot recomputed.") } @@ -112,20 +117,22 @@ built <- DBI::dbGetQuery( ) DBI::dbDisconnect(con) before <- nrow(chunk) -chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] -sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) +chunk <- chunk[ + !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), +] +sprintf( + "Skipped %d already-attempted package versions; %d remaining for this job", + before - nrow(chunk), + nrow(chunk) +) # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, -# saving significant memory for pak subprocess forks +# saving significant memory for the dependency-installer subprocesses s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") sprintf("S3 cache: %s files", length(s3_cache)) n <- nrow(chunk) -# Every `trim_every` packages, bound the pkgcache _metadata dir so a full-platform -# run does not accumulate thousands of ~70 MB snapshots and fill the host disk. -# No-op on amd64 (R_PKG_CACHE_DIR is empty / cache not persisted). -trim_every <- 25L mapply( function(pkg, ver, sens, i) { cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) @@ -151,12 +158,6 @@ mapply( upload = TRUE, store_build_metadata = TRUE ) - if (i %% trim_every == 0L) { - removed <- trim_pkgcache_metadata() - if (removed > 0L) { - cat(sprintf(" [pkgcache trim] removed %d stale _metadata entries\n", removed)) - } - } }, chunk$Package, chunk$Version, diff --git a/local/install-bincraft.R b/local/install-bincraft.R index 4665b3b..e39cb4f 100644 --- a/local/install-bincraft.R +++ b/local/install-bincraft.R @@ -11,10 +11,11 @@ # # How it works: list the remote tags with `git ls-remote` (no token needed for # the public repo), keep the `vX.Y.Z` release tags, pick the highest version, -# and install it with pak. pak is idempotent on the git ref, so re-running keeps -# the package when it is already current and only updates when a newer tag ships. -# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is -# identical across git versions and `system2()` argument handling. +# and install it with uvr via `local/uvr-install.sh`. uvr is idempotent on the +# git ref, so re-running keeps the package when it is already current and only +# updates when a newer tag ships. Filtering/sorting is done in R (not via git's +# `--sort`/refspec) so behaviour is identical across git versions and +# `system2()` argument handling. repo_url <- Sys.getenv( "BINCRAFT_GIT_URL", @@ -44,7 +45,43 @@ latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][ ] message(sprintf("Installing latest bincraft release: %s", latest)) -pak::pak(sprintf("git::%s@%s", repo_url, latest)) + +# uvr addresses Forgejo repos as `forgejo::host/owner/repo@ref` rather than as a +# git URL, so drop the scheme and the trailing `.git` from `repo_url`. +spec <- sprintf( + "forgejo::%s@%s", + sub("\\.git$", "", sub("^[a-z]+://", "", repo_url)), + latest +) + +# `local/uvr-install.sh` when run from the repo root, `/work/local/` in the +# build-one image, which copies the two scripts into a flatter layout. +helper <- Sys.getenv("UVR_INSTALL_SH", unset = "") +if (!nzchar(helper)) { + candidates <- c("local/uvr-install.sh", "/work/local/uvr-install.sh") + found <- candidates[file.exists(candidates)] + if (length(found) == 0L) { + stop("Could not locate uvr-install.sh; set UVR_INSTALL_SH", call. = FALSE) + } + helper <- found[1L] +} + +# Point uvr at the R running this script and at the library it would install +# into, so the per-R-minor passes in the build pipelines (which call a different +# Rscript with R_LIBS_USER pointed elsewhere) target their own R and library. +Sys.setenv( + UVR_R_BIN = file.path(R.home("bin"), "R"), + UVR_TARGET_LIB = .libPaths()[1L] +) + +status <- system2(helper, shQuote(spec)) +if (!identical(status, 0L)) { + stop( + sprintf("uvr failed to install %s (exit %s)", spec, status), + call. = FALSE + ) +} + message(sprintf( "bincraft %s installed (%s)", as.character(utils::packageVersion("bincraft")), diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 7a3652d..887952d 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -15,15 +15,15 @@ suppressPackageStartupMessages(library(data.table)) # Sys.setenv("OS_VERSION" = "3.22") # Sys.setenv("ARCH" = "arm64") -arch = Sys.getenv("ARCH") +arch <- Sys.getenv("ARCH") # target: alpine-322, ubuntu-2404, redhat-9, etc. -platform = paste( +platform <- paste( Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-" ) # Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10") -codename = bincraft::set_codename(NULL) +codename <- bincraft::set_codename(NULL) con <- DBI::dbConnect( RPostgres::Postgres(), @@ -35,8 +35,8 @@ con <- DBI::dbConnect( sslmode = "require" ) -cran_archive = tools::CRAN_archive_db() -cran_release = tools::CRAN_package_db() +cran_archive <- tools::CRAN_archive_db() +cran_release <- tools::CRAN_package_db() # Subset cran_archive to only those packages cran_archive_in_release <- cran_archive[ names(cran_archive) %in% cran_release$Package @@ -84,13 +84,14 @@ s3fs::s3_file_system( region_name = "eu-central-003", refresh = TRUE ) -s3_pkgs = s3fs::s3_dir_ls( +s3_pkgs <- s3fs::s3_dir_ls( sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename), recurse = TRUE ) # Save the raw S3 file listing for the build step to use as s3_package_cache -# This avoids loading s3fs/reticulate in the build container, saving memory for pak forks +# This avoids loading s3fs/reticulate in the build container, saving memory for +# the dependency-installer subprocesses saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds") file_names <- basename(s3_pkgs) diff --git a/local/r-minor-helpers.R b/local/r-minor-helpers.R index 6f01c29..65aafdf 100644 --- a/local/r-minor-helpers.R +++ b/local/r-minor-helpers.R @@ -31,42 +31,3 @@ parse_build_args <- function(args) { ncpus = as.integer(pos[3L]) ) } - -# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the -# "patched" repo mints a new content hash on every PACKAGES change, so each build -# writes a fresh ~70 MB _metadata/pkgs-.rds (+ patched-/) that is -# never reused. Keep the `keep` newest entries by mtime; only remove entries -# older than `min_age_secs`, so a concurrent split-job's in-flight files are -# never deleted (each build uses a unique hash, so aged entries are -# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are -# not matched and thus preserved. Returns the number of entries removed. -trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"), - keep = 20L, - min_age_secs = 600) { - meta <- file.path(cache_dir, "R", "pkgcache", "_metadata") - if (!nzchar(cache_dir) || !dir.exists(meta)) { - return(0L) - } - entries <- c( - Sys.glob(file.path(meta, "patched-*")), - Sys.glob(file.path(meta, "pkgs-*.rds")) - ) - if (length(entries) == 0L) { - return(0L) - } - info <- file.info(entries) - order_new_first <- order(info$mtime, decreasing = TRUE) - ranked <- entries[order_new_first] - ranked_mtime <- info$mtime[order_new_first] - if (length(ranked) <= keep) { - return(0L) - } - candidates <- ranked[(keep + 1L):length(ranked)] - candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)]) - removable <- candidates[candidate_age >= min_age_secs] - if (length(removable) == 0L) { - return(0L) - } - unlink(removable, recursive = TRUE, force = TRUE) - length(removable) -} diff --git a/local/test-package-loading.R b/local/test-package-loading.R index 26a2e82..a3b7a28 100644 --- a/local/test-package-loading.R +++ b/local/test-package-loading.R @@ -1,22 +1,28 @@ -install.packages( - "pak", - repos = sprintf( - "https://r-lib.github.io/p/pak/stable/%s/%s/%s", - .Platform$pkgType, - R.Version()$os, - R.Version()$arch +# Installs every CRAN package one by one and checks that it loads. Dependencies +# go through uvr via local/uvr-install.sh, which bootstraps the uvr binary on +# first use and installs into .libPaths()[1]. +uvr_install <- function(pkg) { + Sys.setenv( + UVR_R_BIN = file.path(R.home("bin"), "R"), + UVR_TARGET_LIB = .libPaths()[1L] ) -) + status <- system2("local/uvr-install.sh", shQuote(pkg)) + if (!identical(status, 0L)) { + stop( + sprintf("uvr failed to install %s (exit %s)", pkg, status), + call. = FALSE + ) + } +} -Sys.setenv(PKG_SYSREQS = TRUE) all_pkgs <- rownames(available.packages()) -to_skip = c("ABRSQOL", "ACA", "ACE.CoCo") -all_pkgs = setdiff(all_pkgs, to_skip) +to_skip <- c("ABRSQOL", "ACA", "ACE.CoCo") +all_pkgs <- setdiff(all_pkgs, to_skip) for (i in all_pkgs) { message(sprintf("\nInstalling %s", i)) - pak::pkg_install(i) + uvr_install(i) library(i, character.only = TRUE) } @@ -323,7 +329,7 @@ if (length(to_process) == 0) { } else { for (i in to_process) { message(sprintf("\nInstalling %s", i)) - pak::pkg_install(i) + uvr_install(i) library(i, character.only = TRUE) } # Update to_skip to include all up to the last processed diff --git a/local/tests/test-trim-pkgcache.R b/local/tests/test-trim-pkgcache.R deleted file mode 100644 index 1a5b418..0000000 --- a/local/tests/test-trim-pkgcache.R +++ /dev/null @@ -1,68 +0,0 @@ -source(file.path("..", "r-minor-helpers.R")) - -# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is -# set to `age_secs` in the past so we can exercise the age gate deterministically. -make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) { - root <- tempfile("pkgcache-") - meta <- file.path(root, "R", "pkgcache", "_metadata") - dir.create(meta, recursive = TRUE) - old <- Sys.time() - age_secs - mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p } - mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p } - for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i))) - for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i))) - if (keep_repos) { - mk_dir(file.path(meta, "CRAN-075c426938")) - mk_dir(file.path(meta, "BioCsoft-1ac964ed6c")) - mk_file(file.path(meta, "bioc-sysreqs.dcf.gz")) - mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive - } - root -} - -n_churn <- function(root) { - meta <- file.path(root, "R", "pkgcache", "_metadata") - length(Sys.glob(file.path(meta, "patched-*"))) + - length(Sys.glob(file.path(meta, "pkgs-*.rds"))) -} - -test_that("empty cache_dir is a no-op", { - expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L) -}) - -test_that("missing _metadata dir is a no-op", { - expect_identical( - trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0), - 0L - ) -}) - -test_that("fewer than keep entries removes nothing", { - root <- make_meta(patched = 2L, pkgs = 2L) - expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L) - expect_identical(n_churn(root), 4L) -}) - -test_that("trims down to keep newest, leaving churn == keep", { - root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old - removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0) - expect_identical(removed, 40L) - expect_identical(n_churn(root), 20L) -}) - -test_that("entries younger than min_age_secs are protected", { - root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60) - # keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed - expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L) - expect_identical(n_churn(root), 30L) -}) - -test_that("stable repo dirs and pkg downloads are never touched", { - root <- make_meta(patched = 30L, pkgs = 30L) - trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0) - meta <- file.path(root, "R", "pkgcache", "_metadata") - expect_true(dir.exists(file.path(meta, "CRAN-075c426938"))) - expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c"))) - expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz"))) - expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg"))) -}) diff --git a/local/uvr-install.sh b/local/uvr-install.sh new file mode 100755 index 0000000..e1c4d0f --- /dev/null +++ b/local/uvr-install.sh @@ -0,0 +1,90 @@ +#!/bin/sh +# Install R packages into the CI library with uvr (https://github.com/nbafrank/uvr). +# +# Usage: +# local/uvr-install.sh httr2 jsonlite +# local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft@v4.4.3 +# +# Replaces `pak::pak(...)`. uvr is project-scoped: `uvr add` refuses to run +# outside a project and always writes to `.uvr/library/`, and only +# `uvr sync --library` can target an existing library. The project is therefore +# minted in a scratch directory under TMPDIR and thrown away afterwards; that +# also keeps `uvr init`'s `.Rprofile` out of the repo checkout, where it would +# hijack `.libPaths()` for every other R call in the pipeline. +# +# Pruning is a no-op here: uvr disables it whenever `--library` is passed, +# precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds +# bincraft and its dependencies alongside whatever this script installs). +# +# System dependencies come from uvr's vendored r-system-requirements rules, so +# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed. +# +# Environment: +# UVR_R_BIN R interpreter to install for; set by install-bincraft.R so +# the per-R-minor passes target their own R, not the primary +# R_VERSION fallback interpreter selector (/opt/R//bin/R) +# UVR_TARGET_LIB target library; defaults to R_LIBS_USER, then to the +# active R's .libPaths()[1] (which is where pak wrote) +# UVR_INSTALL_DIR where the uvr binary lands (default /usr/local/bin) + +set -eu + +# renovate: datasource=github-releases depName=nbafrank/uvr +UVR_PIN="v0.4.4" + +if [ "$#" -eq 0 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi + +# The build images keep R under /opt/R/ and off PATH. uvr resolves the +# interpreter via PATH and never downloads one unless `uvr r install` is run, so +# put the requested R first. +r_bin="${UVR_R_BIN:-}" +if [ -z "$r_bin" ] && [ -n "${R_VERSION:-}" ] && [ -x "/opt/R/${R_VERSION}/bin/R" ]; then + r_bin="/opt/R/${R_VERSION}/bin/R" +fi +if [ -n "$r_bin" ]; then + PATH="$(dirname "$r_bin"):$PATH" + export PATH +else + r_bin="$(command -v R)" +fi + +target_lib="${UVR_TARGET_LIB:-${R_LIBS_USER:-}}" +if [ -z "$target_lib" ]; then + target_lib="$("$r_bin" --no-echo --no-save -e 'cat(.libPaths()[1])')" +fi +if [ -z "$target_lib" ]; then + echo "error: could not determine a target library; set UVR_TARGET_LIB" >&2 + exit 2 +fi +mkdir -p "$target_lib" + +# Pin the manifest to the active R so the lockfile's R stays in step with the +# library's R sentinel. Without that, uvr can decide the library is ABI-stale +# and wipe it -- and this target is shared with bincraft. uvr only discovers R +# via PATH/R_HOME (it does not scan /opt/R), so the R put on PATH above is the +# only candidate this constraint can resolve to. +# shellcheck disable=SC2016 # $major/$minor are R expressions, not shell vars +r_full="$("$r_bin" --no-echo --no-save -e 'cat(paste(R.version$major, R.version$minor, sep = "."))')" + +install_dir="${UVR_INSTALL_DIR:-/usr/local/bin}" +uvr_bin="${install_dir}/uvr" +if [ ! -x "$uvr_bin" ]; then + echo "Bootstrapping uvr ${UVR_PIN} into ${install_dir}" + UVR_INSTALL_DIR="$install_dir" UVR_VERSION="$UVR_PIN" \ + sh -c 'curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | sh' +fi + +project_dir="${TMPDIR:-/tmp}/uvr-ci-$$" +rm -rf "$project_dir" +mkdir -p "$project_dir" +trap 'rm -rf "$project_dir"' EXIT +cd "$project_dir" + +"$uvr_bin" init --here --r-version "$r_full" +# --no-install: resolve and lock only. The install happens in the sync below, +# which is the only command that honours --library. +"$uvr_bin" add --no-install "$@" +"$uvr_bin" sync --library "$target_lib" --install-system-deps diff --git a/renovate.json b/renovate.json index 8c8e208..7b2c3b3 100644 --- a/renovate.json +++ b/renovate.json @@ -1,7 +1,12 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": ["local>devxy/renovate-config"], - "ignorePaths": ["docker/**", ".crow/process-updates.yaml", ".crow/build-all-versions.yaml", ".crow/weekly-rebuild-missing.yaml"], + "ignorePaths": [ + "docker/**", + ".crow/process-updates.yaml", + ".crow/build-all-versions.yaml", + ".crow/weekly-rebuild-missing.yaml" + ], "customManagers": [ { "customType": "regex", @@ -14,6 +19,13 @@ "packageNameTemplate": "https://codefloe.com/rpkgs/bincraft.git", "datasourceTemplate": "git-tags", "extractVersionTemplate": "^v?(?.+)$" + }, + { + "customType": "regex", + "fileMatch": ["^local/uvr-install\\.sh$"], + "matchStrings": [ + "# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+UVR_PIN=\"(?[^\"]+)\"" + ] } ] } From e02be7518fa4789688a44b5d56280fff5d3fd765 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 4 Aug 2026 02:32:01 +0000 Subject: [PATCH 094/125] chore(deps): update dependency nbafrank/uvr to v0.4.5 --- local/uvr-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local/uvr-install.sh b/local/uvr-install.sh index e1c4d0f..14b7f78 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -30,7 +30,7 @@ set -eu # renovate: datasource=github-releases depName=nbafrank/uvr -UVR_PIN="v0.4.4" +UVR_PIN="v0.4.5" if [ "$#" -eq 0 ]; then echo "usage: $0 ..." >&2 From 02dd5a695dc1a1816a8f815a7f5471deed99d2b3 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Wed, 5 Aug 2026 00:32:29 +0000 Subject: [PATCH 095/125] chore(deps): update terraform bunnynet to ~> 0.17 --- .terraform.lock.hcl | 72 ++++++++++++++++++++++----------------------- provider.tf | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index fdf41e1..aca8ce1 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.16.0" - constraints = "~> 0.16" + version = "0.17.0" + constraints = "~> 0.17" hashes = [ - "h1:+7y/HM0f4/cm7dpL5RulSZ3k5EDokVaIutfuanc1hhA=", - "h1:/4n/gUEzVa7LArNrNbZa4RyGC3+MRCCYTuqRqF36Z7U=", - "h1:4cSbrSalUxXnG0RVwrQG51NNleZNFU9GJxWCLK1M9WM=", - "h1:AlidPvq+LqsveQVhc9JYa7OKf4mcR+DjCzMomoBcmkw=", - "h1:Beny1tSl5O/jM7sSP7qYv+NEVTTyS/BGtSxvMFsII/o=", - "h1:E50kVGCtxWYoJgOLtdat1JFc9KCIeMByk65XSIpuaB0=", - "h1:ErNz5hzwf70NouW7a1DAknWI/0/BFUEX53hQ4BnUxWg=", - "h1:GTB8ugjpBEsRZpdVmKJw70IF2B1/fd1Gwven6XKghpk=", - "h1:JEEDgEc2gwjwfvnDwWO2ZHaf9T5WHtb7A6ABx6rQqhE=", - "h1:OVd3EgjJDuyMBFQ4XZYrs09NNEtFw4jGAbwyeos2H3M=", - "h1:U/9QeqZ+Q2MQ7eNfD1eWomLFlbYGwpliHcD/UTFe/D0=", - "h1:VPUe+m5Fx31TAR7udJPGGzk3EpCNizf0TmnKplOJyRM=", - "h1:YkJwUFpUemqi6bwRIEJW4xSpI7CAEXirqhyjl6Rmm68=", - "h1:a/15E8lijUcqvhUGxtqKmJT6jYGmQW9z7yBoOjusDm8=", - "h1:hii960NFcNpr0eZ/dcbnYOf2oZuo2IZl+z1IMTwh5YA=", - "h1:mq/KWNt6PPZLdNpO1ekzrn6EfWz6l8sTHG74dgy07nI=", - "h1:vso6Er2XV/MsUTMtXIlz0AhIFx0TZnK73YNSZfA/9ZY=", - "zh:1f581ed8bc8b676f0127af7c018912ca82972e9348370d9f4dbe5b44005bb0ec", - "zh:1fa5b9c01ffbc7fdc507bbcffc177a1f1deb9f56f3d27183c6d92a11ea3208f8", - "zh:42b6524a9658df7093199b65eb210b3246a9f9822c8778a77e2d00a222441c03", - "zh:4c4f17fe2ae86087626dfd1055b0faea6a23fe3a12aabb2a7de3b9b2df6e3e41", - "zh:5346bb67d7154a8bc8c887b4b86bef95b3e92857d4304f55afea6d56ed152932", - "zh:591df2c3d4552f6f613415c900023d43e73e00e90312b7b1a666fe567219120f", - "zh:7156849aa03348c7556b746d3d9956293678de4a9e30c80480358dcdacb2a7fa", - "zh:75dee86262352f061ad072a5b873d3b3eabbb124ab0f20a803cfd44a87a31067", - "zh:7c33954aff1878bec78a50742897fae78dd2fbd19abeaeaca0c560ccc8f78810", - "zh:7d9cf36633dd991b41891d1c91b45b5c803852a902705453d6be4513524a574c", + "h1:+qDt35lVSK7acw6a1xHuPYrqmZEcHSmtd+6n1TxNuYw=", + "h1:1dCu2l4DhPBjizVAH/WwAjT1Xbo52K4PMvHoD5zUhuU=", + "h1:Dvn46Auwuel4jqrqZXs2D7kdujNhs17LEmqhuY0k4/4=", + "h1:M5eDL3m2uSEr1XATJW0foHzKl8pFhCtgKuOM24bJRwU=", + "h1:PddaC7nM/gY4x9i3xy6TxOs9MAu2/6g58Xs/gv4DRV8=", + "h1:QVIKiZluI+NQAKu8NpFBl3Nvyx+d81vW9btEUdIQREc=", + "h1:S6TnzXHsRoGYvC1vJBkDiVEc0spceksY4n6x5WN5iYw=", + "h1:VcxZDWqCWMSjcUsC1K4sB6uYEoeoou+BC0ePoJXmf3A=", + "h1:W0y/agBVqls1cJlFGFYMu2VnqoPXFzxVHPIYe3OqfYQ=", + "h1:XmNd5fP9a0O77ve5BMQP2vARExgIa7rYl6KvyUYXPSs=", + "h1:e0EFKrWSQwaa/kGhnha4DXk4T68Av8QxP84mRSdWC9M=", + "h1:eM+/lUiU0pNSgQKoqKPgE3xJrJ0MHIpKG+yhaGB/P0M=", + "h1:fPWWA4T0/y7GX+tCGN23l1jODhZ3uCdR/MKgZDXYpAE=", + "h1:g+r2GVi4gVC4DuQg3PL70gW9BDskgWUzCBIMXTUq63A=", + "h1:gaZ8eALDtVHqykVDHav8004gHiMGaYR/3KwET0FUgao=", + "h1:kbqW25eaiv4N/N/z+sxLdJZ15yh5cgnRD/q6RclPMLc=", + "h1:rGjxue3mXRyQQqpywTXC4zK//JAtf0Cz7RP+uPMMJjw=", + "zh:05943fef14c2028f4722bf078aa1889229e94302f7678cc6f63adb669d8ea612", + "zh:26a163930a92a7408f7bbd0130064b84df8a232b500d8c6c3989952986308539", + "zh:41305feaaade55391447521ec309f3c038b631ca542907ad95132fab71a7e116", + "zh:606919a930f0299948504adbdcd0f239a8af5c418f85741c48f8add370a3d038", + "zh:66963d5b445639511939fc508513fd31da3ee1d4ee1a565ee396c9532897a349", + "zh:6c981ec0c8545556395c43e2511861ab65ee9ecf2a960480e7889c3af0d23af3", + "zh:7334a1bdb726ce1f1bf0a3155f30f84f65206980c229c832ff5f0b0718c44e0b", + "zh:75f6c86bf74511e605423332d113711c76c8028361a32282fb3359d6c7ecae9e", + "zh:7aebb1a01cfe8be54903853202ae06eba14ad99c37d230ed93ce7d6633e05e9b", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9552ed1be3059b31544632d6293c2bfcd2474983e116c42702a6c408deb8e44d", - "zh:b3cb1f2bde0fd591d18109119f6235ff41c7ea6441777a356a64c348ee4ccdf7", - "zh:b5c1748403bf43274eba2574219cfa082351c8b1c1537a768acebef5cc5b2c0a", - "zh:c1b032e33aee40945770bbdf6002f777ee7b4ca541c00f4cf5eb23c09d32ef54", - "zh:d469680df9845fb506be27b2c838d39ccd8e98e26540b99936775f954299c35c", - "zh:dd59fa97fea4b10b901a6a749ebff81895f54126a2f08b4cd8f5baf6cfbbd854", - "zh:fb142d8ea02909587a4858374fe33d8609347150f4d32b7e15423320ffbd07b1", + "zh:9041d0e20c9ceea532de6eebf5cb3a27dad0bb49d3f5b5154be2a08d68fbbf1f", + "zh:a6bbf65431a02be4df0ebb1cbe01185ad357ff6e33c01bd0558f59bed90c8f36", + "zh:c6d075a31096f080c388dfe46036f451c0cc114c3311a4f46ab8dbe1938a202f", + "zh:dd8703f7b55b8bc8e10f8718bea889781100b18e932b04898995b63178c3d36e", + "zh:dd92a5cd4e133a4000e7e5bc8cce876ae0ed803543cedd2f3d590661ba244d04", + "zh:e024fdf121bebc48c1e6debea344c6d4f174117f3ae605fca6e13b9705d92d22", + "zh:ee0e80c31b438e35fa1608f6a2f5824d2806db1e5e8b9f7a90986585c7bcb895", + "zh:fc2d4b705411b48f8c045981f9368a3ea2f74969dd6302008c31ff0bedd51f0a", ] } diff --git a/provider.tf b/provider.tf index c22ce2d..badbbc1 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.16" + version = "~> 0.17" } } } From 0da29ab3f6ae2e5548ce22a6e886b3ec62cd79d0 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 7 Aug 2026 09:15:04 +0000 Subject: [PATCH 096/125] fix(index): add a slot repair for a broken Built stamp and retire alpine 3.21 (#150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `arm64/alpine321` and `arm64/alpine322` advertise a broken stamp: ``` arm64/alpine321 :: 22930 Built: R 4.4.0; NA; 2026-07-31 13:35:52 UTC; unix arm64/alpine322 :: 24696 Built: R 4.5.0; NA; 2026-07-31 13:51:54 UTC; unix ``` The per-minor sub-slots (`contrib/4.4`, `4.5`, `4.6`) are affected too. All 18 other slots are correct. uvr matches the stamp's platform triple plus R minor to pick binary over source, so nothing matches `NA` and both slots silently serve as source-only, which is exactly the regression bincraft#85 added the stamp to prevent. `install.packages()` is unaffected, since it reads `Built:` from each tarball's own `DESCRIPTION`. The tarballs are fine (`arm64/alpine322/.../dress.graph_0.8.3.tar.gz` carries `aarch64-unknown-linux-musl`), and so is the R that built them (`r-4.5.0_1_aarch64.apk` ships `R_PLATFORM='aarch64-unknown-linux-musl'`). Only the index is wrong. bincraft#96 stops a stamp like this being written again, but it cannot repair what is already there. ## Why not just re-run the index update `upload_package_index()` reuses the slot's remote `PACKAGES.db`, and cranlike's `update_db()` only reparses files whose md5 changed, so entries already in the database keep the stamp they were written with. Dropping `PACKAGES.db` to force a full reparse does work, and it is what bincraft#85's rollout note suggested, but for an S3 repo cranlike reads each package's metadata from the CRAN *source* mirror on GitHub. A 25k-entry slot is then 25k requests to raw.githubusercontent.com, with a real risk of being rate-limited part-way through and leaving the slot half-written. Only the `Built` column is wrong, so this corrects it in place instead: patch the column in `PACKAGES.db`, put the database back, and let `upload_package_index()` re-emit `PACKAGES*` from it. `update_db()` always rewrites the index files even when nothing was reparsed, so no tarball is re-read and nothing is fetched from GitHub. ## Change - `local/repair-built-stamp.R` — repairs the generic slot plus every per-minor sub-slot. Dry-run by default; `--apply` writes. The replacement comes from `bincraft::built_stamp()` under the R running the script, so it is exactly what a healthy run would have written, and bincraft#96's guard makes a broken build image fail rather than write a second bad stamp. - `.crow/repair-built-stamp.yaml` — manual pipeline, routed by `target_arch` to the matching agent group and platform image, with `dry_run` defaulting to `true`. - `.crow/archive-missed-packages.yaml` — drop the two `alpine321` matrix entries. Alpine 3.21 is EOL: the website advertises only 3.23/3.24 and `process-updates.yaml` already dropped it, so that slot is retired rather than repaired. ## Verification `crow lint .crow/` passes on all nine pipelines. `air format` and `jarl check` are clean; the script parses, and the `/opt/R` minor-version derivation was checked against `4.4.3 / 4.5.3 / 4.6.0 / current` → `4.4 4.5 4.6`. The repair itself is unrun by design — it needs B2 credentials and an arm64 agent. ## Rollout 1. Cut a bincraft release so `local/install-bincraft.R` picks up #96 (it resolves the latest `vX.Y.Z` tag, and #96 is only on `main`). 2. Run this pipeline with `target_arch=arm64`, `OS=alpine`, `OS_VERSION=3.22`, `R_VERSION=4.5.3`, `dry_run=true` and check the reported counts. 3. Re-run with `dry_run=false`. 4. Confirm: `curl -sS https://cran.devxy.io/arm64/alpine322/latest/src/contrib/PACKAGES | grep '^Built:' | sort | uniq -c` `arm64/alpine321` is deliberately left alone. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/150 --- .crow/archive-missed-packages.yaml | 4 - .crow/repair-built-stamp.yaml | 97 ++++++++++++++++ local/repair-built-stamp.R | 175 +++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+), 4 deletions(-) create mode 100644 .crow/repair-built-stamp.yaml create mode 100644 local/repair-built-stamp.R diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 71f808b..bd127d6 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -28,10 +28,6 @@ matrix: ARCH: amd64 - CODENAME: redhat-10 ARCH: arm64 - - CODENAME: alpine321 - ARCH: amd64 - - CODENAME: alpine321 - ARCH: arm64 - CODENAME: alpine322 ARCH: amd64 - CODENAME: alpine322 diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml new file mode 100644 index 0000000..12e2644 --- /dev/null +++ b/.crow/repair-built-stamp.yaml @@ -0,0 +1,97 @@ +### Manual repair of a slot whose PACKAGES index advertises a broken `Built` +### stamp (e.g. `Built: R 4.5.0; NA; ...`). +# +# uvr matches the stamp's platform triple plus R minor to decide binary vs +# source, so an unusable triple turns a whole slot source-only. See +# local/repair-built-stamp.R for why this patches PACKAGES.db in place instead +# of forcing a full reparse. +# +# Run with `dry_run: true` first: it reports how many entries are broken per +# slot and changes nothing. Pick the R version the slot should advertise, which +# is the R_VERSION its entry in .crow/process-updates.yaml uses. +variables: + target_arch: + description: 'Architecture of the slot to repair.' + options: + - amd64 + - arm64 + default: arm64 + OS: + description: 'Base OS image name.' + options: + - alpine + - redhat + - ubuntu + default: alpine + OS_VERSION: + description: 'OS image tag. Must match OS (alpine: 3.22/3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).' + options: + - '3.22' + - '3.23' + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + - 'resolute' + default: '3.22' + R_VERSION: + description: 'R version whose stamp the slot should advertise.' + options: + - 4.5.3 + - 4.4.3 + default: 4.5.3 + dry_run: + description: 'Report what would change without writing anything.' + options: + - 'true' + - 'false' + default: 'true' + +when: + - event: manual + evaluate: 'target_arch == "${ARCH}"' + +skip_clone: true + +labels: + platform: linux/${ARCH} + group: rpkgs-${ARCH} + +matrix: + include: + - ARCH: amd64 + - ARCH: arm64 + +steps: + - name: 'Repair Built stamp' + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - | + if [ "$dry_run" = "false" ]; then + /opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" --apply + else + /opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" + fi + backend_options: + docker: + resources: + requests: + memory: 2Gi + cpu: 1000m + limits: + memory: 8Gi + cpu: 2000m diff --git a/local/repair-built-stamp.R b/local/repair-built-stamp.R new file mode 100644 index 0000000..06f73be --- /dev/null +++ b/local/repair-built-stamp.R @@ -0,0 +1,175 @@ +#!/usr/bin/env Rscript + +### Rewrite a broken `Built` stamp across one arch/codename slot. +### +### A slot's index can end up advertising a stamp whose platform triple is +### unusable, e.g. `Built: R 4.5.0; NA; ...`. uvr picks binary vs source by +### matching that triple plus the R minor, so no client matches it and the whole +### slot silently reverts to source-only, which makes uvr compile everything and +### fail wherever a system `-dev` library is missing. +### +### Rewriting it is not a matter of re-running the normal index update. +### `upload_package_index()` reuses the slot's remote `PACKAGES.db`, and +### cranlike's `update_db()` only reparses files whose md5 changed, so entries +### already in the database keep the stamp they were written with. Dropping +### `PACKAGES.db` to force a full reparse does work, but for an S3 repo cranlike +### reads each package's metadata from the CRAN *source* mirror on GitHub, so a +### 25k-entry slot means 25k requests to raw.githubusercontent.com and a real +### risk of being rate-limited part-way through. +### +### Only the `Built` column is wrong, so correct it in place instead: patch the +### column in `PACKAGES.db`, put the database back, and let +### `upload_package_index()` re-emit `PACKAGES*` from it. `update_db()` always +### rewrites the index files even when nothing was reparsed, so no tarball is +### re-read and nothing is fetched from GitHub. +### +### The replacement comes from `bincraft::built_stamp()` under the R running +### this script, so run it under the R version the slot should advertise (the +### `R_VERSION` its entry in `.crow/process-updates.yaml` uses). That is what a +### healthy `upload_package_index()` run would have written. +### +### Usage, inside the platform's build image: +### Rscript local/repair-built-stamp.R [--apply] +### +### Without `--apply` it reports what it would change and touches nothing. + +suppressPackageStartupMessages({ + library(bincraft) +}) + +args <- commandArgs(trailingOnly = TRUE) +arch <- args[1L] +apply_changes <- "--apply" %in% args + +if (is.na(arch) || !nzchar(arch)) { + stop( + "Usage: Rscript local/repair-built-stamp.R [--apply]", + call. = FALSE + ) +} + +bucket <- "devxy-rpkgs-binaries" +endpoint <- "https://s3.eu-central-003.backblazeb2.com" +region <- "eu-central-003" + +codename <- bincraft::set_codename(NULL) +if (is.null(codename) || is.na(codename) || !nzchar(codename)) { + stop( + "Could not detect a codename from /etc/os-release; run this in a build image.", + call. = FALSE + ) +} + +# built_stamp() refuses an unusable platform, so a broken build image fails here +# rather than writing a second bad stamp over the first one. +stamp <- bincraft::built_stamp() +message(sprintf("Slot: %s/%s", arch, codename)) +message(sprintf("New stamp: %s", stamp)) + +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = endpoint, + region_name = region, + refresh = TRUE +) + +base_dir <- file.path(bucket, arch, codename, "latest", "src", "contrib") + +# A stamp is broken when its platform component is absent or literally "NA". +broken_stamp_where <- paste( + "Built IS NULL", + "OR Built LIKE '%; NA;%'", + "OR Built LIKE '%; ;%'" +) + +# The generic slot plus every per-minor sub-slot, which carry the same stamp and +# are poisoned by the same run. +r_minors <- sub( + "^.*/R/([0-9]+\\.[0-9]+)\\.[0-9]+$", + "\\1", + list.dirs("/opt/R", recursive = FALSE) +) +r_minors <- unique(grep("^[0-9]+\\.[0-9]+$", r_minors, value = TRUE)) +slots <- c(base_dir, file.path(base_dir, r_minors)) + +repair_slot <- function(slot) { + db_remote <- file.path(slot, "PACKAGES.db") + if (!s3fs::s3_file_exists(db_remote)) { + message(sprintf(" %s: no PACKAGES.db, skipping", slot)) + return(invisible(NULL)) + } + + db_local <- tempfile(fileext = ".db") + s3fs::s3_file_download(db_remote, db_local, overwrite = TRUE) + + con <- DBI::dbConnect(RSQLite::SQLite(), db_local) + on.exit(DBI::dbDisconnect(con), add = TRUE) + + total <- DBI::dbGetQuery(con, "SELECT COUNT(*) AS n FROM packages")$n + broken <- DBI::dbGetQuery( + con, + sprintf("SELECT COUNT(*) AS n FROM packages WHERE %s", broken_stamp_where) + )$n + + message(sprintf(" %s: %s entries, %s broken", slot, total, broken)) + if (broken == 0L) { + return(invisible(NULL)) + } + if (!apply_changes) { + message(" (dry run, pass --apply to rewrite)") + return(invisible(NULL)) + } + + DBI::dbExecute( + con, + sprintf("UPDATE packages SET Built = ? WHERE %s", broken_stamp_where), + params = list(stamp) + ) + DBI::dbDisconnect(con) + on.exit() + + s3fs::s3_file_upload(db_local, db_remote, overwrite = TRUE) + message(sprintf(" rewrote %s entries and uploaded PACKAGES.db", broken)) + invisible(NULL) +} + +invisible(lapply(slots, repair_slot)) + +if (!apply_changes) { + message("Dry run complete; nothing was changed.") + quit(save = "no") +} + +# Re-emit PACKAGES/PACKAGES.gz/PACKAGES.rds from the corrected database. Nothing +# is reparsed, because no tarball's md5 changed. +message("Re-emitting index files from the corrected database...") +bincraft::upload_package_index( + codename = codename, + s3_endpoint = endpoint, + s3_region = region, + s3_bucket = bucket, + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") +) +for (minor in r_minors) { + try( + bincraft::upload_package_index( + codename = codename, + r_minor = minor, + s3_endpoint = endpoint, + s3_region = region, + s3_bucket = bucket, + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") + ), + silent = FALSE + ) +} + +message("Done. Verify with:") +message(sprintf( + " curl -sS https://cran.devxy.io/%s/%s/latest/src/contrib/PACKAGES | grep '^Built:' | sort | uniq -c", + arch, + codename +)) From 77ce1d212dfdf791be19c768931b75ddd3991e09 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 7 Aug 2026 09:41:53 +0000 Subject: [PATCH 097/125] fix(ci): gate the Built-stamp repair on its own variable (#151) ## Problem `repair-built-stamp.yaml` gated on `target_arch`: ```yaml when: - event: manual evaluate: 'target_arch == "${ARCH}"' ``` That is the same variable `build-all-versions.yaml` and `build-all-versions-install-deps.yaml` gate on. A manual run passing `target_arch=arm64` to reach the repair therefore matched all three, so triggering a dry-run repair also queued a full arm64 rebuild. I hit this triggering the alpine 3.22 dry run (pipeline 10706), which I killed. `crow pipeline ps` renders empty states on this version, so I could not confirm from the CLI whether the rebuild workflows started before the kill or only sat queued behind the running cron jobs; no output was attributable to them. ## Change Gate on `repair_built_stamp` instead. Every other pipeline here already gates on a variable named after itself (`process_cran_updates`, `weekly_audit_missing`, `weekly_rebuild_missing`), and `target_arch` was the odd one out being shared by two. The header now records the collision and the exact invocation, so the next pipeline added here does not repeat it. ## Verification `crow lint` passes; `prek` clean. Grep of every trigger condition in `.crow/` confirms `repair_built_stamp` is unique and that no other pipeline gates on `OS`, `OS_VERSION`, `R_VERSION` or `dry_run` alone: ``` archive-missed-packages.yaml: task == "archive-missed-packages" build-all-versions.yaml: target_arch == "${ARCH}" build-all-versions-install-deps.yaml: target_arch == "${ARCH}" weekly-audit-missing.yaml: weekly_audit_missing == ... repair-built-stamp.yaml: repair_built_stamp == "${ARCH}" process-updates.yaml: process_cran_updates == ... weekly-rebuild-missing.yaml: weekly_rebuild_missing == ... ``` ## Note for whoever runs it `crow pipeline create` against this instance returned HTTP 504 while still creating the pipeline. Retrying on that error duplicates the run: I created four before noticing. Verify with `pipeline ls` rather than trusting the exit status. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/151 --- .crow/repair-built-stamp.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml index 12e2644..83821e9 100644 --- a/.crow/repair-built-stamp.yaml +++ b/.crow/repair-built-stamp.yaml @@ -9,9 +9,18 @@ # Run with `dry_run: true` first: it reports how many entries are broken per # slot and changes nothing. Pick the R version the slot should advertise, which # is the R_VERSION its entry in .crow/process-updates.yaml uses. +# +# The gate variable is `repair_built_stamp`, not `target_arch`: `target_arch` is +# what build-all-versions and build-all-versions-install-deps gate on, so a +# manual run passing it would start a full rebuild alongside this repair. Every +# pipeline here gates on a variable named after itself for exactly that reason. +# +# crow pipeline create --branch main \ +# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \ +# --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries variables: - target_arch: - description: 'Architecture of the slot to repair.' + repair_built_stamp: + description: 'Architecture of the slot to repair. Also gates this pipeline.' options: - amd64 - arm64 @@ -51,7 +60,7 @@ variables: when: - event: manual - evaluate: 'target_arch == "${ARCH}"' + evaluate: 'repair_built_stamp == "${ARCH}"' skip_clone: true From 2f732457d2af605415b1768a0992db7d9a17c1cf Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 7 Aug 2026 14:12:22 +0000 Subject: [PATCH 098/125] feat(edge): route PACKAGES requests to the per-R-minor slot (#152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `install.packages("curl")` fails in `reg.devxy.io/r/r-alpine:4.5-3.24` with "package 'curl' is not available for this version of R", on both arches. `curl` is not missing from the repo: it is in `…/latest/src/contrib/4.5/` and `…/4.6/`, the per-minor slots that base R cannot address. The image's repo URL resolves to `…/latest/src/contrib`, whose index does not list it. On `amd64/alpine324` that is 2 886 packages invisible to `install.packages()` (23 on `amd64/noble`) — what issue #63 records as "missing binaries". Two further findings while investigating: - The middleware only ever rewrote the bare `cran.rpkgs.com/src/contrib/…` form, and that form was broken for every Linux client on a stock R user agent: `ALPINE_REGEX`/`UBUNTU_REGEX`/`RHEL_REGEX` need a Posit-style UA that carries the distro, so stock R fell through to `extractOs()` and got redirected to `/amd64/linux-musl/latest/…`, a slot that does not exist. - `PACKAGES*` is served `cdn-cache: BYPASS` (bincraft uploads it `no-store`), so the middleware sees every index request and no purge is needed for routing changes to take effect. ## What this changes **`edge/rpkgs-router.ts`** — the middleware, now a reviewed file in this repo rather than dashboard state. It routes `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib//` for slots listed in `UNION_SLOTS`, and nothing else. Tarballs are deliberately left alone. R keeps the `contriburl` it *asked for*, not the one the redirect served it, so every tarball URL is resolved against the flat directory and the union index steers the per-minor ones with a `Path: ` field. Rewriting a tarball request here would send flat-slot packages into a directory that does not hold them. Also in the script: the phantom `linux-gnu`/`linux-musl` fallback is gone (an unidentifiable distro goes to CRAN, as an unparseable UA already did), and every redirect carries `Cache-Control: no-store` since its target depends on the User-Agent. The macOS branches are unchanged. **`cdn.tf`** — `bunnynet_compute_script.rpkgs_router` with `content = file("edge/rpkgs-router.ts")`, the `UNION_SLOTS` variable, and `middleware_script` pointing at the resource instead of the literal `29277`. `UNION_SLOTS` is empty, so merging and applying this changes no client's behaviour. A slot is added only once bincraft has republished its per-minor index as a union (rpkgs/bincraft#97); routing to a raw per-minor index would hide every package it does not carry. Rolling back is a variable edit, not a deploy. **`specs/`, `plans/`** — the design and the implementation plan, including the two approaches that were rejected (edge-side merge, moving the minor up the path) and why. ## Verification `just edge-test` runs 13 routing cases against the SDK's local server, so what is tested is the artifact that gets deployed; pass-through cases proxy to the real origin. All pass. End to end, with the middleware in front of a locally built union index for `amd64/alpine324` (31 507 records), inside the runtime image: ``` curl: 7.1.0 -> …/latest/src/contrib/4.5 -> curl_7.1.0.tar.gz 717 725 B jsonlite: 2.0.0 -> …/latest/src/contrib -> jsonlite_2.0.0.tar.gz 1 055 849 B ``` `tofu validate` passes. `tofu plan` has not been run: no `BUNNYNET_API_KEY` available in this environment. ## Before applying The script pre-dates this configuration, so it must be adopted, not created: ```sh tofu import bunnynet_compute_script.rpkgs_router 29277 tofu plan ``` The plan should show an in-place `content` update and no replacement of the pull zone. Without the import, tofu creates a second script and repoints the zone at it. Note that `name = "rpkgs-router"` will rename the existing script on apply. ## Not fixed here `install.packages("curl")` on `alpine324` will now *resolve*, then fail to build: that slot's tarballs are byte-identical CRAN **source** tarballs (no `Meta/`, no `Built:` in DESCRIPTION) which the index nevertheless stamps `Built: R 4.5.3; …-linux-musl`. Sampled: `amd64/alpine324` 3/12 binary, `arm64/alpine324` 13/30, `amd64/noble` 12/12, `amd64/alpine323` 17/20. That slot needs a rebuild, tracked separately. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/152 --- cdn.tf | 25 +- edge/rpkgs-router.test.ts | 160 ++++++++++++ edge/rpkgs-router.ts | 235 ++++++++++++++++++ justfile | 14 ++ plans/2026-08-07-per-minor-edge-routing.md | 166 +++++++++++++ ...026-08-07-per-minor-edge-routing-design.md | 174 +++++++++++++ 6 files changed, 773 insertions(+), 1 deletion(-) create mode 100644 edge/rpkgs-router.test.ts create mode 100644 edge/rpkgs-router.ts create mode 100644 plans/2026-08-07-per-minor-edge-routing.md create mode 100644 specs/2026-08-07-per-minor-edge-routing-design.md diff --git a/cdn.tf b/cdn.tf index 8a950f5..fe1b25f 100644 --- a/cdn.tf +++ b/cdn.tf @@ -52,6 +52,29 @@ ### cran.rpkgs.com +# The edge middleware that resolves the bare cran.rpkgs.com form to an +# / slot and routes PACKAGES* to the per-R-minor slot. The source of +# truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release. +# +# The script pre-dates this configuration, so it is adopted rather than created: +# tofu import bunnynet_compute_script.rpkgs_router 29277 +resource "bunnynet_compute_script" "rpkgs_router" { + type = "middleware" + name = "rpkgs-router" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +# Slots ("/", comma separated) whose per-minor index bincraft has +# already republished as a union of the per-minor and flat slots. Routing to a +# slot that is not listed here would hide every package the per-minor index does +# not carry, so this stays empty until a slot has been backfilled. +resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + default_value = "" + required = false +} + resource "bunnynet_pullzone" "cran_rpkgs_com" { name = "cran-rpkgs" @@ -64,7 +87,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { origin { type = "OriginUrl" url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" - middleware_script = 29277 + middleware_script = bunnynet_compute_script.rpkgs_router.id } routing { diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts new file mode 100644 index 0000000..553185d --- /dev/null +++ b/edge/rpkgs-router.test.ts @@ -0,0 +1,160 @@ +/** + * Routing matrix for `edge/rpkgs-router.ts`. + * + * The script is exercised through the SDK's local server rather than by + * importing its internals, so what is tested is the artifact that gets + * deployed. Requests that the script passes through are proxied to the real + * origin, which keeps the "no redirect" cases honest: they assert that the + * client reached the flat slot, not merely that no `Location` was set. + * + * Run with `just edge-test`. + */ +import { assertEquals } from 'jsr:@std/assert@1'; + +const SCRIPT = new URL('./rpkgs-router.ts', import.meta.url).pathname; +const BASE = 'http://127.0.0.1:8080'; +const UNION_SLOTS = 'amd64/alpine324'; + +const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; +const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)'; +const UA_CURL = 'curl/8.0.1'; + +const SLOT = '/amd64/alpine324/latest/src/contrib'; +const OTHER_SLOT = '/amd64/noble/latest/src/contrib'; + +interface Probe { + status: number; + location: string | null; + cacheControl: string | null; +} + +async function probe(path: string, userAgent: string): Promise { + const res = await fetch(BASE + path, { + headers: { 'User-Agent': userAgent }, + redirect: 'manual', + }); + await res.body?.cancel(); + return { + status: res.status, + location: res.headers.get('location'), + cacheControl: res.headers.get('cache-control'), + }; +} + +/** Kill tolerantly: the child has already exited if the script failed to load. */ +async function stopServer(child: Deno.ChildProcess): Promise { + try { + child.kill(); + } catch { + // already gone + } + await child.status; +} + +async function startServer(): Promise { + const child = new Deno.Command(Deno.execPath(), { + args: ['run', '-A', SCRIPT], + env: { UNION_SLOTS }, + stdout: 'null', + stderr: 'inherit', + }).spawn(); + + for (let attempt = 0; attempt < 150; attempt++) { + try { + const res = await fetch(`${BASE}/`, { + headers: { 'User-Agent': UA_CURL }, + redirect: 'manual', + }); + await res.body?.cancel(); + return child; + } catch { + await new Promise((resolve) => setTimeout(resolve, 200)); + } + } + + await stopServer(child); + throw new Error('edge script did not start listening on ' + BASE); +} + +Deno.test('rpkgs-router', async (t) => { + const server = await startServer(); + + try { + await t.step("routes an index request to the client's R minor", async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.status, 302); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + + await t.step('routes R 4.6 to its own slot', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R46_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`); + }); + + await t.step('routes PACKAGES and PACKAGES.rds too', async () => { + for (const file of ['PACKAGES', 'PACKAGES.rds']) { + const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/${file}`, `expected ${file} to be routed`); + } + }); + + await t.step('marks the redirect uncacheable', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.cacheControl, 'no-store'); + }); + + await t.step('leaves a slot outside UNION_SLOTS alone', async () => { + const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('never routes a tarball', async () => { + const res = await probe(`${SLOT}/jsonlite_2.0.0.tar.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('does not redirect a path already under a minor', async () => { + const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('leaves a client without an R version alone', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_CURL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('resolves the bare root to slot and minor', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_ALPINE); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + + await t.step('sends an unidentifiable distro to CRAN', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); + }); + + await t.step('keeps the macOS rewrite', async () => { + const res = await probe('/src/contrib/foo_1.0.tar.gz', UA_R45_DARWIN); + assertEquals(res.location, 'https://cran.rpkgs.com/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz'); + }); + + await t.step('keeps the macOS binary passthrough to CRAN', async () => { + const path = '/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz'; + const res = await probe(path, UA_R45_DARWIN); + assertEquals(res.location, `https://cran.r-project.org${path}`); + }); + + await t.step('collapses duplicate slashes before matching', async () => { + const res = await probe(`/amd64/alpine324//latest/src/contrib//PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + } finally { + await stopServer(server); + } +}); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts new file mode 100644 index 0000000..9cd410b --- /dev/null +++ b/edge/rpkgs-router.ts @@ -0,0 +1,235 @@ +/** + * Edge middleware for cran.rpkgs.com. + * + * Two jobs: + * + * 1. Resolve the bare `https://cran.rpkgs.com` form to a concrete + * `/` slot from the User-Agent, or send the client to CRAN when + * the distro cannot be identified. + * 2. Route `PACKAGES*` requests to the per-R-minor slot + * (`…/latest/src/contrib//`), so a stock `install.packages()` sees the + * packages that only exist there. + * + * Only index files are routed. Tarballs are deliberately left alone: R keeps + * the contrib URL it asked for, not the one it was redirected to, so every + * tarball URL is resolved against the flat directory and the union index steers + * the per-minor ones with a `Path: ` field. Rewriting a tarball request + * here would send flat-slot packages into a directory that does not hold them. + * + * Routing is gated on UNION_SLOTS. The raw per-minor index holds only the + * ABI-sensitive subset of a slot; it is safe to route to it only once bincraft + * has republished it as a union of the per-minor and flat slots. + * + * Deployed by OpenTofu from this file (`bunnynet_compute_script.rpkgs_router`). + * Test with `just edge-test`. + */ +import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; + +const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; +const CRAN_ORIGIN = 'https://cran.r-project.org'; + +/** Slots ("/", comma separated) whose per-minor index is a union. */ +const UNION_SLOTS = new Set( + (Deno.env.get('UNION_SLOTS') ?? '') + .split(',') + .map((slot) => slot.trim()) + .filter((slot) => slot.length > 0), +); + +/** `///latest/src/contrib[/]` */ +const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/; + +/** A path that already sits in a per-minor slot, e.g. `4.5/PACKAGES.gz`. */ +const MINOR_DIR_REGEX = /^\d+\.\d+\//; + +/** The only files this script routes. */ +const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; + +const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; + +const MACOS_BIN_REGEX = + /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; + +const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i; + +const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i; +const UBUNTU_CODENAMES: Record = { + '24.04': 'noble', + '22.04': 'jammy', +}; + +const ALPINE_REGEX = /(?:Alpine Linux(?:\s+VERSION_ID=)?|alpine-)\s*(\d+)\.(\d+)/i; + +/** + * R's own User-Agent is `R (4.5.3 x86_64-pc-linux-musl …)`; the Posit-style one + * some sites configure is `R/4.5.3 R (…)`. Both carry the minor, which is why + * per-minor routing works without the distro being identifiable. + */ +const R_MINOR_REGEXES = [/\bR\/(\d+)\.(\d+)/, /\bR \((\d+)\.(\d+)/]; + +function normalizePathname(pathname: string): string { + return pathname.replace(/\/{2,}/g, '/'); +} + +function redirectTo(location: string, status = 302): Response { + return new Response(null, { + status, + headers: { + Location: location, + // The target depends on the User-Agent, so the redirect itself must + // never be cached; only its target is a cacheable, UA-independent URL. + 'Cache-Control': 'no-store', + 'X-Via': 'MyMiddleware', + 'X-Rewritten-By': 'rpkgs-edge-middleware', + }, + }); +} + +function extractRMinor(userAgent: string): string | null { + for (const regex of R_MINOR_REGEXES) { + const match = userAgent.match(regex); + if (match) { + return `${match[1]}.${match[2]}`; + } + } + return null; +} + +function mapArch(arch: string): string { + if (arch === 'aarch64') return 'arm64'; + if (arch === 'x86_64') return 'amd64'; + return arch; +} + +function extractArch(userAgent: string): string { + const match = userAgent.match(/(x86_64|aarch64|arm64|i386|i686)/); + return match ? mapArch(match[1]) : ''; +} + +/** + * Identify the `/` slot from the User-Agent, or null. + * + * A stock R User-Agent carries only `linux-gnu` / `linux-musl`, which are not + * slot names: returning them produced redirects into slots that do not exist + * (`/amd64/linux-musl/latest/…`, a guaranteed 404). An unidentifiable distro + * is reported as such so the caller can fall back to CRAN. + */ +function parseSlot(userAgent: string): string | null { + const arch = extractArch(userAgent); + if (!arch) { + return null; + } + + const rhel = userAgent.match(RHEL_REGEX); + if (rhel) { + return `${arch}/rhel${rhel[2]}`; + } + + const ubuntu = userAgent.match(UBUNTU_REGEX); + if (ubuntu) { + const codename = UBUNTU_CODENAMES[ubuntu[1]]; + if (codename) { + return `${arch}/${codename}`; + } + } + + const alpine = userAgent.match(ALPINE_REGEX); + if (alpine) { + return `${arch}/alpine${alpine[1]}${alpine[2]}`; + } + + return null; +} + +function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: string } | null { + const rverMatch = userAgent.match(/R \((\d+)\.(\d+)/); + const archMatch = userAgent.match(/(aarch64|arm64|x86_64)/); + const osMatch = userAgent.match(/darwin(\d+)/); + + if (!rverMatch || !archMatch || !osMatch) { + return null; + } + + const arch = archMatch[1] === 'aarch64' ? 'arm64' : archMatch[1]; + const darwinVer = parseInt(osMatch[1], 10); + const os = darwinVer >= 21 && darwinVer < 22 ? `monterey-${arch}` : `big-sur-${arch}`; + + return { os, arch, rver: `${rverMatch[1]}.${rverMatch[2]}` }; +} + +/** + * The contrib path a request should be served from, relative to the slot. + * + * Returns the per-minor path for an index file when the slot is known to carry + * a union index and the client's R minor is known; otherwise the flat path, + * which is what every client sees today. + */ +function contribPath(slot: string, rest: string, userAgent: string): string { + const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`; + + if (!INDEX_FILE_REGEX.test(rest) || !UNION_SLOTS.has(slot)) { + return flat; + } + + const rMinor = extractRMinor(userAgent); + return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; +} + +BunnySDK.net.http + .servePullZone({ url: 'https://cran.rpkgs.com/' }) + .onOriginRequest((ctx) => { + const url = new URL(ctx.request.url); + const path = normalizePathname(url.pathname); + const userAgent = ctx.request.headers.get('User-Agent') || ''; + + // macOS clients are served from CRAN's own binary tree. + const srcContrib = path.match(SRC_CONTRIB_REGEX); + if (srcContrib && /darwin/.test(userAgent)) { + const mac = parseMacUserAgent(userAgent); + if (mac) { + return Promise.resolve( + redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`), + ); + } + } + + if (MACOS_BIN_REGEX.test(path)) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + + // Already-qualified slot URLs: what the runtime images have baked in. + const slotPath = path.match(SLOT_PATH_REGEX); + if (slotPath) { + const slot = `${slotPath[1]}/${slotPath[2]}`; + const rest = slotPath[3]; + + // Never rewrite a request that is already in a per-minor slot, or the + // redirect would chase its own tail. + if (MINOR_DIR_REGEX.test(rest)) { + return Promise.resolve(ctx.request); + } + + const target = contribPath(slot, rest, userAgent); + if (target === path) { + return Promise.resolve(ctx.request); + } + return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`)); + } + + // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. + if (path === '/' || path === '/src/contrib' || path.startsWith('/src/contrib/')) { + const slot = parseSlot(userAgent); + if (!slot) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + + const rest = srcContrib ? srcContrib[1] : ''; + return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`)); + } + + return Promise.resolve(ctx.request); + }) + .onOriginResponse((ctx) => { + ctx.response.headers.append('X-Via', 'MyMiddleware'); + return Promise.resolve(ctx.response); + }); diff --git a/justfile b/justfile index a9a705e..6c75b13 100644 --- a/justfile +++ b/justfile @@ -73,3 +73,17 @@ rebuild os tag arch package *versions: --build-arg CACHEBUST="$(date +%s)" \ -f docker/build-one.Dockerfile \ local + +# run the edge middleware routing matrix (uses a local deno, else the deno image) +edge-test: + #!/usr/bin/env bash + set -euo pipefail + if command -v deno >/dev/null 2>&1; then + deno test -A edge/rpkgs-router.test.ts + else + docker run --rm \ + -v "$PWD:/w" -w /w \ + -v deno-cache:/deno-dir \ + denoland/deno:latest \ + deno test -A edge/rpkgs-router.test.ts + fi diff --git a/plans/2026-08-07-per-minor-edge-routing.md b/plans/2026-08-07-per-minor-edge-routing.md new file mode 100644 index 0000000..3c3e320 --- /dev/null +++ b/plans/2026-08-07-per-minor-edge-routing.md @@ -0,0 +1,166 @@ +# Per-R-minor edge routing implementation plan + +Spec: `specs/2026-08-07-per-minor-edge-routing-design.md` + +**Goal:** let a stock `install.packages()` see the per-minor packages by routing `PACKAGES*` requests to `…/src/contrib//`, where `bincraft` publishes a union index. + +**Architecture:** the union is built in `bincraft`; the edge script only redirects index requests, gated on a `UNION_SLOTS` script variable; the script lives in this repo and is applied by OpenTofu. + +**Tech stack:** Deno / TypeScript (Bunny Edge Scripting, SDK 0.12), OpenTofu with `BunnyWay/bunnynet` 0.17, R (bincraft). + +## Global constraints + +- Redirect only `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds`; never a tarball, because the union index already carries the correct tarball URL for both classes of package. +- Every redirect carries `Cache-Control: no-store`; redirect targets stay UA-independent. +- `UNION_SLOTS` is empty by default, so deploying the script is a no-op until a slot is backfilled. +- A slot is `/`, e.g. `amd64/alpine324`. +- Verified prerequisites: `PACKAGES*` is served `cdn-cache: BYPASS`, so the script sees every index request; `Deno.env.get()` reads script variables; the SDK local server listens on `127.0.0.1:8080`. + +--- + +## Task 1: Edge script and its test matrix + +**Files:** + +- Create: `edge/rpkgs-router.ts` +- Create: `edge/rpkgs-router.test.ts` +- Modify: `justfile` (add `edge-test`) + +**Produces:** a single-file script deployable as `bunnynet_compute_script.content`, reading `UNION_SLOTS` from the environment. + +- [ ] **Step 1: write the test matrix first** + +`edge/rpkgs-router.test.ts` spawns `deno run -A edge/rpkgs-router.ts` with `UNION_SLOTS=amd64/alpine324`, waits for `127.0.0.1:8080`, and issues requests with `redirect: "manual"`. + +Cases, asserted on the `location` header (or its absence): + +| # | path | User-Agent | expectation | +| --- | ------------------------------------------------------ | --------------------------------------------- | ------------------------------------------------------------ | +| 1 | `/amd64/alpine324/latest/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)` | 302 → `…/src/contrib/4.5/PACKAGES.gz` | +| 2 | same | `R (4.6.0 …)` | 302 → `…/src/contrib/4.6/PACKAGES.gz` | +| 3 | same, but slot `amd64/noble` | `R (4.5.3 …)` | no redirect (slot not in `UNION_SLOTS`) | +| 4 | `…/src/contrib/curl_7.1.0.tar.gz` | `R (4.5.3 …)` | no redirect | +| 5 | `…/src/contrib/4.5/PACKAGES.gz` | `R (4.5.3 …)` | no redirect (loop guard) | +| 6 | `…/src/contrib/PACKAGES.gz` | `curl/8.0` | no redirect (no R minor) | +| 7 | `/src/contrib/PACKAGES.gz` | alpine UA with `Alpine Linux … 3.24` | 302 → `/amd64/alpine324/latest/src/contrib/4.5/PACKAGES.gz` | +| 8 | `/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)`, no distro | 302 → `cran.r-project.org`, **not** a `linux-musl` slot | +| 9 | `/src/contrib/foo_1.0.tar.gz` | `R (4.5.1 aarch64-apple-darwin20 …)` | 302 → `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | +| 10 | `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | any | 302 → `cran.r-project.org` | +| 11 | any redirect above | — | `cache-control: no-store` | + +- [ ] **Step 2: run the tests and watch them fail** + +`just edge-test` → every case fails, because `edge/rpkgs-router.ts` does not exist. + +- [ ] **Step 3: write `edge/rpkgs-router.ts`** + +Order of evaluation in `onOriginRequest`: + +1. normalise `//` runs in the path +2. darwin `/src/contrib/*` → `/bin/macosx//contrib//` +3. `/bin/macosx/**` → CRAN +4. `/{arch}/{os}/latest/src/contrib/`: pass through if `rest` already starts with `/`, or is not an index file, or the slot is not in `UNION_SLOTS`, or the UA has no R minor; otherwise redirect into `/` +5. `/`, `/src/contrib`, `/src/contrib/**`: resolve arch+os from the UA, redirect to CRAN when the distro is unidentifiable, otherwise redirect to the qualified path, adding `/` under the same index-file rule +6. anything else: pass through + +The R minor comes from either `R/4.5.3` or `R (4.5.3 …)`, so a stock UA is enough. The `linux-gnu` / `linux-musl` fallback in `parseUserAgent` is deleted: those are not slot names. + +- [ ] **Step 4: run the tests until they pass** + +`just edge-test` + +- [ ] **Step 5: commit** + +```bash +git add edge/rpkgs-router.ts edge/rpkgs-router.test.ts justfile +git commit -m "feat(edge): route PACKAGES requests to the per-R-minor slot" +``` + +--- + +## Task 2: Manage the script from OpenTofu + +**Files:** + +- Modify: `cdn.tf` + +**Consumes:** `edge/rpkgs-router.ts` from Task 1. + +- [ ] **Step 1: add the resources** + +```terraform +resource "bunnynet_compute_script" "rpkgs_router" { + type = "middleware" + name = "rpkgs-router" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + default_value = "" + required = false +} +``` + +and replace `middleware_script = 29277` with `middleware_script = bunnynet_compute_script.rpkgs_router.id`. + +- [ ] **Step 2: validate** + +`tofu init -backend=false && tofu validate` + +- [ ] **Step 3: import the existing script (needs `BUNNYNET_API_KEY`)** + +```bash +tofu import bunnynet_compute_script.rpkgs_router 29277 +tofu plan +``` + +The plan must show an in-place `content` update and **no** replacement of the pull zone. A replacement means the import did not take. + +- [ ] **Step 4: commit** + +```bash +git add cdn.tf +git commit -m "feat(cdn): manage the edge middleware script from this repo" +``` + +--- + +## Task 3: Union index writer in bincraft + +**Files (repo `codefloe.com/rpkgs/bincraft`):** + +- Modify: `R/package_index.R` +- Test: `tests/testthat/test-package_index.R` + +**Produces:** `write_union_index(flat_records, minor_records)` returning the merged records, called from `upload_package_index()` when `r_minor` is set. + +- [ ] **Step 1: write the failing tests** + +- a package present in both slots keeps the per-minor record, with `Path = "4.5"` +- a package only in the flat slot survives with no `Path` +- a package only in the per-minor slot survives with `Path = "4.5"` +- a union smaller than the flat input raises an error rather than returning + +- [ ] **Step 2: run them and watch them fail** + +`Rscript -e 'testthat::test_file("tests/testthat/test-package_index.R")'` + +- [ ] **Step 3: implement `write_union_index()` and call it from `upload_package_index()`** + +After `update_PACKAGES()` has written the per-minor index, read the flat slot's `PACKAGES.rds`, set `Path = ` on the per-minor records, drop the flat records for packages the per-minor slot already has, and rewrite `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` in the per-minor slot. + +- [ ] **Step 4: run the tests until they pass** + +- [ ] **Step 5: commit and open the PR against bincraft** + +--- + +## Task 4: Roll out slot by slot + +- [ ] Re-index one slot (`amd64/alpine324`, R 4.5) and confirm the union index lists both `curl` (per-minor, `Path: 4.5`) and `jsonlite` (flat, no `Path`). +- [ ] Set `UNION_SLOTS = "amd64/alpine324"` and confirm in `reg.devxy.io/r/r-alpine:4.5-3.24` that `available.packages()` returns the union count and `"curl" %in% rownames(...)`. +- [ ] Add `arm64/alpine324`, then the remaining slots. + +`install.packages("curl")` will still fail to build on `alpine324` until that slot's source tarballs are replaced with real binaries. That is tracked separately. diff --git a/specs/2026-08-07-per-minor-edge-routing-design.md b/specs/2026-08-07-per-minor-edge-routing-design.md new file mode 100644 index 0000000..b55812c --- /dev/null +++ b/specs/2026-08-07-per-minor-edge-routing-design.md @@ -0,0 +1,174 @@ +# Design: Routing clients to per-R-minor binary slots + +Date: 2026-08-07 +Status: Approved (pending spec review) + +## Problem + +`bincraft` routes ABI-"risky" packages to a per-minor slot `…/latest/src/contrib//` and indexes every directory independently (`upload_package_index()` calls `cranlike::update_PACKAGES()` on one prefix at a time). +Nothing unions those indices, and `contrib.url()` only ever yields `/src/contrib`, so no value of `options(repos)` can address a per-minor slot. +Only `uvr` resolves per-minor URLs, which means the per-minor slots are invisible to `install.packages()` by construction. + +Measured on 2026-08-07: + +| slot | flat `src/contrib` | `src/contrib/4.5` | unique packages only in the per-minor slot | +| ----------------- | ------------------ | ----------------- | ------------------------------------------ | +| `amd64/alpine324` | 21 640 | 3 310 | 2 886 | +| `amd64/noble` | 24 495 | 398 | 23 | + +This is what issue #63 records as "missing binaries" on `alpine324`. +The packages are not missing; they are in a directory base R cannot reach. +The user-visible symptom in `reg.devxy.io/r/r-alpine:4.5-3.24` is: + +``` +> install.packages("curl") +Warning message: +package 'curl' is not available for this version of R +``` + +A second, unrelated defect exists on the same slot and is **out of scope here**: many `alpine324` tarballs are byte-identical CRAN _source_ tarballs that the index nevertheless stamps `Built: R 4.5.3; …-linux-musl`. +Routing exposes `curl`; only a rebuild of that slot makes it install. + +## Goal + +Let a stock `install.packages()` see one complete package list for its own R minor, without duplicating tarballs and without an R-version-varying cache key anywhere in the CDN. + +## Key constraint that drives the design + +R resolves a package's download URL from the index, not from the request path, and it keeps the `contriburl` it _asked for_ rather than the one it was redirected to. +Measured with `options(repos = …/latest)` against a middleware that redirects the index into `4.5/`: + +``` +curl available: TRUE +curl repo: …/latest/src/contrib # the flat URL, not the 4.5 one it was served from +``` + +So the union index is always addressed relative to the **flat** directory, whatever path it was fetched from. +`available.packages()` honours a `Path:` field and folds it into the `Repository` column, which gives the whole routing for free: + +- a per-minor record carries `Path: `, so its tarball is fetched from `…/src/contrib//` +- a flat record carries no `Path`, so its tarball is fetched from `…/src/contrib/` + +Verified end to end against the live CDN with a locally built union index for `amd64/alpine324` (31 507 records): + +``` +curl: 7.1.0 -> …/latest/src/contrib/4.5 -> curl_7.1.0.tar.gz 717 725 B +jsonlite: 2.0.0 -> …/latest/src/contrib -> jsonlite_2.0.0.tar.gz 1 055 849 B +``` + +The corollary is that the edge script must **not** rewrite tarball requests: every tarball URL is already correct when it leaves the client, and redirecting one into `/` would break exactly the flat packages the union is meant to preserve. + +The complementary trick does not work: R's `gzcon()` reads only the first member of a concatenated gzip stream (10 291 of an expected 31 931 records), so an edge-side merge would have to fully decompress and recompress both indices and additionally 404 `PACKAGES.rds` to stop R preferring it. +That is why the union is produced in `bincraft`, not at the edge. + +## Approaches considered + +| Approach | Where the union lives | Verdict | +| ----------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- | +| **A. Union index written by `bincraft` (chosen)** | per-minor `PACKAGES*`, per-minor entries carry `Path: ` | Edge does one redirect; `PACKAGES.rds` stays correct; no duplication | +| B. Merge at the edge | middleware fetches both indices, recompresses | ~2 MB decompress/recompress per cache fill, cache key must include the R minor, breaks R's `.rds` fast path | +| C. Move the minor up the path (`latest//src/contrib/`) | addressable by `options(repos)` directly | No edge logic at all, but a full layout migration and breaks the published URL contract | + +Chosen: **A**. + +## Architecture + +### bincraft: union index (separate PR) + +After writing a per-minor index, republish it as a union of that slot and the flat slot: + +1. Read the flat slot's `PACKAGES.rds` and the per-minor slot's own records. +2. Set `Path: ` on every per-minor record, so its tarball resolves into the per-minor directory. +3. Drop every flat record whose package is already present in the per-minor slot, so the per-minor build always wins, and leave the survivors without a `Path`. +4. Write the merged `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib//`. + +Guard: refuse to publish a union with fewer records than the flat index it was built from. +A truncated union is worse than no union, because it silently removes packages from every client on that minor. + +### Edge script (this repo) + +The script routes `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` requests, and nothing else. + +``` +normalize path +parseClient(UA) -> { rMinor, arch, os } # rMinor from "R (4.5.3 …)" or "R/4.5.3" +darwin branches # unchanged +if path is /{arch}/{os}/latest/src/contrib/PACKAGES* + already under // ? pass through # loop guard + rMinor known && slot in UNION_SLOTS ? 302 -> …/src/contrib//PACKAGES* + else pass through # flat slot, today's behaviour +if path is /src/contrib/… # bare root + resolve arch + os; unknown -> 302 to CRAN + then apply the same PACKAGES* rule +else pass through +``` + +Redirects carry `Cache-Control: no-store`. +Every cacheable URL is therefore UA-independent, and no cache key has to vary by R version. + +### Repaired bare-root detection + +The bare `https://cran.rpkgs.com` form is currently broken for every Linux client that uses a stock R user agent. +`ALPINE_REGEX`, `UBUNTU_REGEX` and `RHEL_REGEX` only match a Posit-style user agent the user has to set by hand; stock R never carries the distro, so the script falls through to `extractOs()` and redirects to a slot that does not exist: + +``` +UA: R (4.5.3 x86_64-pc-linux-musl …) -> 302 /amd64/linux-musl/latest/… (404) +UA: R (4.5.3 x86_64-pc-linux-gnu …) -> 302 /amd64/linux-gnu/latest/… (404) +``` + +The fallback to a phantom `linux-musl` / `linux-gnu` slot is removed. +An unidentifiable distro redirects to CRAN, which is the existing behaviour for an unparseable user agent. +The R _minor_ is always present in a stock user agent, so per-minor routing itself does not depend on distro detection. + +### Rollout gate + +`UNION_SLOTS` is a `bunnynet_compute_script_variable` listing the slots whose per-minor index is already a union. +It is empty by default, so deploying the script changes nothing until `bincraft` has backfilled a slot, and a rollback is a variable edit rather than a code deploy. +All slots currently carry `4.4`, `4.5` and `4.6`; a client on any other minor falls through to the flat slot. + +### Deployment from this repo + +The script is a file in the repo, applied by the existing OpenTofu configuration: + +``` +edge/rpkgs-router.ts # the script +edge/rpkgs-router.test.ts # UA x path -> expected Location matrix +cdn.tf # bunnynet_compute_script + _variable +``` + +Provider `BunnyWay/bunnynet` v0.17.0 (already pinned) ships `bunnynet_compute_script` with `content` loadable via `file()`, plus `bunnynet_compute_script_variable`. +`middleware_script = bunnynet_compute_script.rpkgs_router.id` replaces the hard-coded `29277`, after a one-time `tofu import` of the existing script. + +## Error handling + +- Unknown R minor, or a slot not listed in `UNION_SLOTS`: pass through to the flat slot. + The client sees exactly today's behaviour. +- Unparseable distro on the bare-root form: redirect to CRAN. +- A request already under `…/src/contrib//`: pass through, so a redirect can never loop. +- A per-minor slot that does not exist for a listed minor: the client gets the origin's 404. + `UNION_SLOTS` is the operator's assertion that the slot is ready, so this is a configuration error, not a runtime condition to paper over. + +## Testing + +Local, before any apply: `deno run -A edge/rpkgs-router.ts` serves the middleware against the real origin, so `edge/rpkgs-router.test.ts` drives the whole matrix against that local server. + +- User agent matrix: R 4.4 / 4.5 / 4.6 on musl and gnu, both arches, Posit-style and stock forms, plus a darwin UA and a non-R UA. +- Path matrix: `PACKAGES`, `PACKAGES.gz`, `PACKAGES.rds`, a tarball, a path already under `4.5/`, and `/src/contrib/…` on the bare root. +- Assertion is the `Location` header (or its absence), not the body. + +After apply, a smoke test against `cran.rpkgs.com`: + +- `available.packages()` inside `reg.devxy.io/r/r-alpine:4.5-3.24` returns the union count, and `"curl" %in% rownames(...)` is `TRUE`. +- A flat-slot package still downloads from `…/src/contrib/`, and a per-minor package downloads from `…/src/contrib//`. + +## Out of scope + +- `Meta/archive.rds` stays flat-only, so `remotes::install_version()` does not see per-minor archives. +- The `alpine324` source-tarball defect: that slot serves CRAN sources stamped as binaries, and needs a rebuild independent of this work. +- Any change to how `uvr` resolves per-minor URLs; it already addresses the slots directly. + +## Split of work + +1. `bincraft`: union index writer plus its guard, and a re-index of one slot to validate. +2. This repo: `edge/rpkgs-router.ts`, its test matrix, and the `cdn.tf` resources with `UNION_SLOTS` empty. +3. Enable `UNION_SLOTS` slot by slot as `bincraft` backfills them. From 6bc8213a2f7e229e1d953e170c987023a04dfcd3 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sat, 8 Aug 2026 00:32:07 +0000 Subject: [PATCH 099/125] chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker to v3.10.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80650cc..2334bd5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.8.0 + rev: v3.10.0 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ From 4cb8b692ab4d38b10e785a0f947e83dd3c098bd7 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sun, 9 Aug 2026 00:32:28 +0000 Subject: [PATCH 100/125] chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker to v3.11.1 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2334bd5..82461cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.10.0 + rev: v3.11.1 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ From 1e843523a0312f3943fd11aca8f64e0117f7eca9 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 10:02:05 +0000 Subject: [PATCH 101/125] fix(ci): gate the three ungated pipelines on their own variable (#155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem A manual `crow pipeline create` instantiates **every** pipeline in `.crow/`, and each one decides for itself whether to run. Three had nothing to decide with — their only manual condition was a bare `event: manual`: - `auto-apply-patches` — pushes to `auto/registry-patch-proposals` and opens/updates a PR - `weekly-patch-proposals` — posts and edits two Forgejo issues - `trial-build-registry` — starts a build per matrix row, on both arches So they fired on *any* manual trigger in this repo, whatever it was for. That is how they came to run alongside a manual `process-updates` run for `alpine-324-arm64` (#10723), which is also why that pipeline is marked failure. `repair-built-stamp.yaml` already documents the rule this breaks: > The gate variable is `repair_built_stamp`, not `target_arch` … Every pipeline here gates on a variable named after itself for exactly that reason. ## What this changes Each of the three gets a gate variable named after the pipeline, `evaluate`d on the manual event, defaulting to off: ```yaml variables: auto_apply_patches: description: 'Run the auto-patch proposer. Also gates this pipeline.' options: ['true', 'false'] default: 'false' when: - event: manual evaluate: 'auto_apply_patches == "true"' - event: cron cron: auto-apply-patches ``` Cron triggers are untouched, so the scheduled runs behave exactly as before. The run-manually comments in all three headers were also stale: they documented `--var task=` with `woodpecker-cli`, and no pipeline evaluates a `task` variable. They now show the real invocation. ## Note on the sibling pipelines The already-gated pipelines use `default: all` (e.g. `weekly_rebuild_missing`). If Crow applies a declared default to a variable that an API-created pipeline never passed, those would match on an unrelated manual run too — `weekly-rebuild-missing` would be an expensive way to find out. I could not settle that from #10723 because its step logs have since expired, so I left them alone rather than guess. The three fixed here default to `'false'`, which is safe under either semantics. ## Verification `crow lint .crow/` passes. Auditing every pipeline that accepts a manual event now reports a gate on all ten: ``` build-all-versions-install-deps.yaml: gated auto-apply-patches.yaml: gated weekly-patch-proposals.yaml: gated weekly-audit-missing.yaml: gated repair-built-stamp.yaml: gated archive-missed-packages.yaml: gated weekly-rebuild-missing.yaml: gated trial-build-registry.yaml: gated build-all-versions.yaml: gated process-updates.yaml: gated ``` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/155 --- .crow/auto-apply-patches.yaml | 15 ++++++++++++++- .crow/trial-build-registry.yaml | 16 ++++++++++++++-- .crow/weekly-patch-proposals.yaml | 18 ++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index 0076353..7c33c5b 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -9,14 +9,27 @@ # FORGEJO_TOKEN is used for both the branch push and opening the PR (no separate # write-scoped secret needed). Register the `auto-apply-patches` cron in the crow # UI, or run manually: -# woodpecker-cli pipeline create --var task=auto-apply-patches --branch=main 7 +# crow pipeline create --branch main \ +# --var auto_apply_patches=true devxy/build-cran-binaries +# +# The gate variable is `auto_apply_patches`, named after the pipeline: a manual +# run instantiates every pipeline in `.crow/`, so one without its own gate runs +# on *any* manual trigger in this repo. This one pushes a branch and opens a PR, +# so it must stay off unless it is what was asked for. variables: + auto_apply_patches: + description: 'Run the auto-patch proposer. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' patch_limit: description: 'Max candidates to propose per run (top by failure volume).' default: '10' when: - event: manual + evaluate: 'auto_apply_patches == "true"' - event: cron cron: auto-apply-patches diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 73856a1..2195423 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -7,15 +7,27 @@ # # The repo uses no `pull_request` triggers, so this runs manually against the # branch (or on a cron); point it at the auto-patch branch via `patch_branch`: -# woodpecker-cli pipeline create --var task=trial-build-registry \ -# --var patch_branch=auto/registry-patch-proposals --branch=main 7 +# crow pipeline create --branch main --var trial_build_registry=true \ +# --var patch_branch=auto/registry-patch-proposals devxy/build-cran-binaries +# +# The gate variable is `trial_build_registry`, named after the pipeline: a +# manual run instantiates every pipeline in `.crow/`, so one without its own +# gate runs on *any* manual trigger in this repo. This one starts a build per +# matrix row on both arches, which is far too expensive to fire by accident. variables: + trial_build_registry: + description: 'Trial-build the branch new registry entries. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' patch_branch: description: 'Branch whose new registry entries to trial-build.' default: auto/registry-patch-proposals when: - event: manual + evaluate: 'trial_build_registry == "true"' - event: cron cron: trial-build-registry diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 027d41c..712e87d 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -8,8 +8,26 @@ # Global across platforms (the classifier groups over all of single_builds), so # a single job -- no matrix. Clones read-only; the only writes are the two # Forgejo issues via FORGEJO_TOKEN. +# +# Run manually with: +# crow pipeline create --branch main \ +# --var weekly_patch_proposals=true devxy/build-cran-binaries +# +# The gate variable is `weekly_patch_proposals`, named after the pipeline: a +# manual run instantiates every pipeline in `.crow/`, so one without its own +# gate runs on *any* manual trigger in this repo. This one posts and edits +# Forgejo issues, so an unrelated manual run must not fire it. +variables: + weekly_patch_proposals: + description: 'Run the weekly failure triage. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' + when: - event: manual + evaluate: 'weekly_patch_proposals == "true"' - event: cron cron: weekly-patch-proposals From 0c86b2692d07e18decc55b4738a3cfa2562ab892 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 10:08:40 +0000 Subject: [PATCH 102/125] fix(cdn): declare the User-Agent cache vary instead of dropping it (#156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `tofu plan` after importing the middleware script shows: ``` ~ resource "bunnynet_pullzone" "cran_rpkgs_com" { ~ cache_vary_headers = [ - "User-Agent", ] ``` The zone carries `cache_vary_headers = ["User-Agent"]`, set before this configuration existed. `cdn.tf` never declared it, so the first apply of the managed middleware would remove it — as a side effect of an unrelated change, with no decision recorded anywhere. ## What this changes Declares the attribute with its current value, so the pull zone is a no-op in that plan. ## Why keep it rather than let it go On paper the router makes it redundant. The only UA-dependent responses it produces are redirects, and those carry `Cache-Control: no-store`; their targets are concrete per-slot, per-minor URLs whose content depends only on the path. Dropping the vary would also be a genuine win, since otherwise every distinct R version string (`R (4.5.3 x86_64-pc-linux-musl …)`) keys its own copy of every tarball. It stays anyway, for now: - it is the second line of defence against the one failure mode that would be quiet and confusing — an R 4.6 client served the 4.5 index - Bunny honouring `no-store` on an edge-script response has been confirmed for today's redirects (`cdn-cache: BYPASS` on `max-age=0`), but not for the new script in production - keeping it is the status quo, so it cannot regress anything Removing it is worth doing on its own, once per-minor routing is confirmed live and the redirects can be observed bypassing cache — not as a side effect of enabling that routing. ## Verification `tofu validate` passes. Re-planning after this merges should leave `bunnynet_pullzone.cran_rpkgs_com` unchanged, reducing the plan to the script `content` update and the new `UNION_SLOTS` variable. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/156 --- cdn.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cdn.tf b/cdn.tf index fe1b25f..5e8899d 100644 --- a/cdn.tf +++ b/cdn.tf @@ -105,6 +105,21 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { request_coalescing_enabled = true block_post_requests = true + # Set on the zone since before this configuration existed; declared here so + # `tofu apply` stops silently removing it. + # + # The router makes it redundant on paper: the only UA-dependent responses it + # produces are redirects, and those carry `Cache-Control: no-store`, while + # their targets are concrete per-slot, per-minor URLs whose content depends + # only on the path. Dropping it would also be a real win, because otherwise + # every distinct R version string keys its own copy of every tarball. + # + # It stays for now anyway: it is the second line of defence against the one + # failure that would be quiet and confusing (an R 4.6 client served the 4.5 + # index), and removing it is worth doing on its own once per-minor routing is + # confirmed live, not as a side effect of enabling that routing. + cache_vary_headers = ["User-Agent"] + limit_requests = 5000 limit_connections = 1000 From 070396908819b9365e5cf824e3f387ee875993bf Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 10:37:41 +0000 Subject: [PATCH 103/125] fix(audit): count a source fallback as a missing binary (#157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The audit's definition of "missing" was `missing_dt <- cran_dt[!s3_dt]`, where `s3_dt` came from listing the bucket. But an object being present does not mean a binary was built: when a build fails, bincraft publishes the CRAN source tarball in its place so the package stays installable, and a listing cannot tell the two apart. So every failed build looked done. On `amd64/alpine324` that is 13 547 of 24 134 comparable objects (56%) — all CRAN sources from the June 2026 bootstrap, none ever reported here, so `weekly-rebuild-missing` never retried any of them. `amd64/noble` sits at 4.6% for comparison. ## What this changes Reads the slot's own `PACKAGES` index instead of listing the bucket, and counts only records carrying a `Built` stamp. bincraft writes that stamp only for what it actually built (rpkgs/bincraft#105), so the index answers the question a listing cannot. Side effects of reading the index rather than the bucket: - the audit reports how many records the index holds and how many are served as source, so the gap is visible in the log - `s3fs` is no longer used, so the audit needs no B2 credentials - an object present but absent from the index is now counted as missing, which is correct: clients only see the index ## Sequencing This is inert until the matching bincraft change ships and a slot is re-indexed. A slot last indexed by an older bincraft carries `Built` on every record, including the fallbacks, so the audit reports exactly what it does today. Verified against the live indices: ``` amd64/alpine324: records=24152 built=24152 source-served=0 amd64/noble: records=24681 built=24681 source-served=0 ``` After a re-index those `source-served` counts become the real ones, and the packages behind them start appearing in issue #63 and getting rebuilt. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/157 --- local/weekly-missing-binaries-audit.R | 78 +++++++++++++++------------ 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index 7b0b65d..e7ca776 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -7,7 +7,6 @@ options(error = function() { suppressPackageStartupMessages(library(data.table)) library(DBI, quietly = TRUE) library(RPostgres, quietly = TRUE) -library(s3fs, quietly = TRUE) library(jsonlite, quietly = TRUE) library(httr2, quietly = TRUE) @@ -50,8 +49,8 @@ cat(sprintf( )) # --------------------------------------------------------------------------- -# 1. Query PostgreSQL for known build failures (before s3fs init to avoid -# C++ pointer conflicts between s3fs/curl and RPostgres/libpq) +# 1. Query PostgreSQL for known build failures (before anything that uses curl, +# to avoid C++ pointer conflicts between curl and RPostgres/libpq) # --------------------------------------------------------------------------- cat("Connecting to PostgreSQL...\n") con <- DBI::dbConnect( @@ -97,62 +96,71 @@ cran_dt <- data.table( ) # --------------------------------------------------------------------------- -# 3. S3 tarballs +# 3. Published binaries # --------------------------------------------------------------------------- -cat("Connecting to S3...\n") -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE -) - -s3_path <- sprintf( - "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", +# Read the slot's own index rather than listing the bucket. An object being +# present does not mean a binary was built: when a build fails, bincraft +# publishes the CRAN source tarball in its place so the package stays +# installable, and a bucket listing cannot tell the two apart. That is how +# amd64/alpine324 came to hold 13,547 CRAN sources that this audit never +# reported. bincraft stamps `Built` only on records it actually built, so the +# index answers the question a listing cannot. +# +# A slot last indexed by a bincraft that predates the source-fallback fix +# stamps `Built` on every record, including the fallbacks, so this reports +# exactly what it used to until that slot is re-indexed. +index_url <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", arch, s3_codename ) -cat(sprintf("Listing S3 path: %s\n", s3_path)) +cat(sprintf("Reading package index: %s\n", index_url)) -s3_pkgs <- tryCatch( - s3fs::s3_dir_ls(s3_path, recurse = FALSE), +index <- tryCatch( + { + con <- gzcon(url(index_url, open = "rb")) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "Built")) + }, error = function(e) { cat(sprintf( - "WARNING: Could not list S3 path %s: %s\n", - s3_path, + "WARNING: Could not read %s: %s\n", + index_url, conditionMessage(e) )) - character(0) + NULL } ) -file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) -parts <- parts[sapply(parts, length) == 3] -if (length(parts) == 0) { - s3_dt <- data.table(Package = character(0), Version = character(0)) +if (is.null(index) || nrow(index) == 0) { + binary_dt <- data.table(Package = character(0), Version = character(0)) } else { - s3_dt <- data.table( - Package = sapply(parts, `[`, 2), - Version = sapply(parts, `[`, 3) + built <- !is.na(index[, "Built"]) + binary_dt <- data.table( + Package = as.character(index[built, "Package"]), + Version = as.character(index[built, "Version"]) ) + cat(sprintf( + "Index holds %d records, %d of them built binaries (%d served as CRAN source)\n", + nrow(index), + sum(built), + sum(!built) + )) } cat(sprintf( - "S3 contains %d tarballs for %s/%s\n", - nrow(s3_dt), + "S3 contains %d binaries for %s/%s\n", + nrow(binary_dt), arch, s3_codename )) # --------------------------------------------------------------------------- -# 4. Find missing packages (CRAN release version not in S3) +# 4. Find missing packages (CRAN release version without a binary in S3) # --------------------------------------------------------------------------- setkey(cran_dt, Package, Version) -setkey(s3_dt, Package, Version) -missing_dt <- cran_dt[!s3_dt] +setkey(binary_dt, Package, Version) +missing_dt <- cran_dt[!binary_dt] cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) # --------------------------------------------------------------------------- From f8e31af75b3ae418b858dca3d6baeda73e651834 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 15:31:14 +0000 Subject: [PATCH 104/125] fix(ci): make every manual gate default to a value that matches nothing (#158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem A manual `crow pipeline create` instantiates **every** file in `.crow/`, and a declared variable default is applied even when the run never passed that variable. A gate is therefore only a gate if its default matches nothing. #155 fixed the three pipelines that had no manual gate at all. It missed that a *permissive default* leaves a pipeline just as exposed. Demonstrated the expensive way: creating a pipeline with only ``` --var weekly_audit_missing=alpine-324-amd64 ``` also started `build-all-versions` — because its gate `target_arch` defaults to `amd64`, which matches its own amd64 matrix rows — and `process-updates` across every row, because that gate defaults to `all`. The run was killed before any `Upload package indexes` step produced output and both alpine324 indices were verified unchanged, but `build-all-versions` uploads binaries and rewrites indexes, so the next one might not be caught in time. Before: | pipeline | gate | default | fired on an unrelated manual run | | --- | --- | --- | --- | | `build-all-versions` | `target_arch` | `amd64` | amd64 rows — builds and uploads | | `build-all-versions-install-deps` | `target_arch` | `amd64` | amd64 rows | | `weekly-rebuild-missing` | `weekly_rebuild_missing` | `all` | every row | | `weekly-audit-missing` | `weekly_audit_missing` | `all` | every row | | `process-updates` | `process_cran_updates` | `all` | every row | | `repair-built-stamp` | `repair_built_stamp` | `arm64` | arm64 rows | `archive-missed-packages` was the one that behaved, because its gate variable is never declared and so matches nothing. That is the property this restores everywhere. ## What this changes Each of the six gets a `none` option on its gate variable and defaults to it, so a manual run has to name its target explicitly. The reason is recorded next to the default, where someone would go to change it. `none` is used rather than dropping the default so the expression always has a defined value to compare, instead of relying on undefined-variable semantics. Cron triggers are untouched — they match on the `cron:` name, not the variable. ## Verification `crow lint .crow/` reports all ten configs valid. Auditing every pipeline that accepts a manual event: ``` archive-missed-packages.yaml: gate=task default= auto-apply-patches.yaml: gate=auto_apply_patches default='false' build-all-versions-install-deps.yaml gate=target_arch default=none build-all-versions.yaml: gate=target_arch default=none process-updates.yaml: gate=process_cran_updates default=none repair-built-stamp.yaml: gate=repair_built_stamp default=none trial-build-registry.yaml: gate=trial_build_registry default='false' weekly-audit-missing.yaml: gate=weekly_audit_missing default=none weekly-patch-proposals.yaml: gate=weekly_patch_proposals default='false' weekly-rebuild-missing.yaml: gate=weekly_rebuild_missing default=none ``` Every gate now defaults to something that matches no matrix row. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/158 --- .crow/build-all-versions-install-deps.yaml | 8 ++++++-- .crow/build-all-versions.yaml | 8 ++++++-- .crow/process-updates.yaml | 8 ++++++-- .crow/repair-built-stamp.yaml | 8 ++++++-- .crow/weekly-audit-missing.yaml | 8 ++++++-- .crow/weekly-rebuild-missing.yaml | 8 ++++++-- 6 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 448cff3..2a6d9c2 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -3,12 +3,16 @@ # Variables are declared so the manual-run form exposes them (crow #1165); # they are merged with build-all-versions' identical declarations. variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. target_arch: - description: 'Architecture to build.' + description: 'Architecture to build, or "none" to run nothing.' options: + - none - amd64 - arm64 - default: amd64 + default: none OS: description: 'Base OS image name.' options: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 9ab888a..fb1229d 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -4,12 +4,16 @@ # image and cache volume. Placement is via the group label (rpkgs-amd64/rpkgs-arm64). # Skip list lives in local/excluded-packages.json (read by local/build-all.R). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. target_arch: - description: 'Architecture to build.' + description: 'Architecture to build, or "none" to run nothing.' options: + - none - amd64 - arm64 - default: amd64 + default: none OS: description: 'Base OS image name.' options: diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 1e4833c..7600876 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -7,9 +7,13 @@ # ("all" = every os/arch). # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. process_cran_updates: - description: "Manual run target: a specific -, or 'all' for every os/arch." + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." options: + - none - all - alpine-322-amd64 - alpine-322-arm64 @@ -29,7 +33,7 @@ variables: - ubuntu-2404-arm64 - ubuntu-2604-amd64 - ubuntu-2604-arm64 - default: all + default: none when: - event: cron diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml index 83821e9..e56e1e9 100644 --- a/.crow/repair-built-stamp.yaml +++ b/.crow/repair-built-stamp.yaml @@ -19,12 +19,16 @@ # --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \ # --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. repair_built_stamp: - description: 'Architecture of the slot to repair. Also gates this pipeline.' + description: 'Architecture of the slot to repair, or "none" to run nothing.' options: + - none - amd64 - arm64 - default: arm64 + default: none OS: description: 'Base OS image name.' options: diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 4efe409..3a9d98f 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -7,9 +7,13 @@ # ("all" = every os/arch). # Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. weekly_audit_missing: - description: "Manual run target: a specific -, or 'all' for every os/arch." + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." options: + - none - all - alpine-322-amd64 - alpine-322-arm64 @@ -29,7 +33,7 @@ variables: - ubuntu-2404-arm64 - ubuntu-2604-amd64 - ubuntu-2604-arm64 - default: all + default: none when: - event: cron diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 03a13f9..0353901 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -8,9 +8,13 @@ # single - to run just one. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, or 'all' for every os/arch." + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." options: + - none - all - alpine-322-amd64 - alpine-322-arm64 @@ -30,7 +34,7 @@ variables: - ubuntu-2404-arm64 - ubuntu-2604-amd64 - ubuntu-2604-arm64 - default: all + default: none when: - event: cron From a126d74cd37422d2c0a9e8dec782173f945585eb Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 16:27:51 +0000 Subject: [PATCH 105/125] fix(build): keep source fallbacks out of the S3 package cache (#159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem This is the gap flagged in rpkgs/bincraft#106. `build_binary_package()` has a fast path that compares against `s3_package_cache` instead of querying S3 per package, and that cache is produced here: ```r s3_pkgs <- s3fs::s3_dir_ls(".../latest/src/contrib", recurse = TRUE) saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds") ``` A raw bucket listing cannot tell a binary from a package whose build failed and was published as its CRAN source — the two occupy the same key. So every source fallback reads as "already built" and is skipped for good. That is how `alpine324` accumulated ~13.5k of them. The same listing feeds `s3_dt`, which is subtracted from the build list at line 194 (`pkgs <- pkgs_no_error[!s3_dt]`). That one matters more: it excludes the very packages that need building, before `build_binary_package()` is even called. ## What this changes Drops from the listing every object the slot's own index reports as served from source. bincraft leaves the `Built` stamp off exactly those records (rpkgs/bincraft#105), so the index already carries the answer and no credentials, downloads or extra API calls are needed. Both consumers are fixed: the saved cache and `s3_dt`. The cache stays a plain filename vector, so the build container still needs no `s3fs`/reticulate — that was the point of saving it in the first place. Two deliberately conservative edges: - archived objects have no index record, so they are kept. Unknown means binary, never "rebuild it". - if the index cannot be read, the full listing is kept and a warning is printed, so a CDN blip cannot mass-schedule a rebuild. ## Verification The script parses, and the new block run against the live indices: ``` amd64/alpine324: index=24235 source-served=13542 e.g. AATtools_0.0.3.tar.gz, ABCDscores_7.0.0.tar.gz amd64/noble: index=24681 source-served=0 ``` `alpine324` is re-indexed by bincraft 5.1.1, so 13 542 objects drop out and those packages become buildable. `noble` has not been re-indexed yet, so every record still carries `Built`, nothing is dropped, and its behaviour is exactly what it is today — the safe failure mode this relies on. The new log line makes it visible per run: ``` S3 cache: N objects, M served as CRAN source, K usable binaries ``` ## Sequencing Needs rpkgs/bincraft#106 (and a release) before a rebuild actually builds: this fixes the bulk build path's list, #106 fixes the per-package pre-build skip. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/159 --- local/packages-to-build.R | 59 +++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 887952d..3b508fb 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -89,14 +89,61 @@ s3_pkgs <- s3fs::s3_dir_ls( recurse = TRUE ) -# Save the raw S3 file listing for the build step to use as s3_package_cache +file_names <- basename(s3_pkgs) + +# An object occupying a key is not proof a binary was built: a package whose +# build failed has its CRAN source published under exactly that name. Left in +# the cache, `build_binary_package()` reads it as "already built" and skips the +# package forever, which is how alpine324 accumulated ~13.5k source tarballs. +# +# bincraft stamps `Built` only on records it actually built, so the slot's own +# index distinguishes them. A slot last indexed by a bincraft that predates that +# fix stamps `Built` on everything, so the cache is then unchanged from before. +# Archived objects have no index record and are kept: unknown means binary, +# never "rebuild it". +index_url <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", + arch, + codename +) +source_served <- tryCatch( + { + con_idx <- gzcon(url(index_url, open = "rb")) + on.exit(close(con_idx), add = TRUE) + idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) + sprintf( + "%s_%s.tar.gz", + idx[is.na(idx[, "Built"]), "Package"], + idx[is.na(idx[, "Built"]), "Version"] + ) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read %s (%s); keeping the full S3 cache\n", + index_url, + conditionMessage(e) + )) + character(0) + } +) + +binary_cache <- setdiff(file_names, source_served) +cat(sprintf( + "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", + length(file_names), + length(file_names) - length(binary_cache), + length(binary_cache) +)) + +# Save the S3 file listing for the build step to use as s3_package_cache. # This avoids loading s3fs/reticulate in the build container, saving memory for # the dependency-installer subprocesses -saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds") - -file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) +saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") +# Built from the filtered listing, not the raw one: `s3_dt` is subtracted from +# the build list below, so a source fallback left in here would exclude the very +# package that needs building. +matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", binary_cache) +parts <- regmatches(binary_cache, matches) parts <- parts[sapply(parts, length) == 3] s3_dt <- data.table( Package = sapply(parts, `[`, 2), From 6372f0f928f81c76aaca3c3b6f71541a04a976d4 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 10 Aug 2026 06:30:25 +0000 Subject: [PATCH 106/125] fix(ci): refresh the apt index before uvr installs system dependencies (#161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unblocks the ubuntu 26.04 builds without waiting for an image rebuild. ## What broke ``` i Installing R package dependencies ! Error in installing dependencies for package huge with tag 2.0.1: uvr sync --install-system-deps --library /mnt/cache/R-pkgs-4.4 failed (exit 1) ! WARN Missing system dependencies for 1 package(s) igraph needs: libglpk-dev > Running: apt-get install -y libglpk-dev E: Unable to locate package libglpk-dev ``` Neither the package name nor the sysreq mapping is wrong. `libglpk-dev` is `5.0-2build1` in `universe`, which `ubuntu:resolute` enables by default. What is missing is the apt index: uvr v0.4.5 runs `apt-get install` with no `apt-get update` before it, and every ubuntu build image ends its apt layers with `rm -rf /var/lib/apt/lists/*`, so the shipped image has no index at all. Not specific to 26.04 — jammy and noble fail identically for any sysreq the image's preinstall line doesn't already cover. 26.04 surfaced it first, through `igraph` and everything depending on it (`huge` was the first to fail). ## Fix `apt-get update` before the sync in `local/uvr-install.sh`. That one site covers every pipeline: `build-all-versions`, `build-all-versions-install-deps`, `weekly-rebuild-missing`, `weekly-audit-missing`, `trial-build-registry`, `auto-apply-patches` and `weekly-patch-proposals` all reach it, either directly or through `install-bincraft.R`. The index it populates persists for the rest of the step, so the bincraft-driven `uvr sync` calls that follow are covered too. Guarded on `apt-get` and non-fatal: `apk add` fetches its index implicitly and dnf refreshes expired metadata on its own, so alpine and redhat are unaffected and skip it. ## Why here as well as in the image The real fix is upstream `e491b2e`, which refreshes the index before the auto-install. It landed one day after v0.4.5 was tagged, so no release carries it ([nbafrank/uvr#250](https://github.com/nbafrank/uvr/issues/250)); [build-env-images#23](https://codefloe.com/rpkgs/build-env-images/pulls/23) pins the ubuntu image to a commit that has it. That only reaches CI after an image rebuild is triggered. This change takes effect on the next pipeline run. It also stays useful afterwards: a baked index goes stale within weeks, and a stale index turns the same install into a 404 on the `.deb`. ## Verified In `ubuntu:resolute`, with the image state reproduced exactly (R installed, then `rm -rf /var/lib/apt/lists/*` to empty the index again): | Check | Result | |---|---| | `apt-get install -y libglpk-dev`, empty index | `E: Unable to locate package libglpk-dev` — the reported failure, reproduced | | Same container after `apt-get update` | `Candidate: 5.0-2build1`, `Components: main universe restricted multiverse` | | `igraph` + `UVR_INSTALL_SYSREQS=1 uvr sync`, empty index, uvr with the refresh | `Setting up libglpk-dev:amd64 (5.0-2build1)`, `System dependencies installed.`, `Installed 11 package(s)` | | `sh -n` and `shellcheck local/uvr-install.sh` | pass | ## Reverting Delete the guarded block. It is marked TEMPORARY with the upstream reference, and can go once the images ship a uvr above v0.4.5. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/161 --- local/uvr-install.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/local/uvr-install.sh b/local/uvr-install.sh index 14b7f78..c5d37d3 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -87,4 +87,23 @@ cd "$project_dir" # --no-install: resolve and lock only. The install happens in the sync below, # which is the only command that honours --library. "$uvr_bin" add --no-install "$@" + +# TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs +# `apt-get install` for every resolved system dependency without refreshing the +# index first, and the ubuntu build images end their apt layers with +# `rm -rf /var/lib/apt/lists/*`. With no index apt cannot resolve a package that +# exists and is enabled, so `igraph needs: libglpk-dev` fails the whole build +# with `E: Unable to locate package libglpk-dev` on ubuntu 26.04. +# +# Fixed upstream in `e491b2e`, tagged one day after v0.4.5 (nbafrank/uvr#250), +# and the images pick it up via build-env-images#23 — but only after an image +# rebuild is triggered, which is why this runs here too. +# +# apt only: `apk add` fetches its index implicitly and dnf refreshes expired +# metadata on its own. Non-fatal, since a refresh failure still leaves whatever +# index is already there, and the install's own error is the more actionable one. +if command -v apt-get >/dev/null 2>&1; then + apt-get update -qq || echo "warning: apt-get update failed; continuing" >&2 +fi + "$uvr_bin" sync --library "$target_lib" --install-system-deps From 01b8ab43dfffee0e71bc63617275d5cffe5b2d88 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 10 Aug 2026 06:30:35 +0000 Subject: [PATCH 107/125] fix(rebuild): re-index and purge the CDN after a rebuild (#160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The rebuild now works — `AATtools 0.0.3` was detected as a source fallback, built, and published: ``` ℹ `upload_single_binary()`: Replacing the CRAN source published for AATtools 0.0.3 … with the binary. ✔ Successfully uploaded package AATtools with tag 0.0.3. ``` But clients still get the source, and will for about a year: ``` $ curl -sI .../src/contrib/AATtools_0.0.3.tar.gz etag: "ea8127d953ca6a2f118ea49441772af6" # CRAN's source MD5 cdn-cache: HIT cdn-cachedat: 08/09/2026 16:43:32 # predates the 18:12 upload ``` Two causes, both specific to a rebuild: 1. **The slot is never re-indexed.** `weekly-rebuild-missing` has no `upload_package_index` step, so the index keeps the old MD5 and — for anything that had been served from source — no `Built` stamp. This one self-heals at the next `process-updates` run. 2. **The tarball URL is never purged.** A normal update publishes new packages at *new* URLs, so `purge_cdn_cache.sh` only needs the five index files. A rebuild replaces an object *in place*, and the zone caches tarballs for `cache_expiration_time = 31919000` (~370 days). This does not self-heal. Nothing about a stale package looks wrong from the outside, which is what makes it worth fixing rather than documenting. ## What this changes **Re-index at the end of a rebuild**, flat and per-minor, mirroring the tail of `process-updates`. The codename is detected from the image's `/etc/os-release` (as `local/packages-to-build.R` already does) rather than adding `OS_ID` to all 18 matrix rows. **Purge the zone afterwards**, via a new `scripts/purge_cdn_zone.sh`. One call to `POST /pullzone/{id}/purgeCache` covers every replaced object, and all three hostnames — `cran.devxy.io`, `cran.allianceswisspass.devxy.io`, `cran.rpkgs.com` — share pull zone `3857050`, confirmed from the `cdn-pullzone` response header. Purging per URL was the alternative and is worse here: ~13.5k rate-limited calls per arch, where a single missed call leaves a package silently stale. The cost of the zone purge is a cold cache for everything else, which is why it stays out of the daily update path — `purge_cdn_cache.sh` is untouched. The purge runs on failure too (`when: status: [success, failure]`): a rebuild that died part-way still replaced objects, and those are exactly the ones a stale edge keeps hiding. ## Verification `crow lint .crow/` reports all ten configs valid; `bash -n` on the new script passes; prek hooks pass. Not yet exercised against Bunny — it needs `BUNNYNET_API_KEY`, which is a CI secret. The failure mode is explicit rather than silent: any status other than 200/204 prints the response body and exits non-zero. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/160 --- .crow/weekly-rebuild-missing.yaml | 33 ++++++++++++++++++++ scripts/purge_cdn_zone.sh | 52 +++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100755 scripts/purge_cdn_zone.sh diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 0353901..c86fe5a 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -163,6 +163,17 @@ steps: - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + # A rebuild replaces objects in place, so the slot's index still advertises + # the old MD5 and, for anything that had been served from source, no Built + # stamp. Re-index here rather than waiting for the next process-updates + # run, or the rebuilt binaries stay invisible to clients until then. + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done backend_options: docker: resources: @@ -172,3 +183,25 @@ steps: limits: memory: 18Gi cpu: 3000m + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + # All hostnames on the zone share this id, so one purge covers + # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. + BUNNY_PULLZONE: '3857050' + commands: + - apk add --no-cache -q bash curl git + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" + # A rebuild that died part-way still replaced objects, and those are exactly + # the ones a stale edge would keep hiding, so purge either way. + when: + - status: [success, failure] diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh new file mode 100755 index 0000000..391a814 --- /dev/null +++ b/scripts/purge_cdn_zone.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Purge the entire BunnyCDN pull zone. +# +# `purge_cdn_cache.sh` purges the five index files by URL, which is right after +# a normal update: new packages arrive at new URLs, so only the index is stale. +# +# A rebuild is different. It replaces an object *in place*: a package whose +# build failed was published as its CRAN source, and the rebuilt binary takes +# exactly the same URL. The zone caches tarballs for ~370 days +# (`cache_expiration_time` in cdn.tf), so without a purge every client keeps +# receiving the source tarball for up to a year, and nothing about it looks +# wrong from the outside. +# +# Purging per URL would mean one API call per replaced package -- ~13.5k per +# arch against a rate-limited endpoint, where a single missed call leaves a +# silently stale package. One zone purge is a single call regardless of how many +# objects were replaced. The cost is a cold cache for everything else, which is +# why this is not used by the daily update path. +# +# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io, +# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them. +# +# Usage: +# purge_cdn_zone.sh +# +set -euo pipefail + +if (($# < 2)); then + echo "usage: $0 " >&2 + exit 2 +fi + +api_key="$1" +zone_id="$2" + +echo "Purging BunnyCDN pull zone ${zone_id}" + +status=$( + curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \ + -H "AccessKey: ${api_key}" \ + -H "Content-Length: 0" \ + "https://api.bunny.net/pullzone/${zone_id}/purgeCache" +) + +if [[ "${status}" != "200" && "${status}" != "204" ]]; then + echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 + cat /tmp/purge_zone_response.txt >&2 + exit 1 +fi + +echo "Purged pull zone ${zone_id} (HTTP ${status})" From caf3276fb968c732f0a80792435e63973bf80c45 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Tue, 11 Aug 2026 00:31:56 +0000 Subject: [PATCH 108/125] chore(deps): update dependency nbafrank/uvr to v0.4.6 --- local/uvr-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local/uvr-install.sh b/local/uvr-install.sh index c5d37d3..1e25e47 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -30,7 +30,7 @@ set -eu # renovate: datasource=github-releases depName=nbafrank/uvr -UVR_PIN="v0.4.5" +UVR_PIN="v0.4.6" if [ "$#" -eq 0 ]; then echo "usage: $0 ..." >&2 From 4b7dc28cc83784fd350731b87f0d515df7c21299 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 12 Aug 2026 08:30:29 +0000 Subject: [PATCH 109/125] feat(rebuild): shard the weekly rebuild and make each shard resumable (#163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `weekly-rebuild-missing` runs one job per `-` and walks that slot's list serially in a single `R -q -e` argument. That was cheap while every source fallback was skipped as "already built". Since bincraft #105/#106/#107 and #159 the gate works, and the lists are large: 8 917 source-served records on `amd64/alpine324`, 15 023 on `amd64/resolute`. Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) ran for two days, reached `[8692/23885] cholera`, and was killed there. Two failures follow from that shape: - **No parallelism.** The work is embarrassingly parallel across packages; one job does all of it. - **No resumability and no clean stopping point.** The loop ends only by exhausting the list, so the only way to stop it is a kill. A restart re-walks from the first entry, paying a CRAN version resolution and an S3 `HEAD` per package before reaching new work. And a kill matches neither `success` nor `failure`, so the `Purge CDN cache` step never ran: the ~4 600 binaries 10910 did publish stayed hidden behind stale edge copies. ## What this changes **Three shards per slot.** Each of the 18 `OS`/`ARCH` rows gains `SPLIT_INTO`/`SPLIT_INDEX`, mirroring `build-all-versions.yaml`. Cron and manual routing are unchanged: both filters already match on `${OS}-${ARCH}`, so they now match all three shards of a slot. **`local/rebuild-missing.R`** replaces the ~1 500-character inline one-liner. The slice is interleaved rather than contiguous, because the list is alphabetical and cost clusters by name (`Rcpp*`, `Bioc*`, `rstan*`). **Resume by re-deriving state from the bucket.** One `s3_dir_info()` listing gives ETags for the slot; a package is outstanding iff its object's ETag equals CRAN's published `MD5sum`, i.e. it is still byte-identical to CRAN's source. That is `check_s3_root_package()` evaluated in bulk. No progress file, no volume, no DB cursor, and correct when a sibling shard or a `process-updates` run completes something concurrently. It reads ETags rather than the index's `Built` field the way `packages-to-build.R` does, because the index is no longer rewritten until the dependent pipeline runs and so cannot reflect the current run's progress. Unknown always means "already a binary", never "rebuild it": a multipart ETag, an unreadable CRAN index or an empty listing can never mass-schedule work. **A 20 h wall-clock budget** per shard. It exits 0, so the re-index and purge always fire and the remainder is picked up next run with no bookkeeping. **`.crow/weekly-rebuild-reindex.yaml`** takes over re-indexing and the purge, with `depends_on: [weekly-rebuild-missing]` and `runs_on: [success, failure]`. Three shards writing one slot's `PACKAGES` concurrently would race: `update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work. ## Verification `crow lint .crow/` passes on all 11 pipelines. `prek run` passes. 19 assertions in `local/tests/test-rebuild-missing.R`, 0 failures, covering the partition (disjoint, covering, deterministic, short lists, out-of-range index) and the outstanding filter (source ETag kept, binary ETag dropped, absent object kept, multipart and missing-from-CRAN treated as built). One of those tests caught a real bug before it shipped: an empty ETag table indexed to zero length rather than to `NA`, which recycled the result away and reported "nothing to build" — the dangerous direction. Fixed with an explicit `lookup()`. The filter run against the live `amd64/alpine324` index, using its `MD5sum` column as the ETag (established to match the objects): ``` index packages: 24343 outstanding (filter): 8950 no Built stamp: 8917 filter vs no-Built agreement: 8917 of 8917 outstanding but stamped Built: 33 (version drift vs CRAN) shard sizes: 2984/2983/2983 (sum 8950, unique 8950) ``` It reproduces the source-served set exactly. The extra 33 are packages whose slot version differs from CRAN's current one, so no object exists at the CRAN version key: correctly outstanding. ## Notes for review - The 20 h budget is a chosen default, exposed as `REBUILD_BUDGET_HOURS` in the pipeline. - `depends_on` is file-level, not row-level, so on a full cron run no slot is re-indexed until the slowest of all 54 jobs finishes. The budget bounds that at roughly a day. - An explicit cancel still skips the re-index. Recovery is to trigger `weekly-rebuild-reindex` on its own. - The purge runs on every re-index row rather than one designated slot: a cron fires only its own slot's row, so gating on a named slot would leave every other slot unpurged. - Out of scope: `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row, which is precisely the source-fallback set. Design: `specs/2026-08-12-shard-weekly-rebuild-design.md` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/163 --- .crow/weekly-rebuild-missing.yaml | 306 +++++++++++++++--- .crow/weekly-rebuild-reindex.yaml | 193 +++++++++++ local/rebuild-missing-helpers.R | 87 +++++ local/rebuild-missing.R | 194 +++++++++++ local/tests/test-rebuild-missing.R | 94 ++++++ .../2026-08-12-shard-weekly-rebuild-design.md | 167 ++++++++++ 6 files changed, 1005 insertions(+), 36 deletions(-) create mode 100644 .crow/weekly-rebuild-reindex.yaml create mode 100644 local/rebuild-missing-helpers.R create mode 100644 local/rebuild-missing.R create mode 100644 local/tests/test-rebuild-missing.R create mode 100644 specs/2026-08-12-shard-weekly-rebuild-design.md diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index c86fe5a..5afb3c3 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -1,12 +1,21 @@ # Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). -# One matrix row per OS/arch replaces the former per-platform files. +# Three matrix rows per OS/arch, one per shard of that slot's rebuild list. # Routing is preserved 1:1: # - cron: each existing `weekly-rebuild-missing--` cron fires only -# its matching matrix row (via the per-row `cron:` name filter). +# its matching matrix rows (via the per-row `cron:` name filter), +# which is now all three shards of that slot. # - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the # previous bare manual trigger that ran every os/arch); pick a # single - to run just one. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). +# +# The shard picks up its own slice and re-derives what is still outstanding +# from the bucket, so a restart resumes rather than replaying; see +# local/rebuild-missing.R. +# +# Re-indexing and the CDN purge deliberately do NOT live here. Three shards +# writing one slot's PACKAGES concurrently would race, so they moved to +# .crow/weekly-rebuild-reindex.yaml, which depends on this pipeline. variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed @@ -53,74 +62,326 @@ matrix: ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-8 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-9 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-9 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-10 ARCH: amd64 R_VERSION: 4.5.3 IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-10 ARCH: arm64 R_VERSION: 4.5.3 IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2204 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2204 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2404 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2404 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2604 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2604 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 steps: - name: 'Rebuild missing binaries' @@ -153,6 +414,12 @@ steps: PLATFORM: ${OS} ARCH: ${ARCH} NCPUS: 2 + SPLIT_INTO: ${SPLIT_INTO} + SPLIT_INDEX: ${SPLIT_INDEX} + # Wall clock after which the shard stops cleanly instead of having to be + # killed. A kill matches neither `success` nor `failure`, so it would skip + # the dependent re-index and leave rebuilt binaries behind a stale edge. + REBUILD_BUDGET_HOURS: 20 commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -162,18 +429,7 @@ steps: - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - # A rebuild replaces objects in place, so the slot's index still advertises - # the old MD5 and, for anything that had been served from source, no Built - # stamp. Re-index here rather than waiting for the next process-updates - # run, or the rebuilt binaries stay invisible to clients until then. - # The codename is detected from the image's /etc/os-release. - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 backend_options: docker: resources: @@ -183,25 +439,3 @@ steps: limits: memory: 18Gi cpu: 3000m - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - # All hostnames on the zone share this id, so one purge covers - # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. - BUNNY_PULLZONE: '3857050' - commands: - - apk add --no-cache -q bash curl git - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" - # A rebuild that died part-way still replaced objects, and those are exactly - # the ones a stale edge would keep hiding, so purge either way. - when: - - status: [success, failure] diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml new file mode 100644 index 0000000..224596b --- /dev/null +++ b/.crow/weekly-rebuild-reindex.yaml @@ -0,0 +1,193 @@ +# Re-index and purge after weekly-rebuild-missing. +# +# weekly-rebuild-missing runs three shards per slot. Each of them replaces +# objects in place, so the slot's index still advertises the old MD5 and, for +# anything that had been served from source, no Built stamp. Re-indexing from +# inside a shard would mean three concurrent `upload_package_index()` calls on +# one prefix: `cranlike::update_PACKAGES()` lists the live bucket, so an early +# lister that uploads last publishes an index missing its siblings' work. +# +# So it happens exactly once per slot, here, after every shard has finished. +# `runs_on: [success, failure]` keeps that true when a shard fails; only an +# explicit cancel skips it, and this pipeline can then be triggered on its own. + +variables: + # Mirrors the gate on weekly-rebuild-missing so a manual run re-indexes + # exactly the slots it rebuilt. A manual pipeline creation instantiates every + # file in .crow/, so the default must match no matrix row. + weekly_rebuild_missing: + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-rebuild-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' + +depends_on: + - weekly-rebuild-missing + +runs_on: [success, failure] + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + PLATFORM: ${OS} + ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + # All hostnames on the zone share this id, so one purge covers + # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. + BUNNY_PULLZONE: '3857050' + commands: + - apk add --no-cache -q bash curl git + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" + # Runs on every row rather than on one designated slot: a cron fires only + # its own slot's row, so gating on a named slot would leave every other + # slot unpurged. A manual "all" run therefore purges the zone 18 times, + # which is a cheap API call and rare. + # + # Run it even when the re-index above failed: the objects were still + # replaced, and a stale edge is exactly what keeps them hidden. + when: + - status: [success, failure] diff --git a/local/rebuild-missing-helpers.R b/local/rebuild-missing-helpers.R new file mode 100644 index 0000000..16a588e --- /dev/null +++ b/local/rebuild-missing-helpers.R @@ -0,0 +1,87 @@ +# Pure helpers for local/rebuild-missing.R, kept separate so local/tests can +# source them without executing a rebuild. + +# Interleaved slice of the rebuild list. +# +# The list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, +# rstan*), so contiguous thirds would be badly unbalanced. Interleaving also +# makes each shard's progress counter representative of the slot as a whole. +shard_slice <- function(pkgs, split_into, split_index) { + split_into <- as.integer(split_into) + split_index <- as.integer(split_index) + if (is.na(split_into) || is.na(split_index)) { + stop("shard_slice(): split_into and split_index must be integers") + } + if (split_into < 1L || split_index < 1L || split_index > split_into) { + stop(sprintf( + "shard_slice(): need 1 <= split_index <= split_into, got %s of %s", + split_index, + split_into + )) + } + # seq() errors on a descending range, which is what an empty list or a shard + # index past the end would produce. + if (length(pkgs) < split_index) { + return(pkgs[0L]) + } + pkgs[seq.int(split_index, length(pkgs), by = split_into)] +} + +# Packages that still need building, decided from the bucket rather than from +# remembered progress. +# +# This is bincraft's `check_s3_root_package()` evaluated in bulk: an object +# whose ETag equals CRAN's published MD5sum is byte-identical to CRAN's source, +# so the build that was supposed to replace it has not happened yet. +# +# `etag_by_file` named by `_.tar.gz`, values are unquoted ETags +# `cran_version` named by package +# `cran_md5` named by `_` +# +# Unknown always means "already a binary", never "rebuild it", so an unreadable +# CRAN index or a multipart ETag can never mass-schedule work. +outstanding_packages <- function(pkgs, etag_by_file, cran_version, cran_md5) { + if (length(pkgs) == 0L) { + return(pkgs) + } + + # An empty table indexes to zero length rather than to NA, which would + # recycle the whole result away and silently report "nothing to build". + lookup <- function(table, key) { + if (length(table) == 0L) { + return(rep(NA_character_, length(key))) + } + unname(as.character(table[key])) + } + + version <- lookup(cran_version, pkgs) + file <- sprintf("%s_%s.tar.gz", pkgs, version) + etag <- lookup(etag_by_file, file) + md5 <- lookup(cran_md5, paste(pkgs, version, sep = "_")) + + # No CRAN version means the package cannot be resolved to a tarball at all; + # leave it in and let bincraft report why. + unresolved <- is.na(version) + # No object at the key: never built, so it is outstanding by definition. + absent <- !unresolved & is.na(etag) + # A multipart upload carries a compound ETag rather than an MD5. + unknown <- !is.na(etag) & grepl("-", etag, fixed = TRUE) + + is_source <- !unresolved & + !is.na(etag) & + !unknown & + !is.na(md5) & + etag == md5 + + pkgs[unresolved | absent | is_source] +} + +parse_rebuild_args <- function(args) { + pos <- args[!startsWith(args, "--")] + budget <- as.numeric(pos[3L]) + list( + split_into = as.integer(pos[1L]), + split_index = as.integer(pos[2L]), + budget_hours = if (is.na(budget)) 20 else budget + ) +} diff --git a/local/rebuild-missing.R b/local/rebuild-missing.R new file mode 100644 index 0000000..369024e --- /dev/null +++ b/local/rebuild-missing.R @@ -0,0 +1,194 @@ +### Rebuild one shard of a slot's missing-binary list. +# +# Usage: Rscript local/rebuild-missing.R [budget_hours] +# +# The list itself comes from local/fetch-rebuild-packages-from-issue.R, which +# writes $REBUILD_PKG_LIST (default /tmp/rebuild_pkgs.txt). +# +# Two properties matter here and are the reason this is a script rather than an +# `R -q -e` argument in the pipeline: +# +# * it is restartable. The outstanding set is re-derived from the bucket on +# every start, so a shard that died resumes where it stopped without any +# progress file, and without replaying thousands of per-package HEADs. +# * it terminates. A wall-clock budget stops the loop cleanly instead of the +# run having to be killed, which is what previously skipped the re-index and +# CDN purge and left rebuilt binaries hidden behind stale edge copies. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +source(file.path("local", "rebuild-missing-helpers.R")) + +args <- parse_rebuild_args(commandArgs(trailingOnly = TRUE)) +if (is.na(args$split_into) || is.na(args$split_index)) { + stop("usage: rebuild-missing.R [budget_hours]") +} + +list_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") +pkgs <- if (file.exists(list_file)) readLines(list_file) else character(0) +pkgs <- pkgs[nzchar(pkgs)] +if (length(pkgs) == 0L) { + cat("Nothing to rebuild\n") + q("no") +} + +excluded <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] +pkgs <- setdiff(pkgs, excluded) + +mine <- shard_slice(pkgs, args$split_into, args$split_index) +cat(sprintf( + "Shard %s/%s: %s of %s listed packages\n", + args$split_index, + args$split_into, + length(mine), + length(pkgs) +)) + +### Resume: ask the bucket what is still outstanding + +codename <- bincraft::set_codename(NULL) +local_machine <- Sys.info()[["machine"]] +arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" +slot_dir <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", + arch, + codename +) + +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) + +# One paginated listing instead of a HEAD per package. Not recursed: the +# rebuild passes no `is_r_minor_sensitive`, so it only ever targets the flat +# path, and the resume filter matches that scope deliberately. +info <- tryCatch(s3fs::s3_dir_info(slot_dir), error = function(e) NULL) +etag_by_file <- if (is.null(info) || nrow(info) == 0L) { + cat(sprintf( + "WARNING: could not list %s; building the whole shard\n", + slot_dir + )) + stats::setNames(character(), character()) +} else { + stats::setNames( + gsub('^"|"$', "", as.character(info$etag)), + basename(as.character(info$uri)) + ) +} + +cran <- tryCatch( + { + con <- gzcon(url( + "https://cloud.r-project.org/src/contrib/PACKAGES.gz", + open = "rb" + )) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "MD5sum")) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read CRAN's index (%s)\n", + conditionMessage(e) + )) + NULL + } +) +cran_version <- stats::setNames(character(), character()) +cran_md5 <- stats::setNames(character(), character()) +if (!is.null(cran)) { + cran_version <- stats::setNames( + as.character(cran[, "Version"]), + as.character(cran[, "Package"]) + ) + keep <- !is.na(cran[, "MD5sum"]) + cran_md5 <- stats::setNames( + as.character(cran[keep, "MD5sum"]), + paste(cran[keep, "Package"], cran[keep, "Version"], sep = "_") + ) +} + +before <- length(mine) +mine <- outstanding_packages(mine, etag_by_file, cran_version, cran_md5) +cat(sprintf( + "Resume: %s of %s already carry a binary; %s outstanding\n", + before - length(mine), + before, + length(mine) +)) + +if (length(mine) == 0L) { + cat("Nothing outstanding for this shard\n") + q("no") +} + +### Build + +options( + crayon.enabled = TRUE, + Ncpus = as.integer(Sys.getenv("NCPUS", "2")), + future.globals.onReference = NULL +) + +started <- Sys.time() +n <- length(mine) +completed <- 0L +for (i in seq_along(mine)) { + elapsed <- as.numeric(difftime(Sys.time(), started, units = "hours")) + if (elapsed > args$budget_hours) { + cat(sprintf( + "Budget of %sh reached after %d/%d packages; stopping cleanly. The next run resumes from the bucket.\n", + args$budget_hours, + completed, + n + )) + break + } + + x <- mine[i] + cat(sprintf("[%d/%d] %s\n", i, n, x)) + tryCatch( + bincraft::build_binary_package( + x, + tag_limit = 1L, + patches = "local/patches", + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + upload = TRUE, + store_build_metadata = TRUE + ), + error = function(e) { + cat(sprintf("ERROR building %s - %s\n", x, conditionMessage(e))) + } + ) + completed <- completed + 1L +} + +cat(sprintf( + "Shard %s/%s finished: %d/%d packages processed in %.1fh\n", + args$split_index, + args$split_into, + completed, + n, + as.numeric(difftime(Sys.time(), started, units = "hours")) +)) diff --git a/local/tests/test-rebuild-missing.R b/local/tests/test-rebuild-missing.R new file mode 100644 index 0000000..a5e70b9 --- /dev/null +++ b/local/tests/test-rebuild-missing.R @@ -0,0 +1,94 @@ +source(file.path("..", "rebuild-missing-helpers.R")) + +test_that("shard_slice partitions the list without gaps or overlap", { + pkgs <- letters[1:10] + parts <- lapply(1:3, function(i) shard_slice(pkgs, 3, i)) + + expect_identical(parts[[1]], c("a", "d", "g", "j")) + expect_identical(parts[[2]], c("b", "e", "h")) + expect_identical(parts[[3]], c("c", "f", "i")) + + expect_identical(sort(unlist(parts)), sort(pkgs)) + expect_identical(anyDuplicated(unlist(parts)), 0L) +}) + +test_that("shard_slice is deterministic and survives short lists", { + expect_identical( + shard_slice(letters[1:10], 3, 2), + shard_slice(letters[1:10], 3, 2) + ) + expect_identical(shard_slice(character(0), 3, 1), character(0)) + # more shards than packages: the tail shards get nothing rather than erroring + expect_identical(shard_slice(c("a"), 3, 1), "a") + expect_identical(shard_slice(c("a"), 3, 2), character(0)) +}) + +test_that("shard_slice rejects an out-of-range index", { + expect_error(shard_slice(letters, 3, 4), "split_index") + expect_error(shard_slice(letters, 3, 0), "split_index") +}) + +test_that("outstanding_packages keeps source fallbacks and drops real binaries", { + cran_version <- c(httr = "1.4.8", R6 = "2.6.1", curl = "7.1.0") + cran_md5 <- c( + httr_1.4.8 = "8756015b94a9cff6f410ca4de8557f12", + R6_2.6.1 = "f01b1787f12797c29194d63c9afd5d70", + curl_7.1.0 = "8af2ccbf5d85dc18866f45f1f26f348d" + ) + etag <- c( + # byte-identical to CRAN: the build never happened + "httr_1.4.8.tar.gz" = "8756015b94a9cff6f410ca4de8557f12", + # a real binary was published + "R6_2.6.1.tar.gz" = "9d6087ee9adda3f0a3b8067cfc652c05" + # curl has no object at all + ) + + out <- outstanding_packages( + c("httr", "R6", "curl"), + etag, + cran_version, + cran_md5 + ) + expect_identical(out, c("httr", "curl")) +}) + +test_that("outstanding_packages treats unknowns as already built", { + cran_version <- c(a = "1.0", b = "1.0") + cran_md5 <- c(a_1.0 = "aaaa") + + # a multipart ETag carries no MD5, and `b` is missing from CRAN's index: + # neither may schedule a rebuild + etag <- c("a_1.0.tar.gz" = "abc-3", "b_1.0.tar.gz" = "bbbb") + + expect_identical( + outstanding_packages(c("a", "b"), etag, cran_version, cran_md5), + character(0) + ) +}) + +test_that("outstanding_packages keeps a package CRAN has no version for", { + out <- outstanding_packages( + "ghost", + c(), + c(other = "1.0"), + c(other_1.0 = "aaaa") + ) + expect_identical(out, "ghost") +}) + +test_that("outstanding_packages handles an empty list", { + expect_identical( + outstanding_packages(character(0), c(), c(), c()), + character(0) + ) +}) + +test_that("parse_rebuild_args defaults the budget", { + a <- parse_rebuild_args(c("3", "2")) + expect_identical(a$split_into, 3L) + expect_identical(a$split_index, 2L) + expect_identical(a$budget_hours, 20) + + b <- parse_rebuild_args(c("3", "2", "1.5")) + expect_identical(b$budget_hours, 1.5) +}) diff --git a/specs/2026-08-12-shard-weekly-rebuild-design.md b/specs/2026-08-12-shard-weekly-rebuild-design.md new file mode 100644 index 0000000..f4ccbaa --- /dev/null +++ b/specs/2026-08-12-shard-weekly-rebuild-design.md @@ -0,0 +1,167 @@ +# Design: Sharding and resuming the weekly rebuild + +Date: 2026-08-12 +Status: Approved (pending spec review) + +## Problem + +`weekly-rebuild-missing` runs one job per `-` and walks that slot's rebuild list serially in a single `R -q -e` invocation (`.crow/weekly-rebuild-missing.yaml:165`). +Until 2026-08-09 that was cheap, because every source fallback was skipped as "already built" and the list was effectively empty. +Since bincraft #105/#106/#107 and build-cran-binaries #159 the gate works, and the lists are now large. + +Share of records whose object is byte-identical to CRAN's source, measured against `cran.r-project.org` MD5s on 2026-08-12: + +| slot | records | source-served | share | +| ------------------ | ------: | ------------: | -----------: | +| `amd64/resolute` | 24 212 | 15 023 | 62.1% | +| `arm64/resolute` | 24 291 | 13 670 | 56.3% | +| `arm64/alpine324` | 24 328 | 9 514 | 39.2% | +| `amd64/alpine324` | 24 343 | 8 917 | 36.7% | +| `arm64/rhel10` | 24 695 | 5 384 | 21.9% | +| `amd64/rhel10` | 24 881 | 4 712 | 19.2% | +| 12 remaining slots | ~24 700 | 850 to 2 130 | 3.5% to 8.7% | + +A single serial job cannot absorb that. +Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) started on 2026-08-09, ran for roughly two days, reached `[8692/23885] cholera`, and was killed there. + +Two distinct failures follow from that shape. + +**No parallelism.** The work is embarrassingly parallel across packages, but one job does all of it. + +**No resumability, and no clean stopping point.** The loop has no terminating condition other than exhausting the list, so the only way to stop it is a kill. +A restarted run re-reads the same list and walks it from the first entry. +It skips completed packages via `check_s3_root_package()`, but that costs a CRAN version resolution and an S3 `HEAD` per package, thousands of times, before it reaches new work. +Worse, a kill is not a pipeline failure: the `Purge CDN cache` step is guarded by `when: status: [success, failure]` (`.crow/weekly-rebuild-missing.yaml:206-207`), and on 10910 it produced no output at all. +So the ~4 600 binaries that run did publish stayed hidden behind stale edge copies. + +## Goal + +Turn each slot's rebuild into bounded, parallel, restartable units, without introducing state that can disagree with the bucket. + +## Design + +### 1. Shard the matrix three ways + +Each of the 18 `OS`/`ARCH` rows in `.crow/weekly-rebuild-missing.yaml` gains `SPLIT_INTO: 3` and `SPLIT_INDEX: 1|2|3`, giving 54 rows. +This mirrors `.crow/build-all-versions.yaml:57-98`, which already shards its matrix four ways per arch. + +Routing needs no change. +The cron filter `cron: weekly-rebuild-missing-${OS}-${ARCH}` and the manual `evaluate: weekly_rebuild_missing == "${OS}-${ARCH}"` both match all three shards of a slot. +Placement stays on the `rpkgs-${ARCH}` group label, so shards queue against available capacity rather than oversubscribing it. + +### 2. Extract the loop into `local/rebuild-missing.R` + +The build is currently a single ~1 500-character `R -q -e` argument. +Shard arithmetic and resume logic do not belong in a YAML string, and none of it is testable there. +The loop moves to `local/rebuild-missing.R`, invoked as `Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX`, mirroring `local/build-all.R`. +Its body is unchanged in substance: read `/tmp/rebuild_pkgs.txt`, subtract `local/excluded-packages.json`, loop with `tryCatch` around `bincraft::build_binary_package()`. + +The slice is **interleaved**, not contiguous: + +```r +# the list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, +# rstan*), so contiguous thirds would be badly unbalanced +mine <- pkgs[seq(split_index, length(pkgs), by = split_into)] +``` + +`local/build-all.R:64` uses contiguous chunks via `cut()`. +That is fine there because its list is every CRAN package and version, so the chunks average out. +Here the list is a filtered backlog in which expensive families sit adjacent, so interleaving is the better default. +Interleaving also makes each shard's `[i/n]` progress representative of the slot as a whole. + +### 3. Resume by re-deriving state from the bucket + +Before the loop, the shard performs one `s3fs::s3_dir_info()` on `devxy-rpkgs-binaries///latest/src/contrib` and reads the `etag` column. +It fetches CRAN's `PACKAGES` once for the latest version and published `MD5sum` of every package. +A package is still outstanding if and only if the object at `_.tar.gz` has an ETag equal to CRAN's `MD5sum` for that version, which is the definition `check_s3_root_package()` already applies one package at a time. + +```r +# one paginated listing instead of ~2900 sequential HEAD requests per shard +info <- s3fs::s3_dir_info(slot_dir) +etag <- setNames(gsub('^"|"$', "", info$etag), basename(info$uri)) + +key <- sprintf("%s_%s.tar.gz", mine, cran_version[mine]) +# keep a package when no object exists yet, or when the object is still +# byte-identical to CRAN's source; drop it once a real binary is published +mine <- mine[is.na(etag[key]) | etag[key] == cran_md5[key]] +``` + +This is the whole resume mechanism. +There is no progress file, no volume, and no database cursor. +A restarted shard recomputes ground truth and continues where it stopped, and it is correct even when a sibling shard, a `process-updates` cron, or a manual `just rebuild` completed something in the meantime. + +Three properties make this the right source of truth: + +- **It is what the build itself checks.** Any other store can disagree with the bucket; this one cannot. +- **It is agent-independent.** `.crow/weekly-rebuild-missing.yaml` mounts no `volumes:`, unlike `.crow/build-all-versions.yaml:132-133`, so `/mnt/cache` is per-job and cannot carry progress anyway. +- **It costs one listing.** `cranlike`'s `s3` fork already does exactly this call against this bucket at ~24 000 objects, so the approach is proven at the required scale. + +It must read ETags rather than the slot index's `Built` field, which is how `local/packages-to-build.R:104-130` answers the same question. +Under this design the index is not rewritten until the dependent re-index pipeline runs (section 5), so mid-run it cannot reflect the current run's progress. + +Packages that genuinely fail to build re-publish their CRAN source, so they stay outstanding and would be retried on every restart. +That is already handled upstream: `bincraft::filter_packages_with_errors()` (`R/build_binaries.R:1018`, `:1143`) drops anything with `error_occurred = TRUE`, and `store_build_metadata = TRUE` is passed on every call. +No additional poison-pill filter is needed here. + +Only the flat `src/contrib` path is considered. +The rebuild call passes no `is_r_minor_sensitive`, so it defaults to `FALSE` and only ever targets the flat path; the resume filter matches that scope deliberately. + +### 4. Give each shard a wall-clock budget + +`local/rebuild-missing.R` takes a budget, defaulting to 20 hours, and breaks out of the loop once it is exceeded: + +```r +# exit cleanly rather than being killed, so the dependent re-index still runs +if (difftime(Sys.time(), started, units = "hours") > budget_hours) { + cat(sprintf("Budget of %sh reached after %d/%d packages; stopping cleanly\n", budget_hours, i, n)) + break +} +``` + +It exits 0 and reports how much of the slice it covered. +Every run then has a terminating condition, the re-index and purge always fire, and the remainder is picked up by the next run with no bookkeeping, because section 3 recomputes the outstanding set from scratch. + +### 5. Move the re-index and purge into `.crow/weekly-rebuild-reindex.yaml` + +Three shards per slot means three concurrent `upload_package_index()` calls on the same S3 prefix. +`cranlike::update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work. +The re-index steps (`.crow/weekly-rebuild-missing.yaml:171-176`) and the purge step (`:187-207`) therefore leave that file entirely. + +The new file carries: + +```yaml +depends_on: + - weekly-rebuild-missing +runs_on: [success, failure] +``` + +`runs_on: [success, failure]` validates as a workflow-level key under `crow lint`, so a failing shard no longer withholds the re-index. +The file uses the same 18-row matrix and the same `when:` gating as `weekly-rebuild-missing`, so it only re-indexes slots that actually ran. +Each row re-indexes the flat slot and every per-minor slot. +`scripts/purge_cdn_zone.sh` runs once on a single row, because all hostnames share pull zone `3857050` and 18 identical zone purges would be waste. + +## Failure behaviour + +| case | today | after | +| -------------------------- | ----------------------------------- | ----------------------------------------------------- | +| one package errors | `tryCatch` logs, loop continues | unchanged | +| a shard fails outright | purge runs, re-index does not | re-index and purge run via `runs_on` | +| a shard exceeds its budget | cannot happen, runs until killed | exits 0, re-index and purge run | +| a shard is killed | nothing runs | still nothing; trigger the re-index pipeline alone | +| a shard restarts | re-walks the list, HEAD per package | one listing, resumes at the first outstanding package | + +The known cost of `depends_on` being file-level rather than row-level: on the weekly cron no slot is re-indexed until the slowest of all 54 jobs finishes. +The 20-hour budget bounds that at roughly one day. + +## Out of scope + +- `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row for the platform and arch, which is precisely the source-fallback set. That is a separate change. +- Bunny Perma-Cache eviction. `scripts/purge_cdn_zone.sh` purges the regular edge cache only; see the note in `CLAUDE.md` and issue history. +- The audit that produces the rebuild list is unchanged. + +## Verification + +- `crow lint .crow/` passes for both pipeline files. +- `local/rebuild-missing.R` gets unit coverage in `local/tests/` for the two pure pieces: the interleaved slice (disjoint, covering, deterministic) and the outstanding-set filter (source-served ETag kept, binary ETag dropped, absent object kept). +- A single-slot manual run of `alpine-324-amd64` shard 1 confirms the listing shortcut against the live bucket, and that the reported outstanding count is close to the 8 917 measured above divided by three. +- Restarting that shard mid-run confirms it resumes rather than replaying, by comparing the outstanding count it reports on the second start. From aa4c95457f0ce7f79adea6705d54b098a10c50dd Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 13 Aug 2026 13:38:28 +0000 Subject: [PATCH 110/125] fix(rebuild): harden split workflow setup (#164) ## Motivation Weekly rebuild shards can all hit a transient CRAN DNS/index outage at once, and the dependent CDN purge always fails because it tries to clone over the checkout preserved from the re-index step. ## Changes - Retry `uvr add` resolution up to four times with bounded backoff. - Reuse the existing Crow workspace checkout in the CDN purge step. - Remove the purge step's unused Git package and repository token. ## Validation - `crow lint .crow/` - `shellcheck local/uvr-install.sh scripts/purge_cdn_zone.sh` - `git diff --check` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/164 --- .crow/weekly-rebuild-reindex.yaml | 6 ++---- local/uvr-install.sh | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 224596b..a67875d 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -173,14 +173,12 @@ steps: OTEL_R_METRICS_EXPORTER: none BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN # All hostnames on the zone share this id, so one purge covers # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. BUNNY_PULLZONE: '3857050' commands: - - apk add --no-cache -q bash curl git - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - apk add --no-cache -q bash curl + # Crow carries the checkout from the re-index step into this step. - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" # Runs on every row rather than on one designated slot: a cron fires only # its own slot's row, so gating on a named slot would leave every other diff --git a/local/uvr-install.sh b/local/uvr-install.sh index 1e25e47..3966e85 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -84,9 +84,20 @@ trap 'rm -rf "$project_dir"' EXIT cd "$project_dir" "$uvr_bin" init --here --r-version "$r_full" -# --no-install: resolve and lock only. The install happens in the sync below, -# which is the only command that honours --library. -"$uvr_bin" add --no-install "$@" +# --no-install resolves and locks only; retry because concurrent shards can +# expose short-lived DNS or CRAN-index failures and uvr rolls the manifest back +# cleanly after an unsuccessful resolution. +add_attempt=1 +while ! "$uvr_bin" add --no-install "$@"; do + if [ "$add_attempt" -ge 4 ]; then + echo "error: uvr add failed after ${add_attempt} attempts" >&2 + exit 1 + fi + add_delay=$((add_attempt * 10)) + echo "warning: uvr add attempt ${add_attempt} failed; retrying in ${add_delay}s" >&2 + sleep "$add_delay" + add_attempt=$((add_attempt + 1)) +done # TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs # `apt-get install` for every resolved system dependency without refreshing the From a1c1f5e78f2e22db0497fd749067661f063f35ff Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 13 Aug 2026 14:08:10 +0000 Subject: [PATCH 111/125] fix(cdn): align repository routing across pull zones (#165) ## Motivation `cran.rpkgs.com` and `cran.allianceswisspass.devxy.io` serve the same B2 repository through separate Bunny pull zones, but only the first zone was managed and purged after weekly reindexing. This allowed the Alliance endpoint to retain stale repository metadata and left locked `renv` restores unable to retrieve versions whose binary archive object was absent. ## Changes - Adopt the Alliance SwissPass pull zone `3265648` into OpenTofu and configure it with the shared B2 origin and middleware script. - Purge both Bunny pull zones after the weekly rebuild reindex. - Preserve the requested public hostname in middleware redirects. - Redirect missing archived binaries to the corresponding CRAN source package, checking whether the version is archived or still current. - Cover the existing archived-binary passthrough behavior in the edge routing matrix. ## Verification - `prek run -a` - `just edge-test` - `crow lint .crow/` - `tofu validate` - `bash -n scripts/purge_cdn_zone.sh` ## Deployment Run `tofu apply` to adopt pull zone `3265648`, publish the middleware release, and align both pull zones. After the apply, rerun the Alliance SwissPass CI restore that requested `cli 3.6.5` and `AzureStor 3.7.1`. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/165 --- .crow/weekly-rebuild-reindex.yaml | 8 ++--- cdn.tf | 56 +++++++++++++++++++++++++++++-- edge/rpkgs-router.test.ts | 7 ++++ edge/rpkgs-router.ts | 35 +++++++++++++++---- scripts/purge_cdn_zone.sh | 39 +++++++++++---------- 5 files changed, 115 insertions(+), 30 deletions(-) diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index a67875d..bb00332 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -173,13 +173,13 @@ steps: OTEL_R_METRICS_EXPORTER: none BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY - # All hostnames on the zone share this id, so one purge covers - # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. - BUNNY_PULLZONE: '3857050' + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + BUNNY_PULLZONES: '3857050 3265648' commands: - apk add --no-cache -q bash curl # Crow carries the checkout from the re-index step into this step. - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES # Runs on every row rather than on one designated slot: a cron fires only # its own slot's row, so gating on a named slot would leave every other # slot unpurged. A manual "all" run therefore purges the zone 18 times, diff --git a/cdn.tf b/cdn.tf index 5e8899d..01191a5 100644 --- a/cdn.tf +++ b/cdn.tf @@ -32,7 +32,7 @@ # cache_stale = ["offline", "updating"] # use_background_update = true - # block_ips = var.cdn_block_ips +# block_ips = var.cdn_block_ips # # 50 TB # limit_bandwidth = 50000000000000 @@ -82,7 +82,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { cache_expiration_time = 31919000 websockets_enabled = false - errorpage_whitelabel = true + errorpage_whitelabel = true origin { type = "OriginUrl" @@ -147,6 +147,58 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } +# Alliance SwissPass historically used a separate, manually configured pull +# zone. Adopt it so both public repositories use the same B2 origin, middleware +# release and cache behavior. +import { + to = bunnynet_pullzone.cran_allianceswisspass + id = "3265648" +} + +resource "bunnynet_pullzone" "cran_allianceswisspass" { + name = "cran-allianceswisspass" + + cache_errors = false + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + + origin { + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router.id + } + + routing { + filters = [ + "scripting", + ] + } + + s3_auth_enabled = true + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" + + cache_enabled = true + request_coalescing_enabled = true + block_post_requests = true + cache_vary_headers = ["User-Agent"] + + limit_requests = 5000 + limit_connections = 1000 + + safehop_enabled = true + add_canonical_header = true + cache_stale = ["offline", "updating"] + block_ips = var.cdn_block_ips + + # 50 TB + limit_bandwidth = 50000000000000 + + block_root_path = true +} + # resource "bunnynet_storage_zone" "devxy-r-binaries" { # name = "devxy-r-binaries-storage" # region = "DE" diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 553185d..9493f8a 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -117,6 +117,13 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.status, 200); }); + await t.step('serves an archived binary when it exists', async () => { + const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`; + const res = await probe(path, UA_R45_MUSL); + assertEquals(res.status, 200); + assertEquals(res.location, null); + }); + await t.step('does not redirect a path already under a minor', async () => { const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); assertEquals(res.location, null); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index 9cd410b..9278d4a 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -27,6 +27,7 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; const CRAN_ORIGIN = 'https://cran.r-project.org'; +const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']); /** Slots ("/", comma separated) whose per-minor index is a union. */ const UNION_SLOTS = new Set( @@ -47,6 +48,10 @@ const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; +/** A binary archive URL whose upstream source counterpart CRAN can serve. */ +const ARCHIVE_TARBALL_REGEX = + /^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/; + const MACOS_BIN_REGEX = /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; @@ -85,6 +90,10 @@ function redirectTo(location: string, status = 302): Response { }); } +function publicCdnOrigin(url: URL): string { + return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; +} + function extractRMinor(userAgent: string): string | null { for (const regex of R_MINOR_REGEXES) { const match = userAgent.match(regex); @@ -181,15 +190,14 @@ BunnySDK.net.http const url = new URL(ctx.request.url); const path = normalizePathname(url.pathname); const userAgent = ctx.request.headers.get('User-Agent') || ''; + const publicOrigin = publicCdnOrigin(url); // macOS clients are served from CRAN's own binary tree. const srcContrib = path.match(SRC_CONTRIB_REGEX); if (srcContrib && /darwin/.test(userAgent)) { const mac = parseMacUserAgent(userAgent); if (mac) { - return Promise.resolve( - redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`), - ); + return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`)); } } @@ -213,7 +221,7 @@ BunnySDK.net.http if (target === path) { return Promise.resolve(ctx.request); } - return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`)); + return Promise.resolve(redirectTo(`${publicOrigin}${target}`)); } // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. @@ -224,12 +232,27 @@ BunnySDK.net.http } const rest = srcContrib ? srcContrib[1] : ''; - return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`)); + return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); } return Promise.resolve(ctx.request); }) - .onOriginResponse((ctx) => { + .onOriginResponse(async (ctx) => { + const path = normalizePathname(new URL(ctx.request.url).pathname); + const archive = path.match(ARCHIVE_TARBALL_REGEX); + + // Binary archives can be incomplete when an older build never succeeded. + // Preserve renv/remotes version restores by falling back to CRAN's source + // package only for an absent archived tarball. A requested version can be + // either archived upstream or still current, so probe the archive first. + // Other 404s remain visible. + if (ctx.response.status === 404 && archive) { + const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`; + const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' }); + const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`; + return redirectTo(sourceUrl); + } + ctx.response.headers.append('X-Via', 'MyMiddleware'); return Promise.resolve(ctx.response); }); diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 391a814..4853245 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -18,35 +18,38 @@ # objects were replaced. The cost is a cold cache for everything else, which is # why this is not used by the daily update path. # -# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io, -# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them. +# The public hostnames currently use separate pull zones, so callers must pass +# every zone that serves the repository. # # Usage: -# purge_cdn_zone.sh +# purge_cdn_zone.sh [...] # set -euo pipefail if (($# < 2)); then - echo "usage: $0 " >&2 + echo "usage: $0 [...]" >&2 exit 2 fi api_key="$1" -zone_id="$2" +shift -echo "Purging BunnyCDN pull zone ${zone_id}" +for zone_id in "$@"; do + echo "Purging BunnyCDN pull zone ${zone_id}" -status=$( - curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \ - -H "AccessKey: ${api_key}" \ - -H "Content-Length: 0" \ - "https://api.bunny.net/pullzone/${zone_id}/purgeCache" -) + response_file="/tmp/purge_zone_response_${zone_id}.txt" + status=$( + curl -sS -o "${response_file}" -w '%{http_code}' -X POST \ + -H "AccessKey: ${api_key}" \ + -H "Content-Length: 0" \ + "https://api.bunny.net/pullzone/${zone_id}/purgeCache" + ) -if [[ "${status}" != "200" && "${status}" != "204" ]]; then - echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 - cat /tmp/purge_zone_response.txt >&2 - exit 1 -fi + if [[ "${status}" != "200" && "${status}" != "204" ]]; then + echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 + cat "${response_file}" >&2 + exit 1 + fi -echo "Purged pull zone ${zone_id} (HTTP ${status})" + echo "Purged pull zone ${zone_id} (HTTP ${status})" +done From 9bded261eecba451d4d544fbde68a68cd1180873 Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 13 Aug 2026 14:13:04 +0000 Subject: [PATCH 112/125] fix(cdn): restore Alliance pull-zone hostname (#166) ## Motivation Applying #165 recreated the Alliance SwissPass pull zone without its custom hostname because the hostname association was not represented in OpenTofu. The recreated zone also received a new numeric ID, making the weekly purge configuration stale. ## Changes - Manage `cran.allianceswisspass.devxy.io` as a pull-zone hostname with TLS and forced HTTPS. - Resolve the Alliance pull-zone ID from its hostname before purging instead of persisting a replaceable numeric ID. - Install `jq` in the purge step for the Bunny API lookup. ## Verification - Targeted `prek` hooks pass. - `tofu validate` passes. - `crow lint .crow/` passes. - `just edge-test` passes all 14 routing steps. - `bash -n scripts/purge_cdn_zone.sh` passes. ## Deployment Run `tofu apply` to restore the Alliance hostname on the recreated pull zone. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/166 --- .crow/weekly-rebuild-reindex.yaml | 4 ++-- cdn.tf | 7 ++++++ scripts/purge_cdn_zone.sh | 40 +++++++++++++++++++++++++++---- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index bb00332..250b225 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -175,9 +175,9 @@ steps: from_secret: BUNNYNET_API_KEY # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 3265648' + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' commands: - - apk add --no-cache -q bash curl + - apk add --no-cache -q bash curl jq # Crow carries the checkout from the re-index step into this step. - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES # Runs on every row rather than on one designated slot: a cron fires only diff --git a/cdn.tf b/cdn.tf index 01191a5..96877b2 100644 --- a/cdn.tf +++ b/cdn.tf @@ -199,6 +199,13 @@ resource "bunnynet_pullzone" "cran_allianceswisspass" { block_root_path = true } +resource "bunnynet_pullzone_hostname" "cran_allianceswisspass" { + pullzone = bunnynet_pullzone.cran_allianceswisspass.id + name = "cran.allianceswisspass.devxy.io" + force_ssl = true + tls_enabled = true +} + # resource "bunnynet_storage_zone" "devxy-r-binaries" { # name = "devxy-r-binaries-storage" # region = "DE" diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 4853245..648dfc3 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -19,22 +19,54 @@ # why this is not used by the daily update path. # # The public hostnames currently use separate pull zones, so callers must pass -# every zone that serves the repository. +# every zone that serves the repository. A zone can be identified by its +# numeric ID or by one of its hostnames; hostname lookup avoids persisting IDs +# that change when a zone is recreated. # # Usage: -# purge_cdn_zone.sh [...] +# purge_cdn_zone.sh [...] # set -euo pipefail if (($# < 2)); then - echo "usage: $0 [...]" >&2 + echo "usage: $0 [...]" >&2 exit 2 fi api_key="$1" shift -for zone_id in "$@"; do +resolve_zone_id() { + local zone="$1" + local response_file + local zone_id + + if [[ "${zone}" =~ ^[0-9]+$ ]]; then + echo "${zone}" + return + fi + + response_file=$(mktemp) + curl -sS -o "${response_file}" \ + -H "AccessKey: ${api_key}" \ + "https://api.bunny.net/pullzone" + zone_id=$( + jq -r --arg hostname "${zone}" \ + '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \ + "${response_file}" + ) + rm -f "${response_file}" + + if [[ -z "${zone_id}" ]]; then + echo "Could not find BunnyCDN pull zone for hostname ${zone}" >&2 + exit 1 + fi + + echo "${zone_id}" +} + +for zone in "$@"; do + zone_id=$(resolve_zone_id "${zone}") echo "Purging BunnyCDN pull zone ${zone_id}" response_file="/tmp/purge_zone_response_${zone_id}.txt" From 706fd10d79206ed2784cd7f6bbac1a5e897e390b Mon Sep 17 00:00:00 2001 From: automation-bot Date: Fri, 14 Aug 2026 00:32:00 +0000 Subject: [PATCH 113/125] chore(deps): update terraform bunnynet to ~> 0.18 --- .terraform.lock.hcl | 72 ++++++++++++++++++++++----------------------- provider.tf | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index aca8ce1..a21cf43 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.17.0" - constraints = "~> 0.17" + version = "0.18.0" + constraints = "~> 0.18" hashes = [ - "h1:+qDt35lVSK7acw6a1xHuPYrqmZEcHSmtd+6n1TxNuYw=", - "h1:1dCu2l4DhPBjizVAH/WwAjT1Xbo52K4PMvHoD5zUhuU=", - "h1:Dvn46Auwuel4jqrqZXs2D7kdujNhs17LEmqhuY0k4/4=", - "h1:M5eDL3m2uSEr1XATJW0foHzKl8pFhCtgKuOM24bJRwU=", - "h1:PddaC7nM/gY4x9i3xy6TxOs9MAu2/6g58Xs/gv4DRV8=", - "h1:QVIKiZluI+NQAKu8NpFBl3Nvyx+d81vW9btEUdIQREc=", - "h1:S6TnzXHsRoGYvC1vJBkDiVEc0spceksY4n6x5WN5iYw=", - "h1:VcxZDWqCWMSjcUsC1K4sB6uYEoeoou+BC0ePoJXmf3A=", - "h1:W0y/agBVqls1cJlFGFYMu2VnqoPXFzxVHPIYe3OqfYQ=", - "h1:XmNd5fP9a0O77ve5BMQP2vARExgIa7rYl6KvyUYXPSs=", - "h1:e0EFKrWSQwaa/kGhnha4DXk4T68Av8QxP84mRSdWC9M=", - "h1:eM+/lUiU0pNSgQKoqKPgE3xJrJ0MHIpKG+yhaGB/P0M=", - "h1:fPWWA4T0/y7GX+tCGN23l1jODhZ3uCdR/MKgZDXYpAE=", - "h1:g+r2GVi4gVC4DuQg3PL70gW9BDskgWUzCBIMXTUq63A=", - "h1:gaZ8eALDtVHqykVDHav8004gHiMGaYR/3KwET0FUgao=", - "h1:kbqW25eaiv4N/N/z+sxLdJZ15yh5cgnRD/q6RclPMLc=", - "h1:rGjxue3mXRyQQqpywTXC4zK//JAtf0Cz7RP+uPMMJjw=", - "zh:05943fef14c2028f4722bf078aa1889229e94302f7678cc6f63adb669d8ea612", - "zh:26a163930a92a7408f7bbd0130064b84df8a232b500d8c6c3989952986308539", - "zh:41305feaaade55391447521ec309f3c038b631ca542907ad95132fab71a7e116", - "zh:606919a930f0299948504adbdcd0f239a8af5c418f85741c48f8add370a3d038", - "zh:66963d5b445639511939fc508513fd31da3ee1d4ee1a565ee396c9532897a349", - "zh:6c981ec0c8545556395c43e2511861ab65ee9ecf2a960480e7889c3af0d23af3", - "zh:7334a1bdb726ce1f1bf0a3155f30f84f65206980c229c832ff5f0b0718c44e0b", - "zh:75f6c86bf74511e605423332d113711c76c8028361a32282fb3359d6c7ecae9e", - "zh:7aebb1a01cfe8be54903853202ae06eba14ad99c37d230ed93ce7d6633e05e9b", + "h1:+6VXZmSSeIchab8WY+UFZxaXu6poxZbiJenYwog+Z2M=", + "h1:Bz94gADR83NmrsZC50LheryVcS9erG6tF052hNPqz0U=", + "h1:FGkcn6ieyNPr56Riv75IsCa6okZ9ZHqqFd08Y4i5cPI=", + "h1:GqPjTKaOlhhNIxpPHWJpWdGkEtvUDtuOkjYbVIInfgU=", + "h1:NMDWYRisSFwepjfYsf4MrcyP7Ihjlty937l1BW1Cztg=", + "h1:OusNMAZdIIbxWwHXwH8tYzdbXZPRS8dW7cncSYcHfNI=", + "h1:OzgasS4oZCZjYAOrqGy7RpPNbTPX5wGaaw7TDnasSJM=", + "h1:RyK3DC7cM4T2I83OvnO+UXU+eCXLG5vDlYZ/bWKHq7g=", + "h1:U/JKg4BNWii2mK3bNsOWYMBTpkXqGsWb1ypSJvSKl9Y=", + "h1:UjSUm1AU2wZ8eyC3LDRwWatW9cKD/I0rk7MOeUDDkWc=", + "h1:a888ExTeqWKxaB2GBsDIRSm8jOOMLkPFcABrJxxA8qc=", + "h1:aShYLfSapfo1Ozy33zL5WfmYt5UF2m5EziespPOGNAk=", + "h1:bHdpp2ecmvDoezqg8TirE32SnuDb108/NZ4xBGvJy98=", + "h1:lLYZN5cetXgLPJjo+eR+kwodOYGsQetzV6bg5Ki58tk=", + "h1:szi71DaqI4yB6fI55hBbtVZg1GDWJVkO9+WF/1sZ+IE=", + "h1:vK4jZYZZhD6M4cRmX+171mxoJgb2FSNiBNAUhtm/TNs=", + "h1:yNnSUskn43648XU/YS0e9NUdoHImo7pFMIlBzujBoxY=", + "zh:0c3adf039df2fead1e36b8e7887d965223f03b6b5cdd921ec0c98beaa04fdec0", + "zh:12a3db29733e6619e216fac9aa774cbf67da2105f48c5c4dee8df046d69656fa", + "zh:1330b83c949165434e4d0db5e097a1741376cdb66c06c37b2a97ecea8e06b0f0", + "zh:186113259ab0f80ab9079f0a8a810e93ee097f6740100a0a444f805e8772c12e", + "zh:242fe10185da8a700b0e9a6a52e3d1f6592e8e189aaf5e9a79cf515ad5c7ff25", + "zh:28b005b0aa9485c492326fbb4287d3cb465f25a8d572b8c740e041e040d80aa2", + "zh:3b2cd0daa767dfff67dd39c98d1c82b25e70eb81df77c0d6dd762a0dedec16e1", + "zh:3f14b244740f6c6420d298846708e9f3853c8eda685d3a4d71fb0ffec021ad9c", + "zh:469bfd08deefd90e89c930e5e45bfd06a58a1eadb85dfbee9fc48ae296e8a8c9", + "zh:634d96b84b2e77d25baa074c357e831252c8b8114926ef72e1afb68a4bc9eb65", + "zh:6e837b9485e56f84b5e2bb396fa549b628b8e075257fa01bf8c796ce91d20e58", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9041d0e20c9ceea532de6eebf5cb3a27dad0bb49d3f5b5154be2a08d68fbbf1f", - "zh:a6bbf65431a02be4df0ebb1cbe01185ad357ff6e33c01bd0558f59bed90c8f36", - "zh:c6d075a31096f080c388dfe46036f451c0cc114c3311a4f46ab8dbe1938a202f", - "zh:dd8703f7b55b8bc8e10f8718bea889781100b18e932b04898995b63178c3d36e", - "zh:dd92a5cd4e133a4000e7e5bc8cce876ae0ed803543cedd2f3d590661ba244d04", - "zh:e024fdf121bebc48c1e6debea344c6d4f174117f3ae605fca6e13b9705d92d22", - "zh:ee0e80c31b438e35fa1608f6a2f5824d2806db1e5e8b9f7a90986585c7bcb895", - "zh:fc2d4b705411b48f8c045981f9368a3ea2f74969dd6302008c31ff0bedd51f0a", + "zh:89e02979311e6727a50586b04cb343e9bbec74a6f4d1dba9352971e4334d64c9", + "zh:a6796143fe61ae52d236b7ea96bddbc6317851a0e2117eb2f55cd73c803b4b6f", + "zh:ad2542160e7b57ee8a016cddbf1d32663baf74fe30fcf24df64da75cb13128b0", + "zh:bfee81c153e8b50121fd58eedeeb84bfa64a3d17f23900e35143c4a86474cbd2", + "zh:eeba652697908712cd1ce2c7d2925f6e6bb2d18ea5d3e6ca185e88eeb80f3c80", + "zh:f0b5b8fd6942647c358440591d56a5cb82fda92f7866898654a66ec901174f1f", ] } diff --git a/provider.tf b/provider.tf index badbbc1..d4f2564 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.17" + version = "~> 0.18" } } } From 132d1d2d3cd37611db98733a57d8345a218e152b Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 14 Aug 2026 06:44:10 +0000 Subject: [PATCH 114/125] docs(ci): clarify parallel manual matrix runs --- .crow/process-updates.yaml | 6 +++--- .crow/weekly-audit-missing.yaml | 6 +++--- .crow/weekly-rebuild-missing.yaml | 8 ++++---- .crow/weekly-rebuild-reindex.yaml | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 7600876..021d6cb 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -3,15 +3,15 @@ # Routing is preserved 1:1: # - cron: each existing `process-cran-updates--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `process_cran_updates` dropdown -# ("all" = every os/arch). +# - manual: pick a target from the `process_cran_updates` dropdown; +# "all" fans out every os/arch as parallel matrix workflows. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. process_cran_updates: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 3a9d98f..428aa04 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -3,15 +3,15 @@ # Routing is preserved 1:1: # - cron: each existing `weekly-audit-missing--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `weekly_audit_missing` dropdown -# ("all" = every os/arch). +# - manual: pick a target from the `weekly_audit_missing` dropdown; +# "all" fans out every os/arch as parallel matrix workflows. # Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64). variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. weekly_audit_missing: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 5afb3c3..9639792 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -4,9 +4,9 @@ # - cron: each existing `weekly-rebuild-missing--` cron fires only # its matching matrix rows (via the per-row `cron:` name filter), # which is now all three shards of that slot. -# - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the -# previous bare manual trigger that ran every os/arch); pick a -# single - to run just one. +# - manual: pick a target from the `weekly_rebuild_missing` dropdown; +# "all" fans out every os/arch and shard as parallel matrix +# workflows, while a single - runs its three shards. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). # # The shard picks up its own slice and re-derives what is still outstanding @@ -21,7 +21,7 @@ variables: # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 250b225..2e0f2a0 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -16,7 +16,7 @@ variables: # exactly the slots it rebuilt. A manual pipeline creation instantiates every # file in .crow/, so the default must match no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: - none - all From b75fd2f1c43cf0355e3e2eb9e51d73f16a1321eb Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 14 Aug 2026 06:56:41 +0000 Subject: [PATCH 115/125] fix(ci): split oversized weekly rebuild matrix --- .crow/weekly-rebuild-missing-ubuntu-2604.yaml | 140 ++++++++++++++++++ .crow/weekly-rebuild-missing.yaml | 41 +---- .crow/weekly-rebuild-reindex-ubuntu-2604.yaml | 110 ++++++++++++++ .crow/weekly-rebuild-reindex.yaml | 9 -- 4 files changed, 253 insertions(+), 47 deletions(-) create mode 100644 .crow/weekly-rebuild-missing-ubuntu-2604.yaml create mode 100644 .crow/weekly-rebuild-reindex-ubuntu-2604.yaml diff --git a/.crow/weekly-rebuild-missing-ubuntu-2604.yaml b/.crow/weekly-rebuild-missing-ubuntu-2604.yaml new file mode 100644 index 0000000..0082e0e --- /dev/null +++ b/.crow/weekly-rebuild-missing-ubuntu-2604.yaml @@ -0,0 +1,140 @@ +# ubuntu-2604 rows split from weekly-rebuild-missing.yaml. +# +# The complete rebuild matrix has 54 permutations, while Crow accepts at most +# 50 per workflow. Keeping these six rows in a companion workflow lets a manual +# "all" run fan out every slot and shard without exceeding that compiler limit. + +variables: + # This is the same gate as the main rebuild workflow. The safe default must + # match no row because every file in .crow/ is evaluated on a manual run. + weekly_rebuild_missing: + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-rebuild-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + +steps: + - name: 'Rebuild missing binaries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + GIT_USER: pat-s + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ${OS} + ARCH: ${ARCH} + NCPUS: 2 + SPLIT_INTO: ${SPLIT_INTO} + SPLIT_INDEX: ${SPLIT_INDEX} + # Wall clock after which the shard stops cleanly instead of having to be + # killed. A kill matches neither `success` nor `failure`, so it would skip + # the dependent re-index and leave rebuilt binaries behind a stale edge. + REBUILD_BUDGET_HOURS: 20 + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 + - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 3000m + limits: + memory: 18Gi + cpu: 3000m diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 9639792..aeb2f50 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -1,5 +1,7 @@ -# Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). +# Main weekly-rebuild-missing workflow (all slots except ubuntu-2604). # Three matrix rows per OS/arch, one per shard of that slot's rebuild list. +# ubuntu-2604 lives in weekly-rebuild-missing-ubuntu-2604.yaml so a manual +# "all" run stays below Crow's 50-permutation matrix limit. # Routing is preserved 1:1: # - cron: each existing `weekly-rebuild-missing--` cron fires only # its matching matrix rows (via the per-row `cron:` name filter), @@ -346,43 +348,6 @@ matrix: IMG: ubuntu:noble SPLIT_INTO: 3 SPLIT_INDEX: 3 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-${IMG} diff --git a/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml b/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml new file mode 100644 index 0000000..3ab086a --- /dev/null +++ b/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml @@ -0,0 +1,110 @@ +# Re-index ubuntu-2604 after its split rebuild workflow finishes. + +variables: + # Mirrors the shared rebuild gate. The default must match no matrix row so + # unrelated manual runs do not rewrite package indexes. + weekly_rebuild_missing: + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-rebuild-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' + +depends_on: + - weekly-rebuild-missing-ubuntu-2604 + +runs_on: [success, failure] + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + PLATFORM: ${OS} + ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + commands: + - apk add --no-cache -q bash curl jq + # Crow carries the checkout from the re-index step into this step. + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES + # Run even when the re-index above failed: the objects were still replaced, + # and a stale edge is exactly what keeps them hidden. + when: + - status: [success, failure] diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 2e0f2a0..f38f6d5 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -122,15 +122,6 @@ matrix: ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - steps: - name: 'Re-index the slot' image: reg.devxy.io/rpkgs/build-env-${IMG} From 50495f5c3bb6e3e97d9d42d639c5f9828b6207b6 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 14 Aug 2026 07:01:56 +0000 Subject: [PATCH 116/125] Revert "fix(ci): split oversized weekly rebuild matrix" This reverts commit b75fd2f1c43cf0355e3e2eb9e51d73f16a1321eb. --- .crow/weekly-rebuild-missing-ubuntu-2604.yaml | 140 ------------------ .crow/weekly-rebuild-missing.yaml | 41 ++++- .crow/weekly-rebuild-reindex-ubuntu-2604.yaml | 110 -------------- .crow/weekly-rebuild-reindex.yaml | 9 ++ 4 files changed, 47 insertions(+), 253 deletions(-) delete mode 100644 .crow/weekly-rebuild-missing-ubuntu-2604.yaml delete mode 100644 .crow/weekly-rebuild-reindex-ubuntu-2604.yaml diff --git a/.crow/weekly-rebuild-missing-ubuntu-2604.yaml b/.crow/weekly-rebuild-missing-ubuntu-2604.yaml deleted file mode 100644 index 0082e0e..0000000 --- a/.crow/weekly-rebuild-missing-ubuntu-2604.yaml +++ /dev/null @@ -1,140 +0,0 @@ -# ubuntu-2604 rows split from weekly-rebuild-missing.yaml. -# -# The complete rebuild matrix has 54 permutations, while Crow accepts at most -# 50 per workflow. Keeping these six rows in a companion workflow lets a manual -# "all" run fan out every slot and shard without exceeding that compiler limit. - -variables: - # This is the same gate as the main rebuild workflow. The safe default must - # match no row because every file in .crow/ is evaluated on a manual run. - weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." - options: - - none - - all - - alpine-322-amd64 - - alpine-322-arm64 - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - -when: - - event: cron - cron: weekly-rebuild-missing-${OS}-${ARCH} - - event: manual - evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GIT_USER: pat-s - UVR_CACHE_DIR: /mnt/cache/uvr/cache - UVR_PACKAGES_DIR: /mnt/cache/uvr/packages - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: ${R_VERSION} - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ${OS} - ARCH: ${ARCH} - NCPUS: 2 - SPLIT_INTO: ${SPLIT_INTO} - SPLIT_INDEX: ${SPLIT_INDEX} - # Wall clock after which the shard stops cleanly instead of having to be - # killed. A kill matches neither `success` nor `failure`, so it would skip - # the dependent re-index and leave rebuilt binaries behind a stale edge. - REBUILD_BUDGET_HOURS: 20 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 - backend_options: - docker: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index aeb2f50..9639792 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -1,7 +1,5 @@ -# Main weekly-rebuild-missing workflow (all slots except ubuntu-2604). +# Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). # Three matrix rows per OS/arch, one per shard of that slot's rebuild list. -# ubuntu-2604 lives in weekly-rebuild-missing-ubuntu-2604.yaml so a manual -# "all" run stays below Crow's 50-permutation matrix limit. # Routing is preserved 1:1: # - cron: each existing `weekly-rebuild-missing--` cron fires only # its matching matrix rows (via the per-row `cron:` name filter), @@ -348,6 +346,43 @@ matrix: IMG: ubuntu:noble SPLIT_INTO: 3 SPLIT_INDEX: 3 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-${IMG} diff --git a/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml b/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml deleted file mode 100644 index 3ab086a..0000000 --- a/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Re-index ubuntu-2604 after its split rebuild workflow finishes. - -variables: - # Mirrors the shared rebuild gate. The default must match no matrix row so - # unrelated manual runs do not rewrite package indexes. - weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." - options: - - none - - all - - alpine-322-amd64 - - alpine-322-arm64 - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - -when: - - event: cron - cron: weekly-rebuild-missing-${OS}-${ARCH} - - event: manual - evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' - -depends_on: - - weekly-rebuild-missing-ubuntu-2604 - -runs_on: [success, failure] - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - -steps: - - name: 'Re-index the slot' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GIT_USER: pat-s - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: ${R_VERSION} - PLATFORM: ${OS} - ARCH: ${ARCH} - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - # The codename is detected from the image's /etc/os-release. - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate - # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' - commands: - - apk add --no-cache -q bash curl jq - # Crow carries the checkout from the re-index step into this step. - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES - # Run even when the re-index above failed: the objects were still replaced, - # and a stale edge is exactly what keeps them hidden. - when: - - status: [success, failure] diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index f38f6d5..2e0f2a0 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -122,6 +122,15 @@ matrix: ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + steps: - name: 'Re-index the slot' image: reg.devxy.io/rpkgs/build-env-${IMG} From c7b4dca6e2209c1b56932f37843ef586497fe4e9 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Mon, 17 Aug 2026 00:31:56 +0000 Subject: [PATCH 117/125] chore(deps): lock file maintenance --- .terraform.lock.hcl | 62 ++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index a21cf43..ff935fa 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,38 +2,38 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/http" { - version = "3.6.0" + version = "3.6.1" hashes = [ - "h1:0n4RBz9zNw6TTddh5+x7E8L2+qzPXNwKhK4uoZ/DUwE=", - "h1:22Ob7lpzMBSqdrCvoFN5EgmhGPHPBovV/9qo0c/Cd+A=", - "h1:2IRBvmWOYrq/ooaYYn2i86jZb7iIUvlg0KlmOMfDHoQ=", - "h1:5mucXikk4OcW3un3u94QnMx4AB4Wfih+sXeMd5QxSNk=", - "h1:5oU7Zm+2gAVGmxqtJ9E8uTudUkYy/DEn/y3IWphdv4k=", - "h1:5w0R4b1/VSzpqQF1tXXPr/qmaQLPVRXamOmPKWFcTk4=", - "h1:AEVeJr8xGmwad+JUUQ833C3x5d4W+W2szF5DfwxYppw=", - "h1:CPHJ+0zQbS/cX1m55Y90jIOgf1jV3ocUUnqsXAh+9Eg=", - "h1:JPewnGDOJudNer5+ghqwXoaJkfot3QRq9uiEYvo+JHU=", - "h1:QzbluV2vQLxsJYxjpziQCmPndIoJ/UGS4/UHH/GpwUM=", - "h1:TjUNbUdqweRBq/ycQ4ixpNkx5qaYwpXEOn9QCpqNZP8=", - "h1:XNbcODP60ajj21N/OO7af8bBg1ltIsYkq9egn7BYbiY=", - "h1:tgrbgmX7WYQz9G9ncgu7TkpVB+RlLjJA/Rvp9KPlZH8=", - "h1:vLxthX/ZWsOZ+aHKbAMqmNKqD0K5f4nJ8ppy0Ioyup0=", - "h1:wZOdGBAZkY8OKEPjKz82j1HloAKOmmvtjWyTxM+I110=", - "zh:0f719fa5426bc883e9fa6abf7f6498e48025edafbc29015e2f5c028f1cca3b9d", - "zh:1b4d7dafefd6c61764b2f9ed6943ceb9a200dee3590d18747e3a5f6b20ce85e0", - "zh:1d23a712984866d29f7b07028a4e99c783c71f1a5dddf08bc3d4e7da9d91a1fa", - "zh:257d23d58c3bb024b6bc8eb88736eaf912e934ad47c639d0c3c742bddda849a1", - "zh:479860e1a5468f5e04013b9364c9496d7ed0804bf9a1acd8e07558d57609993d", - "zh:4cb5e681bf599b411b27c4a2c4066a5fb2ed79aaa3a1a3cb5a30002fec062ce9", - "zh:4fb35c3f643dae9f3670d719397a415f815a0b95f8ed7bd8a72f27a94ba78092", - "zh:59ba40825ab38db5b4a0989a2db0df35cc15d8984f898176011ba352f27d77b7", - "zh:61fc1252eb88088638f4c69ea4e2171cde2e5089fa632ac1e943b13787348f73", - "zh:7c5d6dd5f7cbc460e95d368be35c29b4e0402069b8912dbd5d1cd7fa9acef216", - "zh:7f76d756240d4284642f359ad470226e5378670239aadc366ef54d9d914d4d2e", - "zh:8133ad0814098177e0d067c816ccf1bf48bbadacd18f6f2c808c90447505723b", - "zh:c93be06269bb728f1968f8c50506de56c887017ac1d6e4be1f925651d8437eb6", - "zh:ef47b78a10a82e6cf53344a6a85a94041c28286c10a70541c564d762f1cfede0", - "zh:f5796a53a74999135bd9087aff50fddda59129d09b2f9b1902ff8c0c1e047e48", + "h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=", + "h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=", + "h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=", + "h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=", + "h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=", + "h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=", + "h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=", + "h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=", + "h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=", + "h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=", + "h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=", + "h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=", + "h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=", + "h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=", + "h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=", + "zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9", + "zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6", + "zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab", + "zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b", + "zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c", + "zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4", + "zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502", + "zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69", + "zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6", + "zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279", + "zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6", + "zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7", + "zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df", + "zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621", + "zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424", ] } From d1da0c6cb46447d9b295389e4cda211d014d10fb Mon Sep 17 00:00:00 2001 From: automation-bot Date: Sat, 22 Aug 2026 00:32:09 +0000 Subject: [PATCH 118/125] chore(deps): update terraform bunnynet to v0.18.1 --- .terraform.lock.hcl | 70 ++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index ff935fa..ef9e47d 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.18.0" + version = "0.18.1" constraints = "~> 0.18" hashes = [ - "h1:+6VXZmSSeIchab8WY+UFZxaXu6poxZbiJenYwog+Z2M=", - "h1:Bz94gADR83NmrsZC50LheryVcS9erG6tF052hNPqz0U=", - "h1:FGkcn6ieyNPr56Riv75IsCa6okZ9ZHqqFd08Y4i5cPI=", - "h1:GqPjTKaOlhhNIxpPHWJpWdGkEtvUDtuOkjYbVIInfgU=", - "h1:NMDWYRisSFwepjfYsf4MrcyP7Ihjlty937l1BW1Cztg=", - "h1:OusNMAZdIIbxWwHXwH8tYzdbXZPRS8dW7cncSYcHfNI=", - "h1:OzgasS4oZCZjYAOrqGy7RpPNbTPX5wGaaw7TDnasSJM=", - "h1:RyK3DC7cM4T2I83OvnO+UXU+eCXLG5vDlYZ/bWKHq7g=", - "h1:U/JKg4BNWii2mK3bNsOWYMBTpkXqGsWb1ypSJvSKl9Y=", - "h1:UjSUm1AU2wZ8eyC3LDRwWatW9cKD/I0rk7MOeUDDkWc=", - "h1:a888ExTeqWKxaB2GBsDIRSm8jOOMLkPFcABrJxxA8qc=", - "h1:aShYLfSapfo1Ozy33zL5WfmYt5UF2m5EziespPOGNAk=", - "h1:bHdpp2ecmvDoezqg8TirE32SnuDb108/NZ4xBGvJy98=", - "h1:lLYZN5cetXgLPJjo+eR+kwodOYGsQetzV6bg5Ki58tk=", - "h1:szi71DaqI4yB6fI55hBbtVZg1GDWJVkO9+WF/1sZ+IE=", - "h1:vK4jZYZZhD6M4cRmX+171mxoJgb2FSNiBNAUhtm/TNs=", - "h1:yNnSUskn43648XU/YS0e9NUdoHImo7pFMIlBzujBoxY=", - "zh:0c3adf039df2fead1e36b8e7887d965223f03b6b5cdd921ec0c98beaa04fdec0", - "zh:12a3db29733e6619e216fac9aa774cbf67da2105f48c5c4dee8df046d69656fa", - "zh:1330b83c949165434e4d0db5e097a1741376cdb66c06c37b2a97ecea8e06b0f0", - "zh:186113259ab0f80ab9079f0a8a810e93ee097f6740100a0a444f805e8772c12e", - "zh:242fe10185da8a700b0e9a6a52e3d1f6592e8e189aaf5e9a79cf515ad5c7ff25", - "zh:28b005b0aa9485c492326fbb4287d3cb465f25a8d572b8c740e041e040d80aa2", - "zh:3b2cd0daa767dfff67dd39c98d1c82b25e70eb81df77c0d6dd762a0dedec16e1", - "zh:3f14b244740f6c6420d298846708e9f3853c8eda685d3a4d71fb0ffec021ad9c", - "zh:469bfd08deefd90e89c930e5e45bfd06a58a1eadb85dfbee9fc48ae296e8a8c9", - "zh:634d96b84b2e77d25baa074c357e831252c8b8114926ef72e1afb68a4bc9eb65", - "zh:6e837b9485e56f84b5e2bb396fa549b628b8e075257fa01bf8c796ce91d20e58", + "h1:1nbjHMc5QBwiAfkriGGuO+wLkrgwKjbeAK70sNQRDa4=", + "h1:2AkMQZEckeGl8efzFzpGUqIR5o4rhOt4GSmOOMBX8wg=", + "h1:4D42uIgbm5jI0TQWRfVXJZoG6WOyO3Mi0ZgC8KZjPmc=", + "h1:79iaWeho9vn6p5oFHx7cXCzrX9k0R2yS9iF7rNchX7E=", + "h1:7GKjnlsRnA88qGIZ+V4HseEWB1YmuPXkyVen4DOxRBM=", + "h1:9ZpZHafhWNCDM91hE+GK6ikvy9THlU4/opQoC78B6TU=", + "h1:J4eKuqfDRI+e1PRGkt6MJ6UT+ym3pFxYKN+jIm105Es=", + "h1:LvrSDB8WWxh6NcR7X97oxJ9FrB/UVyWau9V+rdxBkxU=", + "h1:NO36kn/RhmKhWVcM0qRUcbZveSv2idT6LUwOREBqBc4=", + "h1:UPrn5yfuJwaEggnIbKqd9eyNCzO9DRJhA0279J1LYLM=", + "h1:b+tjNcFfxqwWGDgumzm4QImqJLyemGanHbZ+F2qw6vU=", + "h1:gppmt6Jbng9QH8ulpSujYeCHSlC5kWEkYoiYzgFISNQ=", + "h1:kZkf+9F8Be+Ztt10sLUWIxnwgfBCHpxi3V66F3+HV5s=", + "h1:mZvRXynxrx0/Q0txogt/5fTVhKmRWX3rWoqLiwLdamw=", + "h1:ogkHPOIbdDgdmfoa0LaPIyX8QTJvWik3andNAJoR6pk=", + "h1:pCkHlvcWgM5FkwEsrxzT9L7S7ZDNfyWagWM0KMXYFBE=", + "h1:rbKmEiaOKwrbrzEf9iihpwFCwvlVVkyxZUfFeVuX180=", + "zh:198f5aab9e8bbbb6fa96e41b7d33a997e72666dfc7369f08849ff12a0f91f7d4", + "zh:1f152ab9c51353422a79d4c4ee965112972b0d4b3246c62a8e6848422f4cac26", + "zh:4776e4fc6f38b1eb4a64c866e617cebdb0344af51c4685aa3e47356a5502707f", + "zh:66828334af0bdecbde3c5913b9c48371ea119569631f43094e28e03ab1b5fa4a", + "zh:800a94528ed366242fb83ba8d22ca1407bb02826ac8805a83a0ab04ab5efb582", + "zh:862f5e7db0f81d5e57292f0cd44b4f530eaa59510756991037d1ed6fbff91e05", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:89e02979311e6727a50586b04cb343e9bbec74a6f4d1dba9352971e4334d64c9", - "zh:a6796143fe61ae52d236b7ea96bddbc6317851a0e2117eb2f55cd73c803b4b6f", - "zh:ad2542160e7b57ee8a016cddbf1d32663baf74fe30fcf24df64da75cb13128b0", - "zh:bfee81c153e8b50121fd58eedeeb84bfa64a3d17f23900e35143c4a86474cbd2", - "zh:eeba652697908712cd1ce2c7d2925f6e6bb2d18ea5d3e6ca185e88eeb80f3c80", - "zh:f0b5b8fd6942647c358440591d56a5cb82fda92f7866898654a66ec901174f1f", + "zh:91f8b0f92b7dd1e131e391c170fcb329c795747553d5cc71296b8803496bc33c", + "zh:95446852691ebbbfcd4998f8941d29577d61b15ac320bc4590ae48f4542aee39", + "zh:9ccb3382c4fc20735c3ee016240afa82d6cf745b8ec458a9e5159d1a03e4cd20", + "zh:a11991312edc9fe2567ecedadb2e87de8b1a41e97a0d117d1a1c58eaa68ae645", + "zh:ab729171af18063c34f5a7d33b598b870e082f2e7ba8ee60ca77dc4dede84432", + "zh:c2b58950da56179b93a7d590d0cf3a488070b973826d788401a94914eaeae072", + "zh:d1b6a91581f1580fad12f11b9593461f566153b6760299ba592746716d595583", + "zh:dad0ce1ac0fc934ab18fe4a744b768d8f9969099488b25d6c18c782b40478bfe", + "zh:e2cb8dfebfa0a358d0b52e15b04cfd82407a4c578c39b6ac62ac919b23ef97a6", + "zh:fc648d9464bcb6b360eb4885791675c2437f0d1c0e1b1b8f34fbb6d93a880ea9", + "zh:fd8250944e3794b9440cdaf4ac06f7a1672744e4bcb2e8a33b94c2ddf32ab988", ] } From f4ab6f9dc55d2e33822abf5d48e6517a33bd46e9 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Wed, 26 Aug 2026 00:32:02 +0000 Subject: [PATCH 119/125] chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker to v3.11.2 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82461cb..2bf38b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.11.1 + rev: v3.11.2 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ From d2333c6cbe599fc5e43c7534d55bb2120e104245 Mon Sep 17 00:00:00 2001 From: automation-bot Date: Thu, 27 Aug 2026 00:33:08 +0000 Subject: [PATCH 120/125] chore(deps): update terraform bunnynet to v0.18.2 --- .terraform.lock.hcl | 70 ++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index ef9e47d..9591b83 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -38,43 +38,43 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.18.1" + version = "0.18.2" constraints = "~> 0.18" hashes = [ - "h1:1nbjHMc5QBwiAfkriGGuO+wLkrgwKjbeAK70sNQRDa4=", - "h1:2AkMQZEckeGl8efzFzpGUqIR5o4rhOt4GSmOOMBX8wg=", - "h1:4D42uIgbm5jI0TQWRfVXJZoG6WOyO3Mi0ZgC8KZjPmc=", - "h1:79iaWeho9vn6p5oFHx7cXCzrX9k0R2yS9iF7rNchX7E=", - "h1:7GKjnlsRnA88qGIZ+V4HseEWB1YmuPXkyVen4DOxRBM=", - "h1:9ZpZHafhWNCDM91hE+GK6ikvy9THlU4/opQoC78B6TU=", - "h1:J4eKuqfDRI+e1PRGkt6MJ6UT+ym3pFxYKN+jIm105Es=", - "h1:LvrSDB8WWxh6NcR7X97oxJ9FrB/UVyWau9V+rdxBkxU=", - "h1:NO36kn/RhmKhWVcM0qRUcbZveSv2idT6LUwOREBqBc4=", - "h1:UPrn5yfuJwaEggnIbKqd9eyNCzO9DRJhA0279J1LYLM=", - "h1:b+tjNcFfxqwWGDgumzm4QImqJLyemGanHbZ+F2qw6vU=", - "h1:gppmt6Jbng9QH8ulpSujYeCHSlC5kWEkYoiYzgFISNQ=", - "h1:kZkf+9F8Be+Ztt10sLUWIxnwgfBCHpxi3V66F3+HV5s=", - "h1:mZvRXynxrx0/Q0txogt/5fTVhKmRWX3rWoqLiwLdamw=", - "h1:ogkHPOIbdDgdmfoa0LaPIyX8QTJvWik3andNAJoR6pk=", - "h1:pCkHlvcWgM5FkwEsrxzT9L7S7ZDNfyWagWM0KMXYFBE=", - "h1:rbKmEiaOKwrbrzEf9iihpwFCwvlVVkyxZUfFeVuX180=", - "zh:198f5aab9e8bbbb6fa96e41b7d33a997e72666dfc7369f08849ff12a0f91f7d4", - "zh:1f152ab9c51353422a79d4c4ee965112972b0d4b3246c62a8e6848422f4cac26", - "zh:4776e4fc6f38b1eb4a64c866e617cebdb0344af51c4685aa3e47356a5502707f", - "zh:66828334af0bdecbde3c5913b9c48371ea119569631f43094e28e03ab1b5fa4a", - "zh:800a94528ed366242fb83ba8d22ca1407bb02826ac8805a83a0ab04ab5efb582", - "zh:862f5e7db0f81d5e57292f0cd44b4f530eaa59510756991037d1ed6fbff91e05", + "h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=", + "h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=", + "h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=", + "h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=", + "h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=", + "h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=", + "h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=", + "h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=", + "h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=", + "h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=", + "h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=", + "h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=", + "h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=", + "h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=", + "h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=", + "h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=", + "h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=", + "zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c", + "zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178", + "zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd", + "zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef", + "zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738", + "zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc", + "zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:91f8b0f92b7dd1e131e391c170fcb329c795747553d5cc71296b8803496bc33c", - "zh:95446852691ebbbfcd4998f8941d29577d61b15ac320bc4590ae48f4542aee39", - "zh:9ccb3382c4fc20735c3ee016240afa82d6cf745b8ec458a9e5159d1a03e4cd20", - "zh:a11991312edc9fe2567ecedadb2e87de8b1a41e97a0d117d1a1c58eaa68ae645", - "zh:ab729171af18063c34f5a7d33b598b870e082f2e7ba8ee60ca77dc4dede84432", - "zh:c2b58950da56179b93a7d590d0cf3a488070b973826d788401a94914eaeae072", - "zh:d1b6a91581f1580fad12f11b9593461f566153b6760299ba592746716d595583", - "zh:dad0ce1ac0fc934ab18fe4a744b768d8f9969099488b25d6c18c782b40478bfe", - "zh:e2cb8dfebfa0a358d0b52e15b04cfd82407a4c578c39b6ac62ac919b23ef97a6", - "zh:fc648d9464bcb6b360eb4885791675c2437f0d1c0e1b1b8f34fbb6d93a880ea9", - "zh:fd8250944e3794b9440cdaf4ac06f7a1672744e4bcb2e8a33b94c2ddf32ab988", + "zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef", + "zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3", + "zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb", + "zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e", + "zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11", + "zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7", + "zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9", + "zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1", + "zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff", + "zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339", ] } From e9782bb529378d2b86e1bf87b1a32d49a1618ea8 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 28 Aug 2026 08:48:58 +0000 Subject: [PATCH 121/125] fix(ci): install RPostgres for metadata updates --- .crow/process-updates.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 021d6cb..73bfff7 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -203,6 +203,7 @@ steps: - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi @@ -218,6 +219,7 @@ steps: LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true + R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' From 77a2f1f04a645f5001755c477c34e285e73215e0 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 07:31:28 +0000 Subject: [PATCH 122/125] fix(ci): install RPostgres for audit workflows (#172) ## Summary - Install RPostgres before running the missing-binaries audit. - Install RPostgres before running weekly patch proposal and automatic patch workflows. ## Validation - `prek run --files .crow/auto-apply-patches.yaml .crow/weekly-audit-missing.yaml .crow/weekly-patch-proposals.yaml` - `crow lint .crow/` - `git diff --check` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/172 --- .crow/auto-apply-patches.yaml | 2 +- .crow/weekly-audit-missing.yaml | 2 +- .crow/weekly-patch-proposals.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index 7c33c5b..5d6e6fe 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -58,7 +58,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT backend_options: kubernetes: diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 428aa04..25875da 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -147,7 +147,7 @@ steps: - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' backend_options: docker: diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 712e87d..7ea6c97 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -53,7 +53,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: From a8820e6e90a14469d4d566b3ee295aaeb5c51eff Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 12:19:52 +0000 Subject: [PATCH 123/125] fix(cdn): route Ubuntu 26.04 to resolute (#173) ## Motivation The generic `https://cran.rpkgs.com/` repository falls back to upstream CRAN on Ubuntu 26.04 because the edge router does not recognize its release version. This causes Resolute users to download and compile source packages even when matching binaries exist. ## Changes - Map Ubuntu 26.04 user agents to the `resolute` repository slot. - Cover ARM64 Ubuntu 26.04 routing with an edge middleware regression test. ## Verification - `just edge-test`, 1 test with 15 steps passed. - Confirmed the current live generic route redirects the Resolute user agent to upstream CRAN. - Confirmed the explicit `amd64/resolute/latest` repository installs `rlang` as a binary in `reg.devxy.io/r/r-ubuntu:4.6-resolute`. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/173 --- edge/rpkgs-router.test.ts | 9 +++++++++ edge/rpkgs-router.ts | 1 + 2 files changed, 10 insertions(+) diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 9493f8a..e9bbcc9 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -18,6 +18,7 @@ const UNION_SLOTS = 'amd64/alpine324'; const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; +const UA_R45_RESOLUTE = 'R/4.5.3 (Ubuntu 26.04) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)'; const UA_CURL = 'curl/8.0.1'; @@ -141,6 +142,14 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); }); + await t.step('resolves Ubuntu 26.04 to the resolute slot', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_RESOLUTE); + assertEquals( + res.location, + 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz', + ); + }); + await t.step('sends an unidentifiable distro to CRAN', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL); assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index 9278d4a..c9195a4 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -59,6 +59,7 @@ const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i; const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i; const UBUNTU_CODENAMES: Record = { + '26.04': 'resolute', '24.04': 'noble', '22.04': 'jammy', }; From b0d58f3f7c7f3577727be25da9368af3f45b7eff Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 12:25:22 +0000 Subject: [PATCH 124/125] feat(cdn): derive Ubuntu routes from codenames (#174) ## Motivation Ubuntu release codenames cannot be derived from version numbers, so the edge router currently needs a code change for every Ubuntu release. The generic repository setup can already read `VERSION_CODENAME` from `/etc/os-release` and send it in the user agent. ## Changes - Prefer a strictly validated `codename=` token for Ubuntu slot routing. - Retain the existing version-to-codename table for clients using the previous user-agent format. - Add a future Ubuntu 28.04 regression case whose codename does not exist in the middleware table. ## User-agent format `R/4.6.1 (Ubuntu 26.04; codename=resolute) (aarch64-unknown-linux-gnu)` ## Verification - `just edge-test`, 1 test with 16 steps passed. - `git diff --check` Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/174 --- edge/rpkgs-router.test.ts | 10 ++++++++++ edge/rpkgs-router.ts | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index e9bbcc9..3020b2d 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -19,6 +19,8 @@ const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; const UA_R45_RESOLUTE = 'R/4.5.3 (Ubuntu 26.04) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; +const UA_R45_FUTURE_UBUNTU = + 'R/4.5.3 (Ubuntu 28.04; codename=dynamic-dugong) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)'; const UA_CURL = 'curl/8.0.1'; @@ -150,6 +152,14 @@ Deno.test('rpkgs-router', async (t) => { ); }); + await t.step('resolves a future Ubuntu release from its codename', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_FUTURE_UBUNTU); + assertEquals( + res.location, + 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz', + ); + }); + await t.step('sends an unidentifiable distro to CRAN', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL); assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index c9195a4..cdeac44 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -58,6 +58,7 @@ const MACOS_BIN_REGEX = const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i; const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i; +const UBUNTU_CODENAME_REGEX = /Ubuntu [\d.]+;\s*codename=([a-z][a-z0-9-]*)/i; const UBUNTU_CODENAMES: Record = { '26.04': 'resolute', '24.04': 'noble', @@ -137,6 +138,11 @@ function parseSlot(userAgent: string): string | null { const ubuntu = userAgent.match(UBUNTU_REGEX); if (ubuntu) { + const codenameMatch = userAgent.match(UBUNTU_CODENAME_REGEX); + if (codenameMatch) { + return `${arch}/${codenameMatch[1].toLowerCase()}`; + } + const codename = UBUNTU_CODENAMES[ubuntu[1]]; if (codename) { return `${arch}/${codename}`; From eeebef8edb8fb166cfb8125c71a690340560dfa3 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 14:26:37 +0000 Subject: [PATCH 125/125] fix(patches): support RcppParallel 6.2.1 This change will:\n\n- Relax the TBB include hunk context to tolerate the upstream TBB_CXXFLAGS block.\n- Pin the source patch to the verified RcppParallel version. --- local/patches/RcppParallel/force-bundled-tbb.patch | 6 +----- local/patches/registry.json | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/local/patches/RcppParallel/force-bundled-tbb.patch b/local/patches/RcppParallel/force-bundled-tbb.patch index c46a05e..5c37fa6 100644 --- a/local/patches/RcppParallel/force-bundled-tbb.patch +++ b/local/patches/RcppParallel/force-bundled-tbb.patch @@ -40,15 +40,11 @@ index 6f6a745..e407986 100644 if (is.null(name)) return(tbbRoot) -@@ -58,7 +58,7 @@ tbbCxxFlags <- function() { - flags <- c("-DRCPP_PARALLEL_USE_TBB=1") - +@@ -58,3 +58,3 @@ tbbCxxFlags <- function() { # if TBB_INC is set, apply those library paths - tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) + tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) if (!file.exists(tbbInc)) { - tbbInc <- system.file("include", package = "RcppParallel") - } @@ -117,7 +117,7 @@ tbbLdFlags <- function() { } diff --git a/local/patches/registry.json b/local/patches/registry.json index a7e29dd..2457304 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -1,7 +1,7 @@ [ { "package": "RcppParallel", - "versions": ">=6.0.0", + "versions": "6.2.1", "platforms": ["*"], "env": {}, "configure_args": [],