diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 71f808b..bd127d6 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -28,10 +28,6 @@ matrix: ARCH: amd64 - CODENAME: redhat-10 ARCH: arm64 - - CODENAME: alpine321 - ARCH: amd64 - - CODENAME: alpine321 - ARCH: arm64 - CODENAME: alpine322 ARCH: amd64 - CODENAME: alpine322 diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml new file mode 100644 index 0000000..7c33c5b --- /dev/null +++ b/.crow/auto-apply-patches.yaml @@ -0,0 +1,76 @@ +# Auto-apply registry patches (issue #115, step 3 automation). +# Classifies `single_builds` failures and, for the top-N auto-proposable +# candidates by failure volume, writes the registry entries onto the reused +# `auto/registry-patch-proposals` branch and opens/updates a single PR. +# Nothing merges: the `trial-build-registry` pipeline is the merge gate, and a +# human reviews the PR. Novel source diffs / unknown signatures are never +# proposed. Global across platforms, so a single job -- no matrix. +# +# FORGEJO_TOKEN is used for both the branch push and opening the PR (no separate +# write-scoped secret needed). Register the `auto-apply-patches` cron in the crow +# UI, or run manually: +# crow pipeline create --branch main \ +# --var auto_apply_patches=true devxy/build-cran-binaries +# +# The gate variable is `auto_apply_patches`, named after the pipeline: a manual +# run instantiates every pipeline in `.crow/`, so one without its own gate runs +# on *any* manual trigger in this repo. This one pushes a branch and opens a PR, +# so it must stay off unless it is what was asked for. +variables: + auto_apply_patches: + description: 'Run the auto-patch proposer. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' + patch_limit: + description: 'Max candidates to propose per run (top by failure volume).' + default: '10' + +when: + - event: manual + evaluate: 'auto_apply_patches == "true"' + - event: cron + cron: auto-apply-patches + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Auto-apply registry patches' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + GIT_USER: devxy-bot + GIT_EMAIL: bot@devxy.io + PATCH_LIMIT: ${patch_limit} + R_VERSION: 4.5.3 + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite + - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 2000m + limits: + memory: 2Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index e0cca86..2a6d9c2 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -3,29 +3,33 @@ # Variables are declared so the manual-run form exposes them (crow #1165); # they are merged with build-all-versions' identical declarations. variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. target_arch: - description: 'Architecture to build.' + description: 'Architecture to build, or "none" to run nothing.' options: + - none - amd64 - arm64 - default: amd64 + default: none OS: - description: "Base OS image name." + description: 'Base OS image name.' options: - alpine - redhat - ubuntu default: alpine OS_VERSION: - description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' options: - - "3.24" - - "8" - - "9" - - "10" - - "jammy" - - "noble" - default: "3.24" + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -66,23 +70,24 @@ steps: from_secret: B2_S3_SECRET_KEY PGPASS: from_secret: PGPASS - R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs + # Keep uvr's downloads and extracted-package entries on the persistent + # volume instead of the container-local ~/.uvr default. + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages CCACHE_DIR: /mnt/cache/ccache volumes: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: # one-time full wipe to fix corrupted .so files from previous failed builds # - rm -rf /mnt/cache/R-pkgs - # Clear churny pkgcache metadata left by a prior crashed run (the "patched" - # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). - # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. - - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 3932811..fb1229d 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -4,32 +4,36 @@ # image and cache volume. Placement is via the group label (rpkgs-amd64/rpkgs-arm64). # Skip list lives in local/excluded-packages.json (read by local/build-all.R). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. target_arch: - description: 'Architecture to build.' + description: 'Architecture to build, or "none" to run nothing.' options: + - none - amd64 - arm64 - default: amd64 + default: none OS: - description: "Base OS image name." + description: 'Base OS image name.' options: - alpine - redhat - ubuntu default: alpine OS_VERSION: - description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' options: - - "3.22" - - "3.23" - - "3.24" - - "8" - - "9" - - "10" - - "jammy" - - "noble" - - "resolute" - default: "3.24" + - '3.22' + - '3.23' + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + - 'resolute' + default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -47,38 +51,49 @@ labels: platform: linux/${ARCH} group: rpkgs-${ARCH} +# Empty UVR_CACHE_DIR/UVR_PACKAGES_DIR fall back to uvr's container-local +# ~/.uvr defaults; amd64 deliberately does not persist them (as with the +# pkgcache dir it replaces), arm64 does. matrix: include: - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 4 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 4 @@ -107,8 +122,10 @@ steps: from_secret: GITHUB_PAT # normal env vars GIT_USER: pat-s - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR} + # set the location of uvr's caches, which persist the R package + # dependencies needed to install the packages themselves + UVR_CACHE_DIR: ${UVR_CACHE_DIR} + UVR_PACKAGES_DIR: ${UVR_PACKAGES_DIR} R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache NCPUS: 2 @@ -116,12 +133,7 @@ steps: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - # Clear churny pkgcache metadata left by a prior crashed run (the "patched" - # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). - # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. Within-run - # growth is bounded separately by trim_pkgcache_metadata() in build-all.R. - - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # The primary pass must not rely on build-all-versions-install-deps having # run on *this* agent: depends_on only orders the steps, but the cache # volume is per-agent, so a job landing on an agent where install-deps did diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 3da41fd..73bfff7 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -3,13 +3,17 @@ # Routing is preserved 1:1: # - cron: each existing `process-cran-updates--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `process_cran_updates` dropdown -# ("all" = every os/arch). +# - manual: pick a target from the `process_cran_updates` dropdown; +# "all" fans out every os/arch as parallel matrix workflows. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. process_cran_updates: - description: "Manual run target: a specific -, or 'all' for every os/arch." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: + - none - all - alpine-322-amd64 - alpine-322-arm64 @@ -29,7 +33,7 @@ variables: - ubuntu-2404-arm64 - ubuntu-2604-amd64 - ubuntu-2604-arm64 - default: all + default: none when: - event: cron @@ -47,111 +51,111 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 OS_ID: alpine322 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 OS_ID: alpine322 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 OS_ID: alpine323 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 OS_ID: alpine323 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine324 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine324 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-8 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-9 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-9 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-10 ARCH: amd64 R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-10 ARCH: arm64 R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2204 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2204 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2404 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2404 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2604 ARCH: amd64 R_VERSION: 4.5.3 IMG: ubuntu:resolute OS_ID: resolute - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2604 ARCH: arm64 R_VERSION: 4.5.3 IMG: ubuntu:resolute OS_ID: resolute - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' steps: - name: 'Processing Updates' @@ -182,8 +186,10 @@ steps: NTFY_AUTH: TRUE NTFY_PASSWORD: from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + # set the location of uvr's caches, which persist the R package + # dependencies needed to install the packages themselves + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache @@ -194,10 +200,11 @@ steps: INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -212,6 +219,7 @@ steps: LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true + R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml new file mode 100644 index 0000000..e56e1e9 --- /dev/null +++ b/.crow/repair-built-stamp.yaml @@ -0,0 +1,110 @@ +### Manual repair of a slot whose PACKAGES index advertises a broken `Built` +### stamp (e.g. `Built: R 4.5.0; NA; ...`). +# +# uvr matches the stamp's platform triple plus R minor to decide binary vs +# source, so an unusable triple turns a whole slot source-only. See +# local/repair-built-stamp.R for why this patches PACKAGES.db in place instead +# of forcing a full reparse. +# +# Run with `dry_run: true` first: it reports how many entries are broken per +# slot and changes nothing. Pick the R version the slot should advertise, which +# is the R_VERSION its entry in .crow/process-updates.yaml uses. +# +# The gate variable is `repair_built_stamp`, not `target_arch`: `target_arch` is +# what build-all-versions and build-all-versions-install-deps gate on, so a +# manual run passing it would start a full rebuild alongside this repair. Every +# pipeline here gates on a variable named after itself for exactly that reason. +# +# crow pipeline create --branch main \ +# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \ +# --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries +variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. + repair_built_stamp: + description: 'Architecture of the slot to repair, or "none" to run nothing.' + options: + - none + - amd64 + - arm64 + default: none + OS: + description: 'Base OS image name.' + options: + - alpine + - redhat + - ubuntu + default: alpine + OS_VERSION: + description: 'OS image tag. Must match OS (alpine: 3.22/3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).' + options: + - '3.22' + - '3.23' + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + - 'resolute' + default: '3.22' + R_VERSION: + description: 'R version whose stamp the slot should advertise.' + options: + - 4.5.3 + - 4.4.3 + default: 4.5.3 + dry_run: + description: 'Report what would change without writing anything.' + options: + - 'true' + - 'false' + default: 'true' + +when: + - event: manual + evaluate: 'repair_built_stamp == "${ARCH}"' + +skip_clone: true + +labels: + platform: linux/${ARCH} + group: rpkgs-${ARCH} + +matrix: + include: + - ARCH: amd64 + - ARCH: arm64 + +steps: + - name: 'Repair Built stamp' + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - | + if [ "$dry_run" = "false" ]; then + /opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" --apply + else + /opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" + fi + backend_options: + docker: + resources: + requests: + memory: 2Gi + cpu: 1000m + limits: + memory: 8Gi + cpu: 2000m diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml new file mode 100644 index 0000000..2195423 --- /dev/null +++ b/.crow/trial-build-registry.yaml @@ -0,0 +1,160 @@ +# Merge gate for the auto-patch PR (issue #115, step 3). +# For each platform, trial-builds every registry entry the auto-patch branch +# ADDS (vs main) in that platform's own `reg.devxy.io/rpkgs/build-env-*` image, +# with the registry applied. A row with no new entries for its platform is a +# fast no-op. The pipeline is green only if every new entry builds, so it gates +# the PR before merge. Nothing is uploaded/archived/recorded. +# +# The repo uses no `pull_request` triggers, so this runs manually against the +# branch (or on a cron); point it at the auto-patch branch via `patch_branch`: +# crow pipeline create --branch main --var trial_build_registry=true \ +# --var patch_branch=auto/registry-patch-proposals devxy/build-cran-binaries +# +# The gate variable is `trial_build_registry`, named after the pipeline: a +# manual run instantiates every pipeline in `.crow/`, so one without its own +# gate runs on *any* manual trigger in this repo. This one starts a build per +# matrix row on both arches, which is far too expensive to fire by accident. +variables: + trial_build_registry: + description: 'Trial-build the branch new registry entries. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' + patch_branch: + description: 'Branch whose new registry entries to trial-build.' + default: auto/registry-patch-proposals + +when: + - event: manual + evaluate: 'trial_build_registry == "true"' + - event: cron + cron: trial-build-registry + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + +steps: + - name: 'Trial-build new registry entries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + PLATFORM: ${OS} + ARCH: ${ARCH} + R_VERSION: ${R_VERSION} + R_LIBS_USER: /mnt/cache/R-pkgs + # Surface the real compiler error when an isolated patched build fails, + # instead of bincraft's opaque "System command 'R' failed" (needs bincraft + # with BINCRAFT_VERBOSE_PATCH_BUILD support; harmless on older versions). + BINCRAFT_VERBOSE_PATCH_BUILD: 'TRUE' + commands: + # Clone main, then check out the auto-patch branch if it exists. When the + # proposer had no candidates it never (re)creates that branch, so a missing + # branch means "nothing to verify" -- no-op cleanly instead of failing the + # clone. + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - git fetch -q origin main + - 'if git ls-remote --exit-code --heads origin ${patch_branch} >/dev/null 2>&1; then git fetch -q origin ${patch_branch} && git checkout -q FETCH_HEAD; else echo "No ${patch_branch} branch; no pending auto-patch proposals to verify."; exit 0; fi' + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/Rscript local/trial-build-registry.R origin/main + backend_options: + kubernetes: + resources: + requests: + memory: 2Gi + cpu: 2000m + limits: + memory: 4Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 6ff71e3..428aa04 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -3,20 +3,24 @@ # Routing is preserved 1:1: # - cron: each existing `weekly-audit-missing--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `weekly_audit_missing` dropdown -# ("all" = every os/arch). +# - manual: pick a target from the `weekly_audit_missing` dropdown; +# "all" fans out every os/arch as parallel matrix workflows. # Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64). variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. weekly_audit_missing: - description: "Manual run target: a specific -, or 'all' for every os/arch." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: + - none - all - - alpine-321-amd64 - - alpine-321-arm64 - alpine-322-amd64 - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -27,7 +31,9 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 - default: all + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none when: - event: cron @@ -42,27 +48,27 @@ labels: matrix: include: - - OS: alpine-321 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: alpine-321 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 @@ -106,6 +112,14 @@ matrix: ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute steps: - name: 'Audit missing binaries' @@ -132,7 +146,8 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' backend_options: docker: diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 0284e93..712e87d 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -8,8 +8,26 @@ # Global across platforms (the classifier groups over all of single_builds), so # a single job -- no matrix. Clones read-only; the only writes are the two # Forgejo issues via FORGEJO_TOKEN. +# +# Run manually with: +# crow pipeline create --branch main \ +# --var weekly_patch_proposals=true devxy/build-cran-binaries +# +# The gate variable is `weekly_patch_proposals`, named after the pipeline: a +# manual run instantiates every pipeline in `.crow/`, so one without its own +# gate runs on *any* manual trigger in this repo. This one posts and edits +# Forgejo issues, so an unrelated manual run must not fire it. +variables: + weekly_patch_proposals: + description: 'Run the weekly failure triage. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' + when: - event: manual + evaluate: 'weekly_patch_proposals == "true"' - event: cron cron: weekly-patch-proposals @@ -35,7 +53,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 934c518..9639792 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -1,21 +1,36 @@ # Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). -# One matrix row per OS/arch replaces the former per-platform files. +# Three matrix rows per OS/arch, one per shard of that slot's rebuild list. # Routing is preserved 1:1: # - cron: each existing `weekly-rebuild-missing--` cron fires only -# its matching matrix row (via the per-row `cron:` name filter). -# - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the -# previous bare manual trigger that ran every os/arch); pick a -# single - to run just one. +# its matching matrix rows (via the per-row `cron:` name filter), +# which is now all three shards of that slot. +# - manual: pick a target from the `weekly_rebuild_missing` dropdown; +# "all" fans out every os/arch and shard as parallel matrix +# workflows, while a single - runs its three shards. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). +# +# The shard picks up its own slice and re-derives what is still outstanding +# from the bucket, so a restart resumes rather than replaying; see +# local/rebuild-missing.R. +# +# Re-indexing and the CDN purge deliberately do NOT live here. Three shards +# writing one slot's PACKAGES concurrently would race, so they moved to +# .crow/weekly-rebuild-reindex.yaml, which depends on this pipeline. variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, or 'all' for every os/arch." + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: + - none - all - alpine-322-amd64 - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -26,7 +41,9 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 - default: all + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none when: - event: cron @@ -44,59 +61,327 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.24 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-8 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-9 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-9 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-10 ARCH: amd64 R_VERSION: 4.5.3 IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: redhat-10 ARCH: arm64 R_VERSION: 4.5.3 IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2204 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2204 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2404 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: ubuntu-2404 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 steps: - name: 'Rebuild missing binaries' @@ -121,23 +406,30 @@ steps: FORGEJO_TOKEN: from_secret: FORGEJO_TOKEN GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache PLATFORM: ${OS} ARCH: ${ARCH} NCPUS: 2 + SPLIT_INTO: ${SPLIT_INTO} + SPLIT_INDEX: ${SPLIT_INDEX} + # Wall clock after which the shard stops cleanly instead of having to be + # killed. A kill matches neither `success` nor `failure`, so it would skip + # the dependent re-index and leave rebuilt binaries behind a stale edge. + REBUILD_BUDGET_HOURS: 20 commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 backend_options: docker: resources: diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml new file mode 100644 index 0000000..2e0f2a0 --- /dev/null +++ b/.crow/weekly-rebuild-reindex.yaml @@ -0,0 +1,191 @@ +# Re-index and purge after weekly-rebuild-missing. +# +# weekly-rebuild-missing runs three shards per slot. Each of them replaces +# objects in place, so the slot's index still advertises the old MD5 and, for +# anything that had been served from source, no Built stamp. Re-indexing from +# inside a shard would mean three concurrent `upload_package_index()` calls on +# one prefix: `cranlike::update_PACKAGES()` lists the live bucket, so an early +# lister that uploads last publishes an index missing its siblings' work. +# +# So it happens exactly once per slot, here, after every shard has finished. +# `runs_on: [success, failure]` keeps that true when a shard fails; only an +# explicit cancel skips it, and this pipeline can then be triggered on its own. + +variables: + # Mirrors the gate on weekly-rebuild-missing so a manual run re-indexes + # exactly the slots it rebuilt. A manual pipeline creation instantiates every + # file in .crow/, so the default must match no matrix row. + weekly_rebuild_missing: + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-rebuild-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' + +depends_on: + - weekly-rebuild-missing + +runs_on: [success, failure] + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + PLATFORM: ${OS} + ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + commands: + - apk add --no-cache -q bash curl jq + # Crow carries the checkout from the re-index step into this step. + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES + # Runs on every row rather than on one designated slot: a cron fires only + # its own slot's row, so gating on a named slot would leave every other + # slot unpurged. A manual "all" run therefore purges the zone 18 times, + # which is a cheap API call and rare. + # + # Run it even when the re-index above failed: the objects were still + # replaced, and a stale edge is exactly what keeps them hidden. + when: + - status: [success, failure] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0f4422..2bf38b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,30 +7,39 @@ exclude: | benchmark/| docker/reprex/ ) +# The `^local/patches/.*\.patch$` excludes below keep unified diffs byte-exact: +# a context line for a blank line is a single space, and stripping it (or +# appending a newline) makes `git apply` reject the patch, which surfaces as +# "patch did not apply cleanly" at build time rather than as a lint failure +# here. The exclusions are per-hook, not global, so `validate patch registry` +# still runs when a patch changes. repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: end-of-file-fixer + exclude: ^local/patches/.*\.patch$ - id: trailing-whitespace + exclude: ^local/patches/.*\.patch$ args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.23.0 + rev: v0.23.2 hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.4 + rev: v3.9.6 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit - rev: 0.10.0 + rev: 0.11.0 hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.8.0 + rev: v3.11.2 hooks: - id: editorconfig-checker + exclude: ^local/patches/.*\.patch$ - repo: https://github.com/adrienverge/yamllint.git rev: v1.38.0 hooks: diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 6ef10fa..9591b83 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,79 +2,79 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/http" { - version = "3.6.0" + version = "3.6.1" hashes = [ - "h1:0n4RBz9zNw6TTddh5+x7E8L2+qzPXNwKhK4uoZ/DUwE=", - "h1:22Ob7lpzMBSqdrCvoFN5EgmhGPHPBovV/9qo0c/Cd+A=", - "h1:2IRBvmWOYrq/ooaYYn2i86jZb7iIUvlg0KlmOMfDHoQ=", - "h1:5mucXikk4OcW3un3u94QnMx4AB4Wfih+sXeMd5QxSNk=", - "h1:5oU7Zm+2gAVGmxqtJ9E8uTudUkYy/DEn/y3IWphdv4k=", - "h1:5w0R4b1/VSzpqQF1tXXPr/qmaQLPVRXamOmPKWFcTk4=", - "h1:AEVeJr8xGmwad+JUUQ833C3x5d4W+W2szF5DfwxYppw=", - "h1:CPHJ+0zQbS/cX1m55Y90jIOgf1jV3ocUUnqsXAh+9Eg=", - "h1:JPewnGDOJudNer5+ghqwXoaJkfot3QRq9uiEYvo+JHU=", - "h1:QzbluV2vQLxsJYxjpziQCmPndIoJ/UGS4/UHH/GpwUM=", - "h1:TjUNbUdqweRBq/ycQ4ixpNkx5qaYwpXEOn9QCpqNZP8=", - "h1:XNbcODP60ajj21N/OO7af8bBg1ltIsYkq9egn7BYbiY=", - "h1:tgrbgmX7WYQz9G9ncgu7TkpVB+RlLjJA/Rvp9KPlZH8=", - "h1:vLxthX/ZWsOZ+aHKbAMqmNKqD0K5f4nJ8ppy0Ioyup0=", - "h1:wZOdGBAZkY8OKEPjKz82j1HloAKOmmvtjWyTxM+I110=", - "zh:0f719fa5426bc883e9fa6abf7f6498e48025edafbc29015e2f5c028f1cca3b9d", - "zh:1b4d7dafefd6c61764b2f9ed6943ceb9a200dee3590d18747e3a5f6b20ce85e0", - "zh:1d23a712984866d29f7b07028a4e99c783c71f1a5dddf08bc3d4e7da9d91a1fa", - "zh:257d23d58c3bb024b6bc8eb88736eaf912e934ad47c639d0c3c742bddda849a1", - "zh:479860e1a5468f5e04013b9364c9496d7ed0804bf9a1acd8e07558d57609993d", - "zh:4cb5e681bf599b411b27c4a2c4066a5fb2ed79aaa3a1a3cb5a30002fec062ce9", - "zh:4fb35c3f643dae9f3670d719397a415f815a0b95f8ed7bd8a72f27a94ba78092", - "zh:59ba40825ab38db5b4a0989a2db0df35cc15d8984f898176011ba352f27d77b7", - "zh:61fc1252eb88088638f4c69ea4e2171cde2e5089fa632ac1e943b13787348f73", - "zh:7c5d6dd5f7cbc460e95d368be35c29b4e0402069b8912dbd5d1cd7fa9acef216", - "zh:7f76d756240d4284642f359ad470226e5378670239aadc366ef54d9d914d4d2e", - "zh:8133ad0814098177e0d067c816ccf1bf48bbadacd18f6f2c808c90447505723b", - "zh:c93be06269bb728f1968f8c50506de56c887017ac1d6e4be1f925651d8437eb6", - "zh:ef47b78a10a82e6cf53344a6a85a94041c28286c10a70541c564d762f1cfede0", - "zh:f5796a53a74999135bd9087aff50fddda59129d09b2f9b1902ff8c0c1e047e48", + "h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=", + "h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=", + "h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=", + "h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=", + "h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=", + "h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=", + "h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=", + "h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=", + "h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=", + "h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=", + "h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=", + "h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=", + "h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=", + "h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=", + "h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=", + "zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9", + "zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6", + "zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab", + "zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b", + "zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c", + "zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4", + "zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502", + "zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69", + "zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6", + "zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279", + "zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6", + "zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7", + "zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df", + "zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621", + "zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424", ] } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.15.1" - constraints = "~> 0.15" + version = "0.18.2" + constraints = "~> 0.18" hashes = [ - "h1:/2NUpbtjkc+w6n5V3kGP0rSzGjN0K2Wdfe2K+CZdmhU=", - "h1:1TOrpmCR0aT5xX1sjt70zqlYkMJnVe6r0nx2B0DS/mE=", - "h1:4uC8r8ILr+vZJ230uGqIUaWQ7uORCrIzoEuYrAq2JuE=", - "h1:6b+2osJUfwcaYZ6mathLPf/58sr/4XkLXQrcSufnkMk=", - "h1:CPldfjf79QS8mIP+GWoS0FVhrFlyjvN2+39zfyP76Ik=", - "h1:EvXICHyGIKpoYlDeKHOPzfmpvdRdhgsOMcR2nb6+tKY=", - "h1:GZLq+nDxS1CyBH0ELGTSQj9X7ozemJ1jpPA4KwbtR+c=", - "h1:ISNFqL745IQgZ6yMLy8ofV8ixbYqZYa9JKdi2W3pmNk=", - "h1:IrNrEuvFd0nYDGQefwmT8d1CSJb9e8LN5w9vw1ODp7E=", - "h1:Ms79slY9bZ94+n4cwIHvI9+/cvbucwo6S1+z5KAiznw=", - "h1:NWA9XSEBcpSkgwwIvl6tHrxGQY3uYhqNS4Vnb9RLyLQ=", - "h1:UjvxxxggicLtiE3yTe1Gx0oLUTeZpWmgIfXuHzWHn1c=", - "h1:VgJjo14DGkU4Jwo4D3GT4/5sq1tdjiZscKS5l3cb890=", - "h1:dBu3AW5YNLIvbBIMNk3wUHKw4TW+BDbj34a+mCqYhWE=", - "h1:i5oGD06nQ3JRsBIa2u4wCej+ETgp970CFl75dOKkHno=", - "h1:mTqR+vD1AWPx+mu7S0/pzBy71z7WOrjH7arOP77PXh4=", - "h1:nmTM61G8vYjpofeEqspMORpsNvTGCNZySGfjdXardL0=", - "zh:0f9bf5aaa47164a4d6ae4433d5e285a9456a5053401b2bad4ed68622f574ddee", - "zh:3039bee421fb8855a919f449fc731d145254371f5f0c39cc4660f3aeed6a8b10", - "zh:36664b08186e0c194747b18dee24ed97327c6e704133d4cf0df27abef1652f86", - "zh:3c7eae99d8c5ff65dfb99c8b9c1980147282d68971e1ef1ff0f126a6bff59d8f", - "zh:5293cc21abf54f4e5745437ca2d40d206aae323b2e1d41cf45dcc63a8868cbb3", - "zh:5994e5145e616e7e881010717e4c7def2945eb6d933f62db4ec3167732ccac84", - "zh:7994db9ed3fdb6cbf21f2154ea962cf82f04e425988bedc6657de2497d6cb6c3", + "h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=", + "h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=", + "h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=", + "h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=", + "h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=", + "h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=", + "h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=", + "h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=", + "h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=", + "h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=", + "h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=", + "h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=", + "h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=", + "h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=", + "h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=", + "h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=", + "h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=", + "zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c", + "zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178", + "zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd", + "zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef", + "zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738", + "zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc", + "zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:93aa863e536ba9376ccf9e614e9edc9b214a2ce8c4316d416a8e249b436f52d2", - "zh:ab5cb4baeda57559686a0ccf0e09158aa64624ee6ba0ef32b769f13b11a43068", - "zh:ae9388b62eede8fd9272407bf75f8241a965bd489d45ec9dd3f9fac696d500e1", - "zh:caa5befd16960e2f69c7ec483e228e5ff43ab0979c17f1b874c9ffaa1c7c0e43", - "zh:cddd3e1067defa06a4e4ad5cb3940c7943e29c42417de57236aa7d3e2aeaae13", - "zh:cdfa44d591d0805116159556947904d70f534c6816188c45cf3a7544d2722ac9", - "zh:d607e9f1f3e09f13404f219e1893e3b3c77aece4afb54e999f934c021f41f576", - "zh:d8a397aca95125c6a0c0c78d2ded5843b9204effa9f5cf7419f017b500ad9228", - "zh:f5499eaff0d221725ad209d27d87c5b46d5c554caad7dc42947c760377abe3b0", - "zh:fc5f5cf433abc83e5169fa222992ec521c0c802075970251e3dd2c1d50c4f5c1", + "zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef", + "zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3", + "zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb", + "zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e", + "zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11", + "zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7", + "zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9", + "zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1", + "zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff", + "zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339", ] } diff --git a/README.md b/README.md index 28541a1..989069e 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ For every package+tag combination: ## Patching packages Some CRAN packages fail to compile on specific platforms due to compiler- or OS-specific issues unrelated to the package itself. -The canonical example is `RcppParallel`, whose bundled TBB sources fail on musl (Alpine) and newer compiler/OS combinations. +The canonical example is `RcppParallel`, whose bundled TBB is linked in a way that lets a system TBB on the build host shadow it, so the published binary depends on a library the consumer does not have. Because such packages are often transitive dependencies of many others, a single failure cascades: all dependents fail even though nothing is wrong with the dependent itself. To address this, frequently-failing packages can be "patched" before they are installed — whether as a direct build target or a transitive dependency pulled in by `pak`. @@ -112,9 +112,9 @@ Processing updates with an existing database file takes around 5 minutes. ### Inferring System Dependencies -R package dependencies and their system dependencies are installed through {pak}. -{pak} allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. -Additionally, it automatically infers package dependencies using JSON rules from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). +R package dependencies and their system dependencies are installed through [uvr](https://github.com/nbafrank/uvr). +`uvr` allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. +Additionally, it automatically infers package dependencies using JSON rules vendored from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). Not all R packages specify required system dependencies in their DESCRIPTION file, and not all listed dependencies have existing rules in `rstudio/r-system-requirements`. For Alpine, no rules existed until recently, establishing a foundation for semi-automated package installation on Alpine Linux. @@ -152,12 +152,13 @@ A Shiny dashboard providing a search functionality of the database and grouped s Is supported by writing `Meta/archive.rds` during each package index update, listing all available archived packages. -### `pak::pak(package@version)` +### `uvr add package@version` -`pak` searches for `Archive/` and can install all versions it finds. +Clients that resolve archived versions search for `Archive/` and can install all versions they find. +This holds for `uvr` as well as for older `pak`-based clients. Ensure to use a clean cache if other repositories have been used previously. -If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. +If in doubt or when testing, call `uvr cache clean`. ## Lessons Learned @@ -180,7 +181,7 @@ If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. ## URL Composition and Platform Identifiers -Platform identifiers have been aligned with those used in to ensure proper recognition by the automatic syslib dependency installer of `pak`, specifically via the environment variable `PKG_SYSREQS_PLATFORM`: +Platform identifiers have been aligned with those used in , which `uvr` vendors and uses for its automatic syslib dependency installer (`uvr sync --install-system-deps`): - redhat-9 - redhat-8 @@ -237,8 +238,8 @@ internal error 1 in memDecompress Solution: ```sh -rm -rf /mnt/cache/R-pkgs/pak /mnt/cache/pkgcache/ /root/.cache/R/ -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +uvr cache clean +rm -rf /mnt/cache/uvr /root/.cache/R/ ``` diff --git a/build-all-versions-install-deps.yaml b/build-all-versions-install-deps.yaml index 926d550..dd02e8a 100644 --- a/build-all-versions-install-deps.yaml +++ b/build-all-versions-install-deps.yaml @@ -30,8 +30,9 @@ # from_secret: HETZNER_S3_SECRET_KEY_K3S # # normal env vars # GIT_USER: pat-s -# # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves -# # R_PKG_CACHE_DIR: /mnt/cache/pkgcache +# # set the location of uvr's caches, which persist the R package dependencies needed to install the packages themselves +# # UVR_CACHE_DIR: /mnt/cache/uvr/cache +# # UVR_PACKAGES_DIR: /mnt/cache/uvr/packages # R_LIBS_USER: /mnt/cache/R-pkgs # CCACHE_DIR: /mnt/cache/ccache # volumes: @@ -39,9 +40,8 @@ # commands: # - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # - mkdir -p /mnt/cache/R-pkgs -# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/pkgcache -# - /opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' -# - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git", dependencies = TRUE)' +# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/uvr +# - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft # - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine322', 'amd64'); saveRDS(pkgs, '/mnt/cache/pkgs_amd64.rds')" # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine323', 'arm64'); saveRDS(pkgs, '/mnt/cache/pkgs_arm64.rds')" diff --git a/cdn.tf b/cdn.tf index 8a950f5..96877b2 100644 --- a/cdn.tf +++ b/cdn.tf @@ -32,7 +32,7 @@ # cache_stale = ["offline", "updating"] # use_background_update = true - # block_ips = var.cdn_block_ips +# block_ips = var.cdn_block_ips # # 50 TB # limit_bandwidth = 50000000000000 @@ -52,6 +52,29 @@ ### cran.rpkgs.com +# The edge middleware that resolves the bare cran.rpkgs.com form to an +# / slot and routes PACKAGES* to the per-R-minor slot. The source of +# truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release. +# +# The script pre-dates this configuration, so it is adopted rather than created: +# tofu import bunnynet_compute_script.rpkgs_router 29277 +resource "bunnynet_compute_script" "rpkgs_router" { + type = "middleware" + name = "rpkgs-router" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +# Slots ("/", comma separated) whose per-minor index bincraft has +# already republished as a union of the per-minor and flat slots. Routing to a +# slot that is not listed here would hide every package the per-minor index does +# not carry, so this stays empty until a slot has been backfilled. +resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + default_value = "" + required = false +} + resource "bunnynet_pullzone" "cran_rpkgs_com" { name = "cran-rpkgs" @@ -59,12 +82,12 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { cache_expiration_time = 31919000 websockets_enabled = false - errorpage_whitelabel = true + errorpage_whitelabel = true origin { type = "OriginUrl" url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" - middleware_script = 29277 + middleware_script = bunnynet_compute_script.rpkgs_router.id } routing { @@ -82,6 +105,21 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { request_coalescing_enabled = true block_post_requests = true + # Set on the zone since before this configuration existed; declared here so + # `tofu apply` stops silently removing it. + # + # The router makes it redundant on paper: the only UA-dependent responses it + # produces are redirects, and those carry `Cache-Control: no-store`, while + # their targets are concrete per-slot, per-minor URLs whose content depends + # only on the path. Dropping it would also be a real win, because otherwise + # every distinct R version string keys its own copy of every tarball. + # + # It stays for now anyway: it is the second line of defence against the one + # failure that would be quiet and confusing (an R 4.6 client served the 4.5 + # index), and removing it is worth doing on its own once per-minor routing is + # confirmed live, not as a side effect of enabling that routing. + cache_vary_headers = ["User-Agent"] + limit_requests = 5000 limit_connections = 1000 @@ -109,6 +147,65 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } +# Alliance SwissPass historically used a separate, manually configured pull +# zone. Adopt it so both public repositories use the same B2 origin, middleware +# release and cache behavior. +import { + to = bunnynet_pullzone.cran_allianceswisspass + id = "3265648" +} + +resource "bunnynet_pullzone" "cran_allianceswisspass" { + name = "cran-allianceswisspass" + + cache_errors = false + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + + origin { + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router.id + } + + routing { + filters = [ + "scripting", + ] + } + + s3_auth_enabled = true + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" + + cache_enabled = true + request_coalescing_enabled = true + block_post_requests = true + cache_vary_headers = ["User-Agent"] + + limit_requests = 5000 + limit_connections = 1000 + + safehop_enabled = true + add_canonical_header = true + cache_stale = ["offline", "updating"] + block_ips = var.cdn_block_ips + + # 50 TB + limit_bandwidth = 50000000000000 + + block_root_path = true +} + +resource "bunnynet_pullzone_hostname" "cran_allianceswisspass" { + pullzone = bunnynet_pullzone.cran_allianceswisspass.id + name = "cran.allianceswisspass.devxy.io" + force_ssl = true + tls_enabled = true +} + # resource "bunnynet_storage_zone" "devxy-r-binaries" { # name = "devxy-r-binaries-storage" # region = "DE" diff --git a/docker/Containerfile-shiny-app b/docker/Containerfile-shiny-app index 7c7994f..4b0bd6c 100644 --- a/docker/Containerfile-shiny-app +++ b/docker/Containerfile-shiny-app @@ -1,7 +1,15 @@ FROM devxygmbh/r-alpine:4.4-3.20 AS build # ARG GITHUB_PAT -RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +RUN apk add --no-cache curl ca-certificates \ + && curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh \ + | UVR_INSTALL_DIR=/usr/local/bin sh + +# One uvr project drives both dependency installs below. UVR_LIBRARY points the +# syncs at the image's R library instead of the project-local .uvr/library/, so +# `Rscript app.R` finds the packages without a uvr-aware .Rprofile. +ENV UVR_LIBRARY=/usr/lib/R/library +RUN mkdir -p /uvr && cd /uvr && uvr init --here COPY --link ./DESCRIPTION . COPY --link ./R ./R @@ -11,13 +19,19 @@ COPY --link ./shiny/app.R /app/app.R RUN ls -la -# install R package deps -RUN R -q -e 'pak::pak()' +# install R package deps. uvr has no equivalent of bare `pak::pak()`, which +# reads the DESCRIPTION in the working directory, so extract the dependency +# names and hand them to `uvr add`. +RUN R -q --no-echo -e "d <- read.dcf('DESCRIPTION'); f <- intersect(colnames(d), c('Depends', 'Imports', 'LinkingTo')); p <- trimws(sub('[(].*', '', unlist(strsplit(paste(d[, f], collapse = ','), ',')))); writeLines(setdiff(p[nzchar(p)], c('R', rownames(installed.packages()))), '/tmp/deps.txt')" \ + && cd /uvr && xargs -r uvr add --no-install < /tmp/deps.txt \ + && uvr sync --install-system-deps RUN R CMD INSTALL --no-docs --without-keep.source . # install shiny app deps -RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); pak::pak(pkgs)" +RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); writeLines(pkgs, '/tmp/app-deps.txt')" \ + && cd /uvr && xargs -r uvr add --no-install < /tmp/app-deps.txt \ + && uvr sync --install-system-deps ENV PGPASS="" diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 5bc107a..8e61ebd 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -17,7 +17,10 @@ ARG CACHEBUST WORKDIR /work COPY build-one.R /work/build-one.R # Resolve and install the latest bincraft release dynamically (no hardcoded pin). +# uvr-install.sh lands under /work/local/ because install-bincraft.R looks for it +# there when the working directory is not a repo checkout. COPY install-bincraft.R /work/install-bincraft.R +COPY uvr-install.sh /work/local/uvr-install.sh # Ship the patch registry so build-one.R's `patches = "local/patches"` resolves # (build context is `local/`, CWD is /work). COPY patches /work/local/patches diff --git a/docker/reprex/alpine.sh b/docker/reprex/alpine.sh index 94806e5..851a9cc 100644 --- a/docker/reprex/alpine.sh +++ b/docker/reprex/alpine.sh @@ -1,10 +1,14 @@ docker run --rm -it --platform linux/arm64 alpine sh -apk add --no-cache R R-dev g++ +apk add --no-cache R R-dev g++ curl ca-certificates -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | UVR_INSTALL_DIR=/usr/local/bin sh -R -q -e 'pak::pak(c("gert", "purrr"))' +# `uvr add` always writes to .uvr/library/; only `uvr sync` honours UVR_LIBRARY, +# so add without installing and let the sync place the packages. +export UVR_LIBRARY=/usr/lib/R/library +mkdir -p /uvr && cd /uvr && uvr init --here +uvr add --no-install gert purrr && uvr sync --install-system-deps R @@ -22,14 +26,20 @@ unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE) tag <- all_tags$name package_name <- rep(package_name, length(tag)) +# uvr has no `pak::local_install_deps()`; read the DESCRIPTION of the checkout +# and `uvr add` the dependency names instead. purrr::walk2(package_name[1], tag, \(x, y) { print(y) system("git config --global advice.detachedHead false") + src <- sprintf("/tmp/%s_%s", x[1], y) system2("git", args = c( "clone", "-q", sprintf("--branch=%s", tail(y, 1)), - sprintf("https://github.com/cran/%s", x[1]), sprintf("/tmp/%s_%s", x[1], y) + sprintf("https://github.com/cran/%s", x[1]), src )) - pak::local_install_deps(sprintf("/tmp/%s_%s", x[1], y)) + d <- read.dcf(file.path(src, "DESCRIPTION")) + f <- intersect(colnames(d), c("Depends", "Imports", "LinkingTo")) + deps <- trimws(sub("[(].*", "", unlist(strsplit(paste(d[, f], collapse = ","), ",")))) + deps <- setdiff(deps[nzchar(deps)], "R") + system2("uvr", c("add", "--no-install", deps)) + system2("uvr", c("sync", "--install-system-deps")) }) - - diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts new file mode 100644 index 0000000..9493f8a --- /dev/null +++ b/edge/rpkgs-router.test.ts @@ -0,0 +1,167 @@ +/** + * Routing matrix for `edge/rpkgs-router.ts`. + * + * The script is exercised through the SDK's local server rather than by + * importing its internals, so what is tested is the artifact that gets + * deployed. Requests that the script passes through are proxied to the real + * origin, which keeps the "no redirect" cases honest: they assert that the + * client reached the flat slot, not merely that no `Location` was set. + * + * Run with `just edge-test`. + */ +import { assertEquals } from 'jsr:@std/assert@1'; + +const SCRIPT = new URL('./rpkgs-router.ts', import.meta.url).pathname; +const BASE = 'http://127.0.0.1:8080'; +const UNION_SLOTS = 'amd64/alpine324'; + +const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; +const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)'; +const UA_CURL = 'curl/8.0.1'; + +const SLOT = '/amd64/alpine324/latest/src/contrib'; +const OTHER_SLOT = '/amd64/noble/latest/src/contrib'; + +interface Probe { + status: number; + location: string | null; + cacheControl: string | null; +} + +async function probe(path: string, userAgent: string): Promise { + const res = await fetch(BASE + path, { + headers: { 'User-Agent': userAgent }, + redirect: 'manual', + }); + await res.body?.cancel(); + return { + status: res.status, + location: res.headers.get('location'), + cacheControl: res.headers.get('cache-control'), + }; +} + +/** Kill tolerantly: the child has already exited if the script failed to load. */ +async function stopServer(child: Deno.ChildProcess): Promise { + try { + child.kill(); + } catch { + // already gone + } + await child.status; +} + +async function startServer(): Promise { + const child = new Deno.Command(Deno.execPath(), { + args: ['run', '-A', SCRIPT], + env: { UNION_SLOTS }, + stdout: 'null', + stderr: 'inherit', + }).spawn(); + + for (let attempt = 0; attempt < 150; attempt++) { + try { + const res = await fetch(`${BASE}/`, { + headers: { 'User-Agent': UA_CURL }, + redirect: 'manual', + }); + await res.body?.cancel(); + return child; + } catch { + await new Promise((resolve) => setTimeout(resolve, 200)); + } + } + + await stopServer(child); + throw new Error('edge script did not start listening on ' + BASE); +} + +Deno.test('rpkgs-router', async (t) => { + const server = await startServer(); + + try { + await t.step("routes an index request to the client's R minor", async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.status, 302); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + + await t.step('routes R 4.6 to its own slot', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R46_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`); + }); + + await t.step('routes PACKAGES and PACKAGES.rds too', async () => { + for (const file of ['PACKAGES', 'PACKAGES.rds']) { + const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/${file}`, `expected ${file} to be routed`); + } + }); + + await t.step('marks the redirect uncacheable', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.cacheControl, 'no-store'); + }); + + await t.step('leaves a slot outside UNION_SLOTS alone', async () => { + const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('never routes a tarball', async () => { + const res = await probe(`${SLOT}/jsonlite_2.0.0.tar.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('serves an archived binary when it exists', async () => { + const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`; + const res = await probe(path, UA_R45_MUSL); + assertEquals(res.status, 200); + assertEquals(res.location, null); + }); + + await t.step('does not redirect a path already under a minor', async () => { + const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('leaves a client without an R version alone', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_CURL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('resolves the bare root to slot and minor', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_ALPINE); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + + await t.step('sends an unidentifiable distro to CRAN', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); + }); + + await t.step('keeps the macOS rewrite', async () => { + const res = await probe('/src/contrib/foo_1.0.tar.gz', UA_R45_DARWIN); + assertEquals(res.location, 'https://cran.rpkgs.com/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz'); + }); + + await t.step('keeps the macOS binary passthrough to CRAN', async () => { + const path = '/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz'; + const res = await probe(path, UA_R45_DARWIN); + assertEquals(res.location, `https://cran.r-project.org${path}`); + }); + + await t.step('collapses duplicate slashes before matching', async () => { + const res = await probe(`/amd64/alpine324//latest/src/contrib//PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + } finally { + await stopServer(server); + } +}); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts new file mode 100644 index 0000000..9278d4a --- /dev/null +++ b/edge/rpkgs-router.ts @@ -0,0 +1,258 @@ +/** + * Edge middleware for cran.rpkgs.com. + * + * Two jobs: + * + * 1. Resolve the bare `https://cran.rpkgs.com` form to a concrete + * `/` slot from the User-Agent, or send the client to CRAN when + * the distro cannot be identified. + * 2. Route `PACKAGES*` requests to the per-R-minor slot + * (`…/latest/src/contrib//`), so a stock `install.packages()` sees the + * packages that only exist there. + * + * Only index files are routed. Tarballs are deliberately left alone: R keeps + * the contrib URL it asked for, not the one it was redirected to, so every + * tarball URL is resolved against the flat directory and the union index steers + * the per-minor ones with a `Path: ` field. Rewriting a tarball request + * here would send flat-slot packages into a directory that does not hold them. + * + * Routing is gated on UNION_SLOTS. The raw per-minor index holds only the + * ABI-sensitive subset of a slot; it is safe to route to it only once bincraft + * has republished it as a union of the per-minor and flat slots. + * + * Deployed by OpenTofu from this file (`bunnynet_compute_script.rpkgs_router`). + * Test with `just edge-test`. + */ +import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; + +const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; +const CRAN_ORIGIN = 'https://cran.r-project.org'; +const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']); + +/** Slots ("/", comma separated) whose per-minor index is a union. */ +const UNION_SLOTS = new Set( + (Deno.env.get('UNION_SLOTS') ?? '') + .split(',') + .map((slot) => slot.trim()) + .filter((slot) => slot.length > 0), +); + +/** `///latest/src/contrib[/]` */ +const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/; + +/** A path that already sits in a per-minor slot, e.g. `4.5/PACKAGES.gz`. */ +const MINOR_DIR_REGEX = /^\d+\.\d+\//; + +/** The only files this script routes. */ +const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; + +const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; + +/** A binary archive URL whose upstream source counterpart CRAN can serve. */ +const ARCHIVE_TARBALL_REGEX = + /^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/; + +const MACOS_BIN_REGEX = + /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; + +const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i; + +const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i; +const UBUNTU_CODENAMES: Record = { + '24.04': 'noble', + '22.04': 'jammy', +}; + +const ALPINE_REGEX = /(?:Alpine Linux(?:\s+VERSION_ID=)?|alpine-)\s*(\d+)\.(\d+)/i; + +/** + * R's own User-Agent is `R (4.5.3 x86_64-pc-linux-musl …)`; the Posit-style one + * some sites configure is `R/4.5.3 R (…)`. Both carry the minor, which is why + * per-minor routing works without the distro being identifiable. + */ +const R_MINOR_REGEXES = [/\bR\/(\d+)\.(\d+)/, /\bR \((\d+)\.(\d+)/]; + +function normalizePathname(pathname: string): string { + return pathname.replace(/\/{2,}/g, '/'); +} + +function redirectTo(location: string, status = 302): Response { + return new Response(null, { + status, + headers: { + Location: location, + // The target depends on the User-Agent, so the redirect itself must + // never be cached; only its target is a cacheable, UA-independent URL. + 'Cache-Control': 'no-store', + 'X-Via': 'MyMiddleware', + 'X-Rewritten-By': 'rpkgs-edge-middleware', + }, + }); +} + +function publicCdnOrigin(url: URL): string { + return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; +} + +function extractRMinor(userAgent: string): string | null { + for (const regex of R_MINOR_REGEXES) { + const match = userAgent.match(regex); + if (match) { + return `${match[1]}.${match[2]}`; + } + } + return null; +} + +function mapArch(arch: string): string { + if (arch === 'aarch64') return 'arm64'; + if (arch === 'x86_64') return 'amd64'; + return arch; +} + +function extractArch(userAgent: string): string { + const match = userAgent.match(/(x86_64|aarch64|arm64|i386|i686)/); + return match ? mapArch(match[1]) : ''; +} + +/** + * Identify the `/` slot from the User-Agent, or null. + * + * A stock R User-Agent carries only `linux-gnu` / `linux-musl`, which are not + * slot names: returning them produced redirects into slots that do not exist + * (`/amd64/linux-musl/latest/…`, a guaranteed 404). An unidentifiable distro + * is reported as such so the caller can fall back to CRAN. + */ +function parseSlot(userAgent: string): string | null { + const arch = extractArch(userAgent); + if (!arch) { + return null; + } + + const rhel = userAgent.match(RHEL_REGEX); + if (rhel) { + return `${arch}/rhel${rhel[2]}`; + } + + const ubuntu = userAgent.match(UBUNTU_REGEX); + if (ubuntu) { + const codename = UBUNTU_CODENAMES[ubuntu[1]]; + if (codename) { + return `${arch}/${codename}`; + } + } + + const alpine = userAgent.match(ALPINE_REGEX); + if (alpine) { + return `${arch}/alpine${alpine[1]}${alpine[2]}`; + } + + return null; +} + +function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: string } | null { + const rverMatch = userAgent.match(/R \((\d+)\.(\d+)/); + const archMatch = userAgent.match(/(aarch64|arm64|x86_64)/); + const osMatch = userAgent.match(/darwin(\d+)/); + + if (!rverMatch || !archMatch || !osMatch) { + return null; + } + + const arch = archMatch[1] === 'aarch64' ? 'arm64' : archMatch[1]; + const darwinVer = parseInt(osMatch[1], 10); + const os = darwinVer >= 21 && darwinVer < 22 ? `monterey-${arch}` : `big-sur-${arch}`; + + return { os, arch, rver: `${rverMatch[1]}.${rverMatch[2]}` }; +} + +/** + * The contrib path a request should be served from, relative to the slot. + * + * Returns the per-minor path for an index file when the slot is known to carry + * a union index and the client's R minor is known; otherwise the flat path, + * which is what every client sees today. + */ +function contribPath(slot: string, rest: string, userAgent: string): string { + const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`; + + if (!INDEX_FILE_REGEX.test(rest) || !UNION_SLOTS.has(slot)) { + return flat; + } + + const rMinor = extractRMinor(userAgent); + return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; +} + +BunnySDK.net.http + .servePullZone({ url: 'https://cran.rpkgs.com/' }) + .onOriginRequest((ctx) => { + const url = new URL(ctx.request.url); + const path = normalizePathname(url.pathname); + const userAgent = ctx.request.headers.get('User-Agent') || ''; + const publicOrigin = publicCdnOrigin(url); + + // macOS clients are served from CRAN's own binary tree. + const srcContrib = path.match(SRC_CONTRIB_REGEX); + if (srcContrib && /darwin/.test(userAgent)) { + const mac = parseMacUserAgent(userAgent); + if (mac) { + return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`)); + } + } + + if (MACOS_BIN_REGEX.test(path)) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + + // Already-qualified slot URLs: what the runtime images have baked in. + const slotPath = path.match(SLOT_PATH_REGEX); + if (slotPath) { + const slot = `${slotPath[1]}/${slotPath[2]}`; + const rest = slotPath[3]; + + // Never rewrite a request that is already in a per-minor slot, or the + // redirect would chase its own tail. + if (MINOR_DIR_REGEX.test(rest)) { + return Promise.resolve(ctx.request); + } + + const target = contribPath(slot, rest, userAgent); + if (target === path) { + return Promise.resolve(ctx.request); + } + return Promise.resolve(redirectTo(`${publicOrigin}${target}`)); + } + + // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. + if (path === '/' || path === '/src/contrib' || path.startsWith('/src/contrib/')) { + const slot = parseSlot(userAgent); + if (!slot) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + + const rest = srcContrib ? srcContrib[1] : ''; + return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); + } + + return Promise.resolve(ctx.request); + }) + .onOriginResponse(async (ctx) => { + const path = normalizePathname(new URL(ctx.request.url).pathname); + const archive = path.match(ARCHIVE_TARBALL_REGEX); + + // Binary archives can be incomplete when an older build never succeeded. + // Preserve renv/remotes version restores by falling back to CRAN's source + // package only for an absent archived tarball. A requested version can be + // either archived upstream or still current, so probe the archive first. + // Other 404s remain visible. + if (ctx.response.status === 404 && archive) { + const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`; + const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' }); + const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`; + return redirectTo(sourceUrl); + } + + ctx.response.headers.append('X-Via', 'MyMiddleware'); + return Promise.resolve(ctx.response); + }); diff --git a/justfile b/justfile index a9a705e..6c75b13 100644 --- a/justfile +++ b/justfile @@ -73,3 +73,17 @@ rebuild os tag arch package *versions: --build-arg CACHEBUST="$(date +%s)" \ -f docker/build-one.Dockerfile \ local + +# run the edge middleware routing matrix (uses a local deno, else the deno image) +edge-test: + #!/usr/bin/env bash + set -euo pipefail + if command -v deno >/dev/null 2>&1; then + deno test -A edge/rpkgs-router.test.ts + else + docker run --rm \ + -v "$PWD:/w" -w /w \ + -v deno-cache:/deno-dir \ + denoland/deno:latest \ + deno test -A edge/rpkgs-router.test.ts + fi diff --git a/local/build-all.R b/local/build-all.R index 1df1f83..37fc593 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -27,7 +27,9 @@ package_cache_files <- c( "/mnt/cache/packages/s3_cache.rds" ) if (!all(file.exists(package_cache_files))) { - message("Package snapshot missing from cache; recomputing via packages-to-build.R") + message( + "Package snapshot missing from cache; recomputing via packages-to-build.R" + ) dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { tmp <- paste0(path, ".tmp.", Sys.getpid()) @@ -36,7 +38,10 @@ if (!all(file.exists(package_cache_files))) { } source(file.path("local", "packages-to-build.R")) save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds") - save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds") + save_rds_atomic( + pkgs[r_minor_sensitive == TRUE], + "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" + ) message("Package snapshot recomputed.") } @@ -112,20 +117,22 @@ built <- DBI::dbGetQuery( ) DBI::dbDisconnect(con) before <- nrow(chunk) -chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] -sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) +chunk <- chunk[ + !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), +] +sprintf( + "Skipped %d already-attempted package versions; %d remaining for this job", + before - nrow(chunk), + nrow(chunk) +) # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, -# saving significant memory for pak subprocess forks +# saving significant memory for the dependency-installer subprocesses s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") sprintf("S3 cache: %s files", length(s3_cache)) n <- nrow(chunk) -# Every `trim_every` packages, bound the pkgcache _metadata dir so a full-platform -# run does not accumulate thousands of ~70 MB snapshots and fill the host disk. -# No-op on amd64 (R_PKG_CACHE_DIR is empty / cache not persisted). -trim_every <- 25L mapply( function(pkg, ver, sens, i) { cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) @@ -151,12 +158,6 @@ mapply( upload = TRUE, store_build_metadata = TRUE ) - if (i %% trim_every == 0L) { - removed <- trim_pkgcache_metadata() - if (removed > 0L) { - cat(sprintf(" [pkgcache trim] removed %d stale _metadata entries\n", removed)) - } - } }, chunk$Package, chunk$Version, diff --git a/local/dedupe-audit-issue.R b/local/dedupe-audit-issue.R new file mode 100644 index 0000000..c78015a --- /dev/null +++ b/local/dedupe-audit-issue.R @@ -0,0 +1,150 @@ +# One-off maintenance: collapse the duplicate arch subsections that accumulated +# in the "Missing package binaries for latest version ()" issues. +# +# A bug in weekly-missing-binaries-audit.R matched the existing "### " +# subsection by its bare header while writing headers with a +# " (N missing, M to rebuild)" suffix, so every audit run appended a fresh block +# instead of replacing it. This script rewrites each "## " section to +# keep only the *last* (freshest) block per arch. The audit fix prevents further +# accumulation; this cleans up what is already there. +# +# Env: FORGEJO_TOKEN (required). DRY_RUN=1 to preview counts without patching. + +library(httr2, quietly = TRUE) + +forgejo_base <- "https://git.devxy.io/api/v1" +repo <- "devxy/build-cran-binaries" +token <- Sys.getenv("FORGEJO_TOKEN") +dry_run <- nchar(Sys.getenv("DRY_RUN")) > 0 + +if (nchar(token) == 0) { + stop("FORGEJO_TOKEN env var is not set") +} + +issue_titles <- c( + "Missing package binaries for latest version (Alpine)", + "Missing package binaries for latest version (Ubuntu)", + "Missing package binaries for latest version (Red Hat)" +) + +# Collapse one "## " block: keep only the last block per arch, +# emitted in order of first appearance. `pl[1]` is the "## " header. +dedupe_platform <- function(pl) { + sub_hdr <- which(grepl("^### ", pl)) + if (length(sub_hdr) == 0) { + return(pl) + } + preamble <- pl[seq_len(sub_hdr[1] - 1)] + sub_end <- c(sub_hdr[-1] - 1, length(pl)) + blocks <- lapply(seq_along(sub_hdr), function(k) { + pl[seq(sub_hdr[k], sub_end[k])] + }) + arches <- vapply( + blocks, + function(b) sub("^### (\\S+).*", "\\1", b[1]), + character(1) + ) + # Index of the last block for each arch, kept in first-appearance order. + last_idx <- vapply( + unique(arches), + function(a) max(which(arches == a)), + integer(1) + ) + keep <- sort(last_idx) + out <- preamble + for (i in keep) { + out <- c(out, blocks[[i]]) + } + out +} + +process_issue <- function(title) { + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(title, reserved = TRUE) + ) + resp <- request(search_url) |> + req_headers(Authorization = paste("token", token)) |> + req_perform() + issues <- resp_body_json(resp, simplifyVector = FALSE) + match_idx <- which(vapply(issues, function(x) x$title, character(1)) == title) + if (length(match_idx) == 0) { + cat(sprintf("[skip] No issue found: %s\n", title)) + return(invisible()) + } + + issue_number <- issues[[match_idx[1]]]$number + body <- issues[[match_idx[1]]]$body + if (is.null(body) || nchar(body) == 0) { + cat(sprintf("[skip] Empty body: #%d %s\n", issue_number, title)) + return(invisible()) + } + + lines <- strsplit(body, "\n", fixed = TRUE)[[1]] + before <- sum(grepl("^### ", lines)) + + # Split off the "## Excluded packages" footer so it is preserved verbatim. + excl_idx <- which(lines == "## Excluded packages") + footer <- character(0) + if (length(excl_idx) > 0) { + pre_dash <- which(lines == "---" & seq_along(lines) < excl_idx[1]) + cut <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_idx[1] + footer <- lines[seq(cut, length(lines))] + lines <- lines[seq_len(cut - 1)] + } + + # Platform headers ("## "); everything before the first is preamble. + plat_idx <- which(grepl("^## ", lines)) + if (length(plat_idx) == 0) { + cat(sprintf("[skip] No platform sections: #%d %s\n", issue_number, title)) + return(invisible()) + } + top <- lines[seq_len(plat_idx[1] - 1)] + plat_end <- c(plat_idx[-1] - 1, length(lines)) + + new_lines <- top + for (j in seq_along(plat_idx)) { + pl <- lines[seq(plat_idx[j], plat_end[j])] + new_lines <- c(new_lines, dedupe_platform(pl)) + } + if (length(footer) > 0) { + new_lines <- c(new_lines, footer) + } + + after <- sum(grepl("^### ", new_lines)) + cat(sprintf( + "#%d %s: %d -> %d arch subsections%s\n", + issue_number, + title, + before, + after, + if (dry_run) " (dry run, not patched)" else "" + )) + + if (dry_run) { + return(invisible()) + } + + patch_url <- sprintf( + "%s/repos/%s/issues/%d", + forgejo_base, + repo, + issue_number + ) + request(patch_url) |> + req_headers( + Authorization = paste("token", token), + `Content-Type` = "application/json" + ) |> + req_body_json(list(body = paste(new_lines, collapse = "\n"))) |> + req_method("PATCH") |> + req_perform() + cat(sprintf(" patched #%d\n", issue_number)) +} + +for (t in issue_titles) { + process_issue(t) +} +cat("Done.\n") diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index 4ce30cf..73922b3 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -155,6 +155,44 @@ fingerprint_error <- function(error_text, package = NULL, max_chars = 200L) { fp } +# --------------------------------------------------------------------------- +# Dependency-cascade detection +# --------------------------------------------------------------------------- +# If a build's error_text shows the failure was actually in a DIFFERENT package +# (a dependency that would not compile), return that dependency's name; +# otherwise NA. Used to avoid proposing a per-package fix for a package that +# only fails because a shared dependency does not build (e.g. the ~73 Stan +# packages that fail while building `rstan`). Generalises the RcppParallel +# `applies_to` guard to any dependency named in the log. +failing_dependency <- function(error_text, package) { + if (length(error_text) == 0L || is.na(error_text) || !nzchar(error_text)) { + return(NA_character_) + } + x <- as.character(error_text) + # optional opening quote before the package name: apostrophe, double-quote, + # backtick, or curly quotes -- written as \u escapes so the pattern stays + # valid UTF-8 regardless of source encoding. + q <- "[\u0027\u0022\u0060\u2018\u2019]?" + name <- "([A-Za-z][A-Za-z0-9._]+)" + # Markers R/pak emit naming the package that actually failed to compile. + pats <- c( + paste0("compilation failed for package ", q, name), + paste0("Failed to build source package ", q, name), + paste0("Error in building package ", q, name), + paste0("dependenc(?:y|ies) ", q, name, q, "?[^\\n]*not available") + ) + deps <- character(0L) + for (p in pats) { + hits <- regmatches(x, gregexpr(p, x, perl = TRUE))[[1L]] + if (length(hits) > 0L) { + deps <- c(deps, sub(p, "\\1", hits, perl = TRUE)) + } + } + deps <- sub("[._]+$", "", deps) # drop a trailing sentence period (e.g. "rstan.") + deps <- setdiff(unique(deps), package) # a package failing on its OWN code is not a cascade + if (length(deps) == 0L) NA_character_ else deps[[1L]] +} + # --------------------------------------------------------------------------- # Classification # --------------------------------------------------------------------------- @@ -291,18 +329,38 @@ build_triage_report <- function( unregistered <- setdiff(pkgs, registered_pkgs) auto_proposable <- isTRUE(sig$auto) && sig$matched - # A signature whose fix is package-specific (`applies_to`) may only be - # proposed for that package. Everything else matching it is a downstream - # failure blocked on that package (e.g. RcppParallel dependents carrying - # RcppParallel's own error text) -- never propose those a bogus entry. + # Decide, per package, whether it is genuinely fixable or merely blocked on + # a dependency (so a per-package entry would be useless). Two blocking modes: + # 1. `applies_to`: a package-specific patch (e.g. RcppParallel's) is only + # valid for its own package; other matches are downstream of it. + # 2. data-driven cascade: the package's error_text shows a *different* + # package failed to compile (e.g. the ~73 Stan packages blocked on rstan). + # `blocked_map` maps a blocked package -> the dependency it waits on. + g$blocked_dep <- vapply( + seq_len(nrow(g)), + function(i) failing_dependency(g$error_text[[i]], g$name[[i]]), + character(1L) + ) proposable <- unregistered - blocked_on <- NULL + blocked_map <- list() if (!is.null(sig$applies_to)) { - proposable <- intersect(unregistered, sig$applies_to) - downstream <- setdiff(pkgs, sig$applies_to) - if (length(downstream) > 0L) { - blocked_on <- sig$applies_to + for (p in setdiff(pkgs, sig$applies_to)) { + blocked_map[[p]] <- sig$applies_to } + proposable <- intersect(proposable, sig$applies_to) + } + for (p in proposable) { + deps <- g$blocked_dep[g$name == p] + if (!any(is.na(deps))) { + # every failing build of p is a cascade -> blocked, not fixable here + blocked_map[[p]] <- unique(deps) + } + } + proposable <- setdiff(proposable, names(blocked_map)) + blocked_packages <- names(blocked_map) + blocked_on <- unique(unlist(blocked_map, use.names = FALSE)) + if (length(blocked_on) == 0L) { + blocked_on <- NULL } proposed <- list() @@ -326,6 +384,8 @@ build_triage_report <- function( suggested_fix = sig$fix, applies_to = sig$applies_to, blocked_on = blocked_on, + blocked_packages = blocked_packages, + blocked_map = if (length(blocked_map) > 0L) blocked_map else NULL, fingerprint = names(fp_tab)[[1L]], fingerprint_variants = length(fp_tab), build_count = nrow(g), diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R index 51f5bc7..ce7b683 100644 --- a/local/failing-builds-report.R +++ b/local/failing-builds-report.R @@ -194,15 +194,17 @@ for (r in report) { ) cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "") } - } else if (!is.null(r$blocked_on)) { + } else if (r$auto_proposable && length(r$blocked_packages) == 0L) { + cat(" (all affected packages already have a registry entry)\n") + } + # Blocked packages are shown even when the group also has proposals. + if (length(r$blocked_packages) > 0L) { cat(sprintf( - " (blocked on %s -- these %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", + " (blocked on %s -- %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", toString(r$blocked_on), - length(r$packages), + length(r$blocked_packages), toString(r$blocked_on) )) - } else if (r$auto_proposable) { - cat(" (all affected packages already have a registry entry)\n") } } diff --git a/local/fetch-rebuild-packages-from-issue.R b/local/fetch-rebuild-packages-from-issue.R index 26e544d..9d8043c 100644 --- a/local/fetch-rebuild-packages-from-issue.R +++ b/local/fetch-rebuild-packages-from-issue.R @@ -4,7 +4,6 @@ forgejo_base <- "https://git.devxy.io/api/v1" repo <- "devxy/build-cran-binaries" platform <- Sys.getenv("PLATFORM") arch <- Sys.getenv("ARCH") -token <- Sys.getenv("FORGEJO_TOKEN") output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") if (nchar(platform) == 0) { @@ -13,6 +12,36 @@ if (nchar(platform) == 0) { if (nchar(arch) == 0) { stop("ARCH env var is not set") } + +# Prefer the audit's freshly-written RDS. The audit overwrites it each run +# (saveRDS), so unlike the Forgejo issue body it is never subject to the +# duplicate-subsection accumulation bug. Fall back to parsing the issue when the +# RDS is absent (e.g. a fresh runner with no shared cache). +rds_file <- file.path( + Sys.getenv("REBUILD_PKG_RDS_DIR", "/mnt/cache/packages"), + sprintf("weekly_rebuild_%s_%s.rds", platform, arch) +) +if (file.exists(rds_file)) { + pkgs <- tryCatch(as.character(readRDS(rds_file)), error = function(e) NULL) + if (!is.null(pkgs) && length(pkgs) > 0) { + cat(sprintf( + "Using audit RDS %s: %d rebuildable packages for %s/%s\n", + rds_file, + length(pkgs), + platform, + arch + )) + writeLines(pkgs, output_file) + cat(sprintf("Wrote package list to %s\n", output_file)) + q("no") + } + cat(sprintf( + "RDS %s present but empty/unreadable -- falling back to issue\n", + rds_file + )) +} + +token <- Sys.getenv("FORGEJO_TOKEN") if (nchar(token) == 0) { stop("FORGEJO_TOKEN env var is not set") } diff --git a/local/install-bincraft.R b/local/install-bincraft.R index 4665b3b..e39cb4f 100644 --- a/local/install-bincraft.R +++ b/local/install-bincraft.R @@ -11,10 +11,11 @@ # # How it works: list the remote tags with `git ls-remote` (no token needed for # the public repo), keep the `vX.Y.Z` release tags, pick the highest version, -# and install it with pak. pak is idempotent on the git ref, so re-running keeps -# the package when it is already current and only updates when a newer tag ships. -# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is -# identical across git versions and `system2()` argument handling. +# and install it with uvr via `local/uvr-install.sh`. uvr is idempotent on the +# git ref, so re-running keeps the package when it is already current and only +# updates when a newer tag ships. Filtering/sorting is done in R (not via git's +# `--sort`/refspec) so behaviour is identical across git versions and +# `system2()` argument handling. repo_url <- Sys.getenv( "BINCRAFT_GIT_URL", @@ -44,7 +45,43 @@ latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][ ] message(sprintf("Installing latest bincraft release: %s", latest)) -pak::pak(sprintf("git::%s@%s", repo_url, latest)) + +# uvr addresses Forgejo repos as `forgejo::host/owner/repo@ref` rather than as a +# git URL, so drop the scheme and the trailing `.git` from `repo_url`. +spec <- sprintf( + "forgejo::%s@%s", + sub("\\.git$", "", sub("^[a-z]+://", "", repo_url)), + latest +) + +# `local/uvr-install.sh` when run from the repo root, `/work/local/` in the +# build-one image, which copies the two scripts into a flatter layout. +helper <- Sys.getenv("UVR_INSTALL_SH", unset = "") +if (!nzchar(helper)) { + candidates <- c("local/uvr-install.sh", "/work/local/uvr-install.sh") + found <- candidates[file.exists(candidates)] + if (length(found) == 0L) { + stop("Could not locate uvr-install.sh; set UVR_INSTALL_SH", call. = FALSE) + } + helper <- found[1L] +} + +# Point uvr at the R running this script and at the library it would install +# into, so the per-R-minor passes in the build pipelines (which call a different +# Rscript with R_LIBS_USER pointed elsewhere) target their own R and library. +Sys.setenv( + UVR_R_BIN = file.path(R.home("bin"), "R"), + UVR_TARGET_LIB = .libPaths()[1L] +) + +status <- system2(helper, shQuote(spec)) +if (!identical(status, 0L)) { + stop( + sprintf("uvr failed to install %s (exit %s)", spec, status), + call. = FALSE + ) +} + message(sprintf( "bincraft %s installed (%s)", as.character(utils::packageVersion("bincraft")), diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 7a3652d..3b508fb 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -15,15 +15,15 @@ suppressPackageStartupMessages(library(data.table)) # Sys.setenv("OS_VERSION" = "3.22") # Sys.setenv("ARCH" = "arm64") -arch = Sys.getenv("ARCH") +arch <- Sys.getenv("ARCH") # target: alpine-322, ubuntu-2404, redhat-9, etc. -platform = paste( +platform <- paste( Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-" ) # Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10") -codename = bincraft::set_codename(NULL) +codename <- bincraft::set_codename(NULL) con <- DBI::dbConnect( RPostgres::Postgres(), @@ -35,8 +35,8 @@ con <- DBI::dbConnect( sslmode = "require" ) -cran_archive = tools::CRAN_archive_db() -cran_release = tools::CRAN_package_db() +cran_archive <- tools::CRAN_archive_db() +cran_release <- tools::CRAN_package_db() # Subset cran_archive to only those packages cran_archive_in_release <- cran_archive[ names(cran_archive) %in% cran_release$Package @@ -84,18 +84,66 @@ s3fs::s3_file_system( region_name = "eu-central-003", refresh = TRUE ) -s3_pkgs = s3fs::s3_dir_ls( +s3_pkgs <- s3fs::s3_dir_ls( sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename), recurse = TRUE ) -# Save the raw S3 file listing for the build step to use as s3_package_cache -# This avoids loading s3fs/reticulate in the build container, saving memory for pak forks -saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds") - file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) + +# An object occupying a key is not proof a binary was built: a package whose +# build failed has its CRAN source published under exactly that name. Left in +# the cache, `build_binary_package()` reads it as "already built" and skips the +# package forever, which is how alpine324 accumulated ~13.5k source tarballs. +# +# bincraft stamps `Built` only on records it actually built, so the slot's own +# index distinguishes them. A slot last indexed by a bincraft that predates that +# fix stamps `Built` on everything, so the cache is then unchanged from before. +# Archived objects have no index record and are kept: unknown means binary, +# never "rebuild it". +index_url <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", + arch, + codename +) +source_served <- tryCatch( + { + con_idx <- gzcon(url(index_url, open = "rb")) + on.exit(close(con_idx), add = TRUE) + idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) + sprintf( + "%s_%s.tar.gz", + idx[is.na(idx[, "Built"]), "Package"], + idx[is.na(idx[, "Built"]), "Version"] + ) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read %s (%s); keeping the full S3 cache\n", + index_url, + conditionMessage(e) + )) + character(0) + } +) + +binary_cache <- setdiff(file_names, source_served) +cat(sprintf( + "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", + length(file_names), + length(file_names) - length(binary_cache), + length(binary_cache) +)) + +# Save the S3 file listing for the build step to use as s3_package_cache. +# This avoids loading s3fs/reticulate in the build container, saving memory for +# the dependency-installer subprocesses +saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") +# Built from the filtered listing, not the raw one: `s3_dt` is subtracted from +# the build list below, so a source fallback left in here would exclude the very +# package that needs building. +matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", binary_cache) +parts <- regmatches(binary_cache, matches) parts <- parts[sapply(parts, length) == 3] s3_dt <- data.table( Package = sapply(parts, `[`, 2), diff --git a/local/patches/README.md b/local/patches/README.md index 992395f..ce60c9a 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -81,6 +81,20 @@ Run the trial build inside the failing platform's build-env image; it uploads/ar Rscript local/trial-build-patch.R ``` +#### Autonomous PR + trial-build gate + +`--open-pr` closes the loop: it writes the top-N candidates (by failure volume) onto the reused `auto/registry-patch-proposals` branch and opens/updates a single PR. +`.crow/auto-apply-patches.yaml` runs this on a cron. +`FORGEJO_TOKEN` is used for both the branch push and the PR (no separate write-scoped secret). + +```bash +# Bounded batch; opens/updates one PR. +PGPASS=... FORGEJO_TOKEN=... Rscript local/propose-patches.R --open-pr --limit 10 +``` + +The merge gate is `.crow/trial-build-registry.yaml`: matrixed over the build-env images, each platform trial-builds only the entries the branch **adds** (`local/trial-build-registry.R`, which diffs the registry against `main`) and is green only if every new entry builds. +The repo uses no `pull_request` triggers, so this gate runs manually or on a cron against the auto-patch branch (`--var patch_branch=...`); wiring it to fire on the PR needs `event: pull_request` enabled on the forge. + `--write` and `--open-issue` also append to `local/patches/proposals-log.json`, a ledger of what was proposed. ### Feedback loop (step 4) diff --git a/local/patches/RcppParallel/disable-tbb.patch b/local/patches/RcppParallel/disable-tbb.patch deleted file mode 100644 index 0fe8ad9..0000000 --- a/local/patches/RcppParallel/disable-tbb.patch +++ /dev/null @@ -1,16 +0,0 @@ -diff --git a/src/Makevars.in b/src/Makevars.in -index be8445f..faee771 100644 ---- a/src/Makevars.in -+++ b/src/Makevars.in -@@ -60,7 +60,10 @@ else - endif - - ifeq ($(UNAME), Linux) -- USE_TBB=Linux -+ # bincraft patch: the bundled Intel TBB build hangs/fails on musl (Alpine) -+ # and newer toolchains (g++ 15). Skip it (leave USE_TBB unset) and force the -+ # TinyThread backend so RcppParallel still builds. -+ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=0 - endif - - ifeq ($(UNAME), SunOS) diff --git a/local/patches/RcppParallel/force-bundled-tbb.patch b/local/patches/RcppParallel/force-bundled-tbb.patch new file mode 100644 index 0000000..c46a05e --- /dev/null +++ b/local/patches/RcppParallel/force-bundled-tbb.patch @@ -0,0 +1,154 @@ +diff --git a/R/aaa.R b/R/aaa.R +index 568a2aa..bfdab4c 100644 +--- a/R/aaa.R ++++ b/R/aaa.R +@@ -5,4 +5,21 @@ TBB_LIB <- "" + TBB_INC <- "" + + TBB_NAME <- "tbb" +-TBB_MALLOC_NAME <- "tbbmalloc" +\ No newline at end of file ++TBB_MALLOC_NAME <- "tbbmalloc" ++ ++# bincraft patch: our build images (and plenty of user environments) export ++# TBB_ROOT / TBB_LIB / TBB_INC. A binary we publish always carries its own ++# oneTBB in RcppParallel/lib -- see the companion changes in ++# tools/config/configure.R and src/install.libs.R -- so honouring those ++# variables at run time is actively harmful: .onLoad() would dyn.load() a ++# second, unrelated TBB into the process next to the bundled one (two copies ++# of the same symbols in the global scope, which segfaults R on load), and ++# RcppParallelLibs() / CxxFlags() would hand that system TBB to dependents ++# such as rstan, putting NEEDED libtbb.so.12 back into their binaries. Read ++# the variables only when explicitly opted back in. ++bincraftGetenv <- function(name, unset = "") { ++ if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") ++ Sys.getenv(name, unset = unset) ++ else ++ unset ++} +\ No newline at end of file +diff --git a/R/tbb.R b/R/tbb.R +index 6f6a745..e407986 100644 +--- a/R/tbb.R ++++ b/R/tbb.R +@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { + sysname <- Sys.info()[["sysname"]] + + # find root for TBB install +- tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) ++ tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) + if (is.null(name)) + return(tbbRoot) + +@@ -58,7 +58,7 @@ tbbCxxFlags <- function() { + flags <- c("-DRCPP_PARALLEL_USE_TBB=1") + + # if TBB_INC is set, apply those library paths +- tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) ++ tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) + if (!file.exists(tbbInc)) { + tbbInc <- system.file("include", package = "RcppParallel") + } +@@ -117,7 +117,7 @@ tbbLdFlags <- function() { + } + + # shortcut if TBB_LIB defined +- tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) ++ tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) + if (nzchar(tbbLib)) { + if (R.version$os == "emscripten") { + fmt <- "-L%1$s -l%2$s" +diff --git a/src/install.libs.R b/src/install.libs.R +index 3b3cfda..c0e6f3e 100644 +--- a/src/install.libs.R ++++ b/src/install.libs.R +@@ -477,6 +477,18 @@ prependFlags <- function(prependFlags, toFlags) { + tbbLib <- Sys.getenv("TBB_LIB") + tbbInc <- Sys.getenv("TBB_INC") + ++# bincraft patch: the companion change in tools/config/configure.R stops an ++# ambient TBB_LIB / TBB_INC from selecting a system TBB, but this script is ++# also run directly by `R CMD INSTALL` (not only through the `tbb` rule in ++# src/Makevars, which passes the configured values), so at install time it ++# still sees the image's environment and would symlink the system libraries ++# into RcppParallel/lib. Drop them here for the same reason, under the same ++# opt-out. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbLib <- "" ++ tbbInc <- "" ++} ++ + args <- commandArgs(trailingOnly = TRUE) + if (identical(args, "build")) { + if (nzchar(tbbLib) && nzchar(tbbInc)) { +diff --git a/tools/config/configure.R b/tools/config/configure.R +index 6293fe1..eae4aaa 100644 +--- a/tools/config/configure.R ++++ b/tools/config/configure.R +@@ -40,6 +40,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) + tbbLib <- Sys.getenv("TBB_LIB", unset = NA) + tbbInc <- Sys.getenv("TBB_INC", unset = NA) + ++# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Several of ++# our build images export these (a leftover from RcppParallel 5.x, whose ++# bundled Intel TBB would not build on musl or with modern g++), and any of ++# them switches the branches below to a system TBB. The published binary then ++# records NEEDED libtbb.so.12 (or libtbb.so.2 for the classic Intel TBB) and ++# gets a RcppParallel/lib full of absolute symlinks into the image's library ++# dir, so it cannot dyn.load on a consumer machine without that exact TBB. ++# 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, ++# so the bundled copy is always the right choice here; forcing it in the ++# package rather than relying on the image environment keeps the binary ++# correct whichever image version CI happens to pull. Set ++# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbRoot <- NA ++ tbbLib <- NA ++ tbbInc <- NA ++} ++ + tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") + tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") + +@@ -186,12 +204,23 @@ define( + ) + + # set PKG_LIBS ++# ++# bincraft patch: the library directories below are passed as plain '-L', not ++# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) ++# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', ++# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build ++# host with a distro TBB installed, '-ltbb' resolves to that library and ++# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the ++# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads ++# the system TBB rather than the copy shipped in RcppParallel/lib, and fails ++# outright on a machine that has no system TBB. gcc places a plain '-L' before ++# its built-in dirs, so the intended library is found first. + pkgLibs <- if (!is.na(tbbLib)) { + + # a TBB supplied via TBB_LIB / TBB_ROOT. an rpath is meaningless on Windows, + # where the loader has no equivalent -- see R/zzz.R for how we resolve there + c( +- "-Wl,-L\"$(TBB_LIB)\"", ++ "-L\"$(TBB_LIB)\"", + if (.Platform$OS.type != "windows") + sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), + "-l$(TBB_NAME)", +@@ -201,14 +230,14 @@ pkgLibs <- if (!is.na(tbbLib)) { + } else if (R.version$os == "emscripten") { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)" + ) + + } else { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" + ) diff --git a/local/patches/registry.json b/local/patches/registry.json index e53f1c9..a7e29dd 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -1,13 +1,13 @@ [ { "package": "RcppParallel", - "versions": "*", - "platforms": ["alpine", "ubuntu-2604"], + "versions": ">=6.0.0", + "platforms": ["*"], "env": {}, "configure_args": [], "makevars": {}, - "patch": "RcppParallel/disable-tbb.patch", - "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); patch unsets USE_TBB and forces -DRCPP_PARALLEL_USE_TBB=0 so RcppParallel skips the bundled build and uses the TinyThread backend" + "patch": "RcppParallel/force-bundled-tbb.patch", + "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, so the system-TBB workaround needed for 5.x is gone, but two things still steer the build back to a system TBB. (1) Ambient TBB_ROOT/TBB_LIB/TBB_INC: several build images still export these (build-env-images dropped them, but the images are rebuilt only by cron/manual runs, so a stale image keeps them), and R reads ~/.Renviron *after* the process environment, so no pipeline-side env override can undo it. configure.R then takes the system-TBB branch, install.libs.R symlinks the image's libraries into RcppParallel/lib as absolute paths, and the binary records NEEDED libtbb.so.12 (libtbb.so.2 for the classic Intel TBB on el8/el9) -- it cannot dyn.load on a consumer machine without that exact TBB. Both files are patched to ignore those variables (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE); install.libs.R needs it separately because R CMD INSTALL runs it outside the src/Makevars rule that passes the configured values. (2) Link order: the bundled branch passes its build dir as '-Wl,-Ltbb/build/lib_release', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,', so '-ltbb' would still resolve to a distro TBB when one is installed; a plain '-L' puts the bundled dir first. Verified on the current build-env-ubuntu:jammy image (which still exports TBB_INC/TBB_LIB): NEEDED libtbb.so, RUNPATH $ORIGIN/../lib, real libtbb.so.2 in RcppParallel/lib, and the package loads with every system libtbb moved away." }, { "package": "fs", @@ -18,17 +18,5 @@ "makevars": {}, "patch": "fs/force-vendored-libuv.patch", "reason": "fs 2.x configure links system libuv whenever pkg-config finds libuv-devel (installed as a build-time sysreq), producing an fs.so with NEEDED libuv.so.1. That binary fails to dyn.load on consumer machines lacking runtime libuv, because install.packages/renv do not install SystemRequirements (only pak does, and only in the build container). The patch short-circuits configure to copy src/Makevars.vendor and build the bundled static libuv (needs cmake) so the binary is self-contained on every platform. An env/pkg-config override was tried first but the rebuilt binary still linked libuv.so.1, so a source patch is used instead." - }, - { - "package": "rstan", - "versions": "*", - "platforms": ["*"], - "env": {}, - "configure_args": [], - "makevars": { - "CPPFLAGS": "-DTBB_INTERFACE_NEW -I/usr/local/include" - }, - "patch": null, - "reason": "StanHeaders' init_threadpool_tbb.hpp unconditionally includes the legacy (removed in oneTBB 2021+) for version detection, breaking compilation of Module.cpp against the bundled oneTBB. Pre-defining TBB_INTERFACE_NEW skips that include and selects the modern tbb/global_control.h + tbb/task_arena.h path that the bundled TBB provides (-I/usr/local/include preserves the default CPPFLAGS the override replaces)" } ] diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 5efc87c..28f65db 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -160,19 +160,71 @@ unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) { ) } -# Groups blocked on a dependency (a package-specific fix pinned via `applies_to` -# whose dependents merely carry its error): report the dependency + how many -# dependents wait on it, so fixing it once is recognised as clearing the batch. -blocked_summary <- function(report, max_pkgs = 15L) { - bl <- Filter(function(g) !is.null(g$blocked_on), report) - lapply(bl, function(g) { +# Aggregate blocked packages across ALL groups by the dependency they wait on, +# so one dependency (RcppParallel, rstan, sf, ...) is a single line -- deduped +# and ranked by how many distinct dependents it blocks -- instead of repeating +# once per fingerprint group. Reads each group's `blocked_map` (package -> the +# dependency it is blocked on). Returns records sorted by dependent count desc, +# each with up to `max_pkgs` example dependents. +blocked_by_dependency <- function(report, max_pkgs = 15L) { + acc <- list() # dependency -> character vector of dependent packages + for (g in report) { + bm <- g$blocked_map + if (is.null(bm) || length(bm) == 0L) { + next + } + for (pkg in names(bm)) { + for (dep in as.character(unlist(bm[[pkg]]))) { + acc[[dep]] <- unique(c(acc[[dep]], pkg)) + } + } + } + if (length(acc) == 0L) { + return(list()) + } + out <- lapply(names(acc), function(dep) { + pkgs <- acc[[dep]] list( - blocked_on = g$blocked_on, - n_packages = length(g$packages), - packages = utils::head(g$packages, max_pkgs), - packages_truncated = length(g$packages) > max_pkgs + dependency = dep, + n_packages = length(pkgs), + packages = utils::head(pkgs, max_pkgs), + packages_truncated = length(pkgs) > max_pkgs ) }) + out[order(-vapply(out, function(x) x$n_packages, integer(1L)))] +} + +# Does a registry entry's `platforms` apply to a build on `os` (e.g. +# "ubuntu-2604")? Mirrors bincraft's token match: an entry applies if any of its +# platform tokens is "*", the OS codename, or the distro family ("ubuntu"). +entry_applies_to_os <- function(entry_platforms, os) { + toks <- as.character(unlist(entry_platforms)) + family <- sub("-.*$", "", os) # ubuntu-2604 -> ubuntu + any(toks %in% c("*", os, family)) +} + +# Registry entries present in `current` but not in `base` (matched on +# package|platforms|versions), optionally restricted to those that apply to a +# given `os`. Used by the trial-build gate to build only the entries a PR adds. +new_registry_packages <- function(current, base, os = NULL) { + key <- function(e) { + sprintf( + "%s|%s|%s", + e$package %||% "?", + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + e$versions %||% "?" + ) + } + base_keys <- vapply(base %||% list(), key, character(1L)) + added <- Filter(function(e) !(key(e) %in% base_keys), current %||% list()) + if (!is.null(os)) { + added <- Filter(function(e) entry_applies_to_os(e$platforms, os), added) + } + unique(vapply( + added, + function(e) as.character(e$package %||% ""), + character(1L) + )) } `%||%` <- function(a, b) if (is.null(a)) b else a diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R index 4d82a11..f8e1284 100644 --- a/local/proposal-tracking.R +++ b/local/proposal-tracking.R @@ -127,13 +127,13 @@ if (length(retire) > 0L) { # --------------------------------------------------------------------------- # Blind spots: failures the classifier could not auto-propose. # --------------------------------------------------------------------------- -blocked <- blocked_summary(report) +blocked <- blocked_by_dependency(report) unmatched <- unclassified_summary(report) cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") if (length(blocked) > 0L) { for (b in blocked) { - cat(sprintf(" %s: %d dependent(s) waiting\n", b$blocked_on, b$n_packages)) + cat(sprintf(" %-20s %5d dependent(s)\n", b$dependency, b$n_packages)) } } else { cat(" (none)\n") @@ -207,8 +207,8 @@ if (do_issue) { body_lines <- c( body_lines, sprintf( - "- **%s**: %d dependent(s) waiting (e.g. %s%s)", - b$blocked_on, + "- **%s**: %d dependent(s) (e.g. %s%s)", + b$dependency, b$n_packages, toString(b$packages), if (isTRUE(b$packages_truncated)) ", ..." else "" diff --git a/local/propose-patches.R b/local/propose-patches.R index 7ff195b..360df9f 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -15,10 +15,17 @@ # # Usage: # PGPASS=... Rscript local/propose-patches.R [--platform P] [--arch A] [--min N] +# [--limit N] # (default) print candidates + validation, take no action # --write append candidates to local/patches/registry.json and # the proposals ledger (commit + open a PR yourself) # --open-issue post/update a Forgejo tracking issue (needs FORGEJO_TOKEN) +# --open-pr write the entries, push the `auto/registry-patch-proposals` +# branch, and open/update a PR autonomously. Uses +# FORGEJO_TOKEN for both the push and the PR API. The +# `trial-build-registry` pipeline is the merge gate. +# --limit N only act on the top-N candidates by failure volume +# (bounded batch; the rest are picked up on the next run) # --json PATH also write the machine-readable candidate list to PATH options(error = function() { @@ -58,6 +65,8 @@ if (is.na(min_count)) { } do_write <- "--write" %in% args do_issue <- "--open-issue" %in% args +do_pr <- "--open-pr" %in% args +limit <- suppressWarnings(as.integer(opt_val("--limit", NA_character_))) if (nchar(Sys.getenv("PGPASS")) == 0L) { stop("PGPASS env var is not set; a DB password is required.") @@ -65,6 +74,8 @@ if (nchar(Sys.getenv("PGPASS")) == 0L) { registry_file <- file.path(script_dir, "patches", "registry.json") ledger_file <- file.path(script_dir, "patches", "proposals-log.json") +# Branch the autonomous PR reuses, so re-runs update one PR instead of piling up. +pr_branch <- "auto/registry-patch-proposals" # --------------------------------------------------------------------------- # Query failing builds (same shape as failing-builds-report.R) @@ -118,7 +129,8 @@ registered_pkgs <- unique(vapply( report <- build_triage_report(failures, registered_pkgs) report <- Filter(function(r) r$build_count >= min_count, report) -# Flatten auto-proposable groups into candidate records. +# Flatten auto-proposable groups into candidate records, carrying the group's +# build volume so we can prioritise the highest-impact fixes. candidates <- list() for (r in report) { if (is.null(r$proposed_entries)) { @@ -130,23 +142,26 @@ for (r in report) { signature = r$signature, confidence = r$confidence, tier = r$tier, + build_count = r$build_count, entry = r$proposed_entries[[pkg]] ) } } -# Groups blocked on a dependency (e.g. RcppParallel dependents) are reported, -# not proposed: fixing the named dependency clears them all at once. -blocked <- Filter(function(r) !is.null(r$blocked_on), report) +# Packages blocked on a dependency are reported (aggregated by dependency, +# ranked by impact), not proposed: fixing the named dependency clears the batch. +blocked <- blocked_by_dependency(report) if (length(blocked) > 0L) { - cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") - for (r in blocked) { - cat(sprintf( - " %s: %d package(s) fail because %s does not build\n", - toString(r$blocked_on), - length(r$packages), - toString(r$blocked_on) - )) + n_blocked_pkgs <- length(unique(unlist( + lapply(report, function(r) r$blocked_packages) + ))) + cat(sprintf( + "\nBlocked on a dependency (%d dependencies block %d dependents; fix the dependency, not each dependent):\n", + length(blocked), + n_blocked_pkgs + )) + for (b in blocked) { + cat(sprintf(" %-20s %5d dependent(s)\n", b$dependency, b$n_packages)) } } @@ -173,6 +188,22 @@ if (length(candidates) == 0L) { q(status = 0) } +# Prioritise by failure volume, then apply --limit so one run tackles a bounded +# batch (the rest are picked up on the next run). +candidates <- candidates[order( + -vapply(candidates, function(c) c$build_count %||% 0L, integer(1L)) +)] +deferred <- 0L +if (!is.na(limit) && limit >= 0L && length(candidates) > limit) { + deferred <- length(candidates) - limit + candidates <- utils::head(candidates, limit) + cat(sprintf( + "\nLimiting to top %d candidate(s) by failure volume; %d deferred to a later run.\n", + limit, + deferred + )) +} + cat(sprintf( "\n%d candidate registry %s:\n", length(candidates), @@ -386,8 +417,162 @@ if (do_write) { cat(sprintf("Opened tracking issue #%d.\n", created$number)) } save_ledger(merge_ledger(load_ledger(), new_ledger_records)) +} else if (do_pr) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-pr requires FORGEJO_TOKEN (to open the PR).") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + + # Write the entries + ledger, then commit them onto the reused auto branch. + jsonlite::write_json( + merged, + registry_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) + + git <- function(...) { + # system2() with captured output runs via /bin/sh, so shell-quote every arg + # (commit messages contain "()", refs contain "^{}", etc.). + st <- system2("git", shQuote(c(...)), stdout = TRUE, stderr = TRUE) + if (!identical(attr(st, "status"), NULL)) { + stop(sprintf( + "git %s failed:\n%s", + paste(..., collapse = " "), + paste(st, collapse = "\n") + )) + } + invisible(st) + } + git("config", "user.name", Sys.getenv("GIT_USER", "devxy-bot")) + git( + "config", + "user.email", + Sys.getenv("GIT_EMAIL", "bot@devxy.io") + ) + git("checkout", "-B", pr_branch) + git( + "add", + file.path(script_dir, "patches", "registry.json"), + ledger_file + ) + git( + "commit", + "-m", + sprintf( + "feat(patches): auto-propose %d registry %s from classified failures", + length(candidate_entries), + if (length(candidate_entries) == 1L) "entry" else "entries" + ) + ) + # Push over HTTPS with FORGEJO_TOKEN (same token used for the PR API), so no + # separate write-scoped secret is needed. The read-only `origin` clone URL + # can't push, so build an authenticated URL explicitly. + push_target <- sprintf( + "https://%s:%s@git.devxy.io/%s.git", + Sys.getenv("GIT_REMOTE_USER", "pat-s"), + forgejo_token, + repo + ) + git("push", "-f", push_target, sprintf("HEAD:refs/heads/%s", pr_branch)) + + pr_title <- sprintf( + "feat(patches): auto-proposed registry patches (%s)", + now + ) + body_lines <- c( + sprintf( + "_Auto-generated %s by `local/propose-patches.R --open-pr` from classified `single_builds` failures._", + now + ), + "", + sprintf( + "Adds %d known-lever registry %s (top by failure volume%s).", + length(candidate_entries), + if (length(candidate_entries) == 1L) "entry" else "entries", + if (deferred > 0L) { + sprintf("; %d deferred to a later run", deferred) + } else { + "" + } + ), + "", + "**Merge gate:** the `trial-build-registry` pipeline builds each new entry in its target build-env image; merge only once it is green.", + "Novel source diffs and unknown signatures are never auto-proposed.", + "", + "| package | signature | platforms |", + "| --- | --- | --- |" + ) + for (c in candidates) { + body_lines <- c( + body_lines, + sprintf( + "| %s | %s | %s |", + c$package, + c$signature, + toString(as.character(c$entry$platforms)) + ) + ) + } + new_body <- paste(body_lines, collapse = "\n") + + # One PR per reused branch: update if open, else create. + pulls_url <- sprintf( + "%s/repos/%s/pulls?state=open&limit=50", + forgejo_base, + repo + ) + open_pulls <- httr2::request(pulls_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + open_pulls, + function(p) identical(p$head$ref, pr_branch), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- open_pulls[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/pulls/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = pr_title, body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("\nUpdated auto-patch PR #%d (branch %s).\n", num, pr_branch)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/pulls", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list( + title = pr_title, + head = pr_branch, + base = "main", + body = new_body + )) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf( + "\nOpened auto-patch PR #%d (branch %s).\n", + created$number, + pr_branch + )) + } } else { cat( - "\nDry run: no changes made. Re-run with --write or --open-issue to act.\n" + "\nDry run: no changes made. Re-run with --write, --open-issue, or --open-pr to act.\n" ) } diff --git a/local/r-minor-helpers.R b/local/r-minor-helpers.R index 6f01c29..65aafdf 100644 --- a/local/r-minor-helpers.R +++ b/local/r-minor-helpers.R @@ -31,42 +31,3 @@ parse_build_args <- function(args) { ncpus = as.integer(pos[3L]) ) } - -# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the -# "patched" repo mints a new content hash on every PACKAGES change, so each build -# writes a fresh ~70 MB _metadata/pkgs-.rds (+ patched-/) that is -# never reused. Keep the `keep` newest entries by mtime; only remove entries -# older than `min_age_secs`, so a concurrent split-job's in-flight files are -# never deleted (each build uses a unique hash, so aged entries are -# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are -# not matched and thus preserved. Returns the number of entries removed. -trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"), - keep = 20L, - min_age_secs = 600) { - meta <- file.path(cache_dir, "R", "pkgcache", "_metadata") - if (!nzchar(cache_dir) || !dir.exists(meta)) { - return(0L) - } - entries <- c( - Sys.glob(file.path(meta, "patched-*")), - Sys.glob(file.path(meta, "pkgs-*.rds")) - ) - if (length(entries) == 0L) { - return(0L) - } - info <- file.info(entries) - order_new_first <- order(info$mtime, decreasing = TRUE) - ranked <- entries[order_new_first] - ranked_mtime <- info$mtime[order_new_first] - if (length(ranked) <= keep) { - return(0L) - } - candidates <- ranked[(keep + 1L):length(ranked)] - candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)]) - removable <- candidates[candidate_age >= min_age_secs] - if (length(removable) == 0L) { - return(0L) - } - unlink(removable, recursive = TRUE, force = TRUE) - length(removable) -} diff --git a/local/rebuild-missing-helpers.R b/local/rebuild-missing-helpers.R new file mode 100644 index 0000000..16a588e --- /dev/null +++ b/local/rebuild-missing-helpers.R @@ -0,0 +1,87 @@ +# Pure helpers for local/rebuild-missing.R, kept separate so local/tests can +# source them without executing a rebuild. + +# Interleaved slice of the rebuild list. +# +# The list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, +# rstan*), so contiguous thirds would be badly unbalanced. Interleaving also +# makes each shard's progress counter representative of the slot as a whole. +shard_slice <- function(pkgs, split_into, split_index) { + split_into <- as.integer(split_into) + split_index <- as.integer(split_index) + if (is.na(split_into) || is.na(split_index)) { + stop("shard_slice(): split_into and split_index must be integers") + } + if (split_into < 1L || split_index < 1L || split_index > split_into) { + stop(sprintf( + "shard_slice(): need 1 <= split_index <= split_into, got %s of %s", + split_index, + split_into + )) + } + # seq() errors on a descending range, which is what an empty list or a shard + # index past the end would produce. + if (length(pkgs) < split_index) { + return(pkgs[0L]) + } + pkgs[seq.int(split_index, length(pkgs), by = split_into)] +} + +# Packages that still need building, decided from the bucket rather than from +# remembered progress. +# +# This is bincraft's `check_s3_root_package()` evaluated in bulk: an object +# whose ETag equals CRAN's published MD5sum is byte-identical to CRAN's source, +# so the build that was supposed to replace it has not happened yet. +# +# `etag_by_file` named by `_.tar.gz`, values are unquoted ETags +# `cran_version` named by package +# `cran_md5` named by `_` +# +# Unknown always means "already a binary", never "rebuild it", so an unreadable +# CRAN index or a multipart ETag can never mass-schedule work. +outstanding_packages <- function(pkgs, etag_by_file, cran_version, cran_md5) { + if (length(pkgs) == 0L) { + return(pkgs) + } + + # An empty table indexes to zero length rather than to NA, which would + # recycle the whole result away and silently report "nothing to build". + lookup <- function(table, key) { + if (length(table) == 0L) { + return(rep(NA_character_, length(key))) + } + unname(as.character(table[key])) + } + + version <- lookup(cran_version, pkgs) + file <- sprintf("%s_%s.tar.gz", pkgs, version) + etag <- lookup(etag_by_file, file) + md5 <- lookup(cran_md5, paste(pkgs, version, sep = "_")) + + # No CRAN version means the package cannot be resolved to a tarball at all; + # leave it in and let bincraft report why. + unresolved <- is.na(version) + # No object at the key: never built, so it is outstanding by definition. + absent <- !unresolved & is.na(etag) + # A multipart upload carries a compound ETag rather than an MD5. + unknown <- !is.na(etag) & grepl("-", etag, fixed = TRUE) + + is_source <- !unresolved & + !is.na(etag) & + !unknown & + !is.na(md5) & + etag == md5 + + pkgs[unresolved | absent | is_source] +} + +parse_rebuild_args <- function(args) { + pos <- args[!startsWith(args, "--")] + budget <- as.numeric(pos[3L]) + list( + split_into = as.integer(pos[1L]), + split_index = as.integer(pos[2L]), + budget_hours = if (is.na(budget)) 20 else budget + ) +} diff --git a/local/rebuild-missing.R b/local/rebuild-missing.R new file mode 100644 index 0000000..369024e --- /dev/null +++ b/local/rebuild-missing.R @@ -0,0 +1,194 @@ +### Rebuild one shard of a slot's missing-binary list. +# +# Usage: Rscript local/rebuild-missing.R [budget_hours] +# +# The list itself comes from local/fetch-rebuild-packages-from-issue.R, which +# writes $REBUILD_PKG_LIST (default /tmp/rebuild_pkgs.txt). +# +# Two properties matter here and are the reason this is a script rather than an +# `R -q -e` argument in the pipeline: +# +# * it is restartable. The outstanding set is re-derived from the bucket on +# every start, so a shard that died resumes where it stopped without any +# progress file, and without replaying thousands of per-package HEADs. +# * it terminates. A wall-clock budget stops the loop cleanly instead of the +# run having to be killed, which is what previously skipped the re-index and +# CDN purge and left rebuilt binaries hidden behind stale edge copies. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +source(file.path("local", "rebuild-missing-helpers.R")) + +args <- parse_rebuild_args(commandArgs(trailingOnly = TRUE)) +if (is.na(args$split_into) || is.na(args$split_index)) { + stop("usage: rebuild-missing.R [budget_hours]") +} + +list_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") +pkgs <- if (file.exists(list_file)) readLines(list_file) else character(0) +pkgs <- pkgs[nzchar(pkgs)] +if (length(pkgs) == 0L) { + cat("Nothing to rebuild\n") + q("no") +} + +excluded <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] +pkgs <- setdiff(pkgs, excluded) + +mine <- shard_slice(pkgs, args$split_into, args$split_index) +cat(sprintf( + "Shard %s/%s: %s of %s listed packages\n", + args$split_index, + args$split_into, + length(mine), + length(pkgs) +)) + +### Resume: ask the bucket what is still outstanding + +codename <- bincraft::set_codename(NULL) +local_machine <- Sys.info()[["machine"]] +arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" +slot_dir <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", + arch, + codename +) + +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) + +# One paginated listing instead of a HEAD per package. Not recursed: the +# rebuild passes no `is_r_minor_sensitive`, so it only ever targets the flat +# path, and the resume filter matches that scope deliberately. +info <- tryCatch(s3fs::s3_dir_info(slot_dir), error = function(e) NULL) +etag_by_file <- if (is.null(info) || nrow(info) == 0L) { + cat(sprintf( + "WARNING: could not list %s; building the whole shard\n", + slot_dir + )) + stats::setNames(character(), character()) +} else { + stats::setNames( + gsub('^"|"$', "", as.character(info$etag)), + basename(as.character(info$uri)) + ) +} + +cran <- tryCatch( + { + con <- gzcon(url( + "https://cloud.r-project.org/src/contrib/PACKAGES.gz", + open = "rb" + )) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "MD5sum")) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read CRAN's index (%s)\n", + conditionMessage(e) + )) + NULL + } +) +cran_version <- stats::setNames(character(), character()) +cran_md5 <- stats::setNames(character(), character()) +if (!is.null(cran)) { + cran_version <- stats::setNames( + as.character(cran[, "Version"]), + as.character(cran[, "Package"]) + ) + keep <- !is.na(cran[, "MD5sum"]) + cran_md5 <- stats::setNames( + as.character(cran[keep, "MD5sum"]), + paste(cran[keep, "Package"], cran[keep, "Version"], sep = "_") + ) +} + +before <- length(mine) +mine <- outstanding_packages(mine, etag_by_file, cran_version, cran_md5) +cat(sprintf( + "Resume: %s of %s already carry a binary; %s outstanding\n", + before - length(mine), + before, + length(mine) +)) + +if (length(mine) == 0L) { + cat("Nothing outstanding for this shard\n") + q("no") +} + +### Build + +options( + crayon.enabled = TRUE, + Ncpus = as.integer(Sys.getenv("NCPUS", "2")), + future.globals.onReference = NULL +) + +started <- Sys.time() +n <- length(mine) +completed <- 0L +for (i in seq_along(mine)) { + elapsed <- as.numeric(difftime(Sys.time(), started, units = "hours")) + if (elapsed > args$budget_hours) { + cat(sprintf( + "Budget of %sh reached after %d/%d packages; stopping cleanly. The next run resumes from the bucket.\n", + args$budget_hours, + completed, + n + )) + break + } + + x <- mine[i] + cat(sprintf("[%d/%d] %s\n", i, n, x)) + tryCatch( + bincraft::build_binary_package( + x, + tag_limit = 1L, + patches = "local/patches", + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + upload = TRUE, + store_build_metadata = TRUE + ), + error = function(e) { + cat(sprintf("ERROR building %s - %s\n", x, conditionMessage(e))) + } + ) + completed <- completed + 1L +} + +cat(sprintf( + "Shard %s/%s finished: %d/%d packages processed in %.1fh\n", + args$split_index, + args$split_into, + completed, + n, + as.numeric(difftime(Sys.time(), started, units = "hours")) +)) diff --git a/local/repair-built-stamp.R b/local/repair-built-stamp.R new file mode 100644 index 0000000..06f73be --- /dev/null +++ b/local/repair-built-stamp.R @@ -0,0 +1,175 @@ +#!/usr/bin/env Rscript + +### Rewrite a broken `Built` stamp across one arch/codename slot. +### +### A slot's index can end up advertising a stamp whose platform triple is +### unusable, e.g. `Built: R 4.5.0; NA; ...`. uvr picks binary vs source by +### matching that triple plus the R minor, so no client matches it and the whole +### slot silently reverts to source-only, which makes uvr compile everything and +### fail wherever a system `-dev` library is missing. +### +### Rewriting it is not a matter of re-running the normal index update. +### `upload_package_index()` reuses the slot's remote `PACKAGES.db`, and +### cranlike's `update_db()` only reparses files whose md5 changed, so entries +### already in the database keep the stamp they were written with. Dropping +### `PACKAGES.db` to force a full reparse does work, but for an S3 repo cranlike +### reads each package's metadata from the CRAN *source* mirror on GitHub, so a +### 25k-entry slot means 25k requests to raw.githubusercontent.com and a real +### risk of being rate-limited part-way through. +### +### Only the `Built` column is wrong, so correct it in place instead: patch the +### column in `PACKAGES.db`, put the database back, and let +### `upload_package_index()` re-emit `PACKAGES*` from it. `update_db()` always +### rewrites the index files even when nothing was reparsed, so no tarball is +### re-read and nothing is fetched from GitHub. +### +### The replacement comes from `bincraft::built_stamp()` under the R running +### this script, so run it under the R version the slot should advertise (the +### `R_VERSION` its entry in `.crow/process-updates.yaml` uses). That is what a +### healthy `upload_package_index()` run would have written. +### +### Usage, inside the platform's build image: +### Rscript local/repair-built-stamp.R [--apply] +### +### Without `--apply` it reports what it would change and touches nothing. + +suppressPackageStartupMessages({ + library(bincraft) +}) + +args <- commandArgs(trailingOnly = TRUE) +arch <- args[1L] +apply_changes <- "--apply" %in% args + +if (is.na(arch) || !nzchar(arch)) { + stop( + "Usage: Rscript local/repair-built-stamp.R [--apply]", + call. = FALSE + ) +} + +bucket <- "devxy-rpkgs-binaries" +endpoint <- "https://s3.eu-central-003.backblazeb2.com" +region <- "eu-central-003" + +codename <- bincraft::set_codename(NULL) +if (is.null(codename) || is.na(codename) || !nzchar(codename)) { + stop( + "Could not detect a codename from /etc/os-release; run this in a build image.", + call. = FALSE + ) +} + +# built_stamp() refuses an unusable platform, so a broken build image fails here +# rather than writing a second bad stamp over the first one. +stamp <- bincraft::built_stamp() +message(sprintf("Slot: %s/%s", arch, codename)) +message(sprintf("New stamp: %s", stamp)) + +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = endpoint, + region_name = region, + refresh = TRUE +) + +base_dir <- file.path(bucket, arch, codename, "latest", "src", "contrib") + +# A stamp is broken when its platform component is absent or literally "NA". +broken_stamp_where <- paste( + "Built IS NULL", + "OR Built LIKE '%; NA;%'", + "OR Built LIKE '%; ;%'" +) + +# The generic slot plus every per-minor sub-slot, which carry the same stamp and +# are poisoned by the same run. +r_minors <- sub( + "^.*/R/([0-9]+\\.[0-9]+)\\.[0-9]+$", + "\\1", + list.dirs("/opt/R", recursive = FALSE) +) +r_minors <- unique(grep("^[0-9]+\\.[0-9]+$", r_minors, value = TRUE)) +slots <- c(base_dir, file.path(base_dir, r_minors)) + +repair_slot <- function(slot) { + db_remote <- file.path(slot, "PACKAGES.db") + if (!s3fs::s3_file_exists(db_remote)) { + message(sprintf(" %s: no PACKAGES.db, skipping", slot)) + return(invisible(NULL)) + } + + db_local <- tempfile(fileext = ".db") + s3fs::s3_file_download(db_remote, db_local, overwrite = TRUE) + + con <- DBI::dbConnect(RSQLite::SQLite(), db_local) + on.exit(DBI::dbDisconnect(con), add = TRUE) + + total <- DBI::dbGetQuery(con, "SELECT COUNT(*) AS n FROM packages")$n + broken <- DBI::dbGetQuery( + con, + sprintf("SELECT COUNT(*) AS n FROM packages WHERE %s", broken_stamp_where) + )$n + + message(sprintf(" %s: %s entries, %s broken", slot, total, broken)) + if (broken == 0L) { + return(invisible(NULL)) + } + if (!apply_changes) { + message(" (dry run, pass --apply to rewrite)") + return(invisible(NULL)) + } + + DBI::dbExecute( + con, + sprintf("UPDATE packages SET Built = ? WHERE %s", broken_stamp_where), + params = list(stamp) + ) + DBI::dbDisconnect(con) + on.exit() + + s3fs::s3_file_upload(db_local, db_remote, overwrite = TRUE) + message(sprintf(" rewrote %s entries and uploaded PACKAGES.db", broken)) + invisible(NULL) +} + +invisible(lapply(slots, repair_slot)) + +if (!apply_changes) { + message("Dry run complete; nothing was changed.") + quit(save = "no") +} + +# Re-emit PACKAGES/PACKAGES.gz/PACKAGES.rds from the corrected database. Nothing +# is reparsed, because no tarball's md5 changed. +message("Re-emitting index files from the corrected database...") +bincraft::upload_package_index( + codename = codename, + s3_endpoint = endpoint, + s3_region = region, + s3_bucket = bucket, + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") +) +for (minor in r_minors) { + try( + bincraft::upload_package_index( + codename = codename, + r_minor = minor, + s3_endpoint = endpoint, + s3_region = region, + s3_bucket = bucket, + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") + ), + silent = FALSE + ) +} + +message("Done. Verify with:") +message(sprintf( + " curl -sS https://cran.devxy.io/%s/%s/latest/src/contrib/PACKAGES | grep '^Built:' | sort | uniq -c", + arch, + codename +)) diff --git a/local/test-package-loading.R b/local/test-package-loading.R index 26a2e82..a3b7a28 100644 --- a/local/test-package-loading.R +++ b/local/test-package-loading.R @@ -1,22 +1,28 @@ -install.packages( - "pak", - repos = sprintf( - "https://r-lib.github.io/p/pak/stable/%s/%s/%s", - .Platform$pkgType, - R.Version()$os, - R.Version()$arch +# Installs every CRAN package one by one and checks that it loads. Dependencies +# go through uvr via local/uvr-install.sh, which bootstraps the uvr binary on +# first use and installs into .libPaths()[1]. +uvr_install <- function(pkg) { + Sys.setenv( + UVR_R_BIN = file.path(R.home("bin"), "R"), + UVR_TARGET_LIB = .libPaths()[1L] ) -) + status <- system2("local/uvr-install.sh", shQuote(pkg)) + if (!identical(status, 0L)) { + stop( + sprintf("uvr failed to install %s (exit %s)", pkg, status), + call. = FALSE + ) + } +} -Sys.setenv(PKG_SYSREQS = TRUE) all_pkgs <- rownames(available.packages()) -to_skip = c("ABRSQOL", "ACA", "ACE.CoCo") -all_pkgs = setdiff(all_pkgs, to_skip) +to_skip <- c("ABRSQOL", "ACA", "ACE.CoCo") +all_pkgs <- setdiff(all_pkgs, to_skip) for (i in all_pkgs) { message(sprintf("\nInstalling %s", i)) - pak::pkg_install(i) + uvr_install(i) library(i, character.only = TRUE) } @@ -323,7 +329,7 @@ if (length(to_process) == 0) { } else { for (i in to_process) { message(sprintf("\nInstalling %s", i)) - pak::pkg_install(i) + uvr_install(i) library(i, character.only = TRUE) } # Update to_skip to include all up to the last processed diff --git a/local/tests/test-failing-builds-classify.R b/local/tests/test-failing-builds-classify.R index a365e14..d4df9f5 100644 --- a/local/tests/test-failing-builds-classify.R +++ b/local/tests/test-failing-builds-classify.R @@ -105,6 +105,49 @@ test_that("RcppParallel dependents are blocked, not proposed a per-package patch expect_identical(grp$blocked_on, "RcppParallel") }) +test_that("failing_dependency names the dependency that actually failed", { + # A leaf package (BFpack) that fails building its rstan dependency. + txt <- paste( + "Error in installing dependencies for package BFpack with tag 1.6.1", + "Failed to build source package rstan.", + "ERROR: compilation failed for package ‘rstan’", + sep = "\n" + ) + expect_identical(failing_dependency(txt, "BFpack"), "rstan") + # A package failing in its OWN compilation is not a cascade. + own <- "ERROR: compilation failed for package ‘BFpack’" + expect_true(is.na(failing_dependency(own, "BFpack"))) + expect_true(is.na(failing_dependency(NA_character_, "x"))) + expect_true(is.na(failing_dependency("some unrelated error", "x"))) +}) + +test_that("Stan packages blocked on rstan are not proposed a per-package entry", { + # BFpack/GMLTM fail building rstan; the tbb error is in rstan's compile. + cascade <- paste( + "Failed to build source package rstan.", + "fatal error: tbb/tbb_stddef.h: No such file or directory", + sep = "\n" + ) + failures <- data.frame( + name = c("BFpack", "GMLTM", "someOwnPkg"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = c( + cascade, + cascade, + # someOwnPkg fails in its OWN compile on the same header -> fixable. + "someOwnPkg.cpp: fatal error: tbb/tbb_stddef.h: No such file or directory" + ), + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + grp <- Filter(function(g) g$signature == "tbb-stddef-removed", report)[[1L]] + # Only the own-compile package is proposed; the rstan cascades are blocked. + expect_identical(names(grp$proposed_entries), "someOwnPkg") + expect_setequal(grp$blocked_packages, c("BFpack", "GMLTM")) + expect_identical(grp$blocked_on, "rstan") +}) + test_that("RcppParallel itself is still proposed when it is the failing package", { failures <- data.frame( name = "RcppParallel", diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index 02a29f1..869d2b3 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -146,20 +146,86 @@ test_that("unclassified_summary ranks unknown groups and caps output", { expect_identical(capped$dropped_groups, 1L) }) -test_that("blocked_summary lists each dependency and its dependent count", { +test_that("blocked_by_dependency aggregates across groups, deduped and ranked", { + # RcppParallel dependents split across platforms/fingerprints -> separate + # groups, but one aggregated line; a dependent seen twice is counted once. failures <- data.frame( - name = c("ACEsimFit", "AovBay", "AdaptGauss"), - platform = "ubuntu-2604", + name = c("ACEsimFit", "AovBay", "AdaptGauss", "ACEsimFit", "loner"), + platform = c( + "ubuntu-2604", + "ubuntu-2604", + "alpine-324", + "alpine-324", + "ubuntu-2604" + ), arch = "amd64", - error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + error_text = c( + rep("Error: USE_TBB=Linux is not supported on this toolchain", 4L), + "Failed to build source package rstan.\nfatal error: tbb/tbb_stddef.h" + ), stringsAsFactors = FALSE ) report <- build_triage_report(failures, registered_pkgs = character(0L)) - b <- blocked_summary(report, max_pkgs = 2L) - expect_length(b, 1L) - expect_identical(b[[1L]]$blocked_on, "RcppParallel") - expect_identical(b[[1L]]$n_packages, 3L) - expect_true(b[[1L]]$packages_truncated) + agg <- blocked_by_dependency(report, max_pkgs = 2L) + deps <- vapply(agg, function(b) b$dependency, character(1L)) + expect_true("RcppParallel" %in% deps && "rstan" %in% deps) + rcpp <- Filter(function(b) b$dependency == "RcppParallel", agg)[[1L]] + # ACEsimFit appears in two groups -> counted once (3 distinct dependents). + expect_identical(rcpp$n_packages, 3L) + expect_true(rcpp$packages_truncated) # capped at max_pkgs = 2 + # Ranked by dependent count: RcppParallel (3) before rstan (1). + expect_identical(deps[[1L]], "RcppParallel") +}) + +test_that("entry_applies_to_os matches codename, family, and wildcard", { + expect_true(entry_applies_to_os(list("ubuntu-2604"), "ubuntu-2604")) + expect_true(entry_applies_to_os(list("ubuntu"), "ubuntu-2604")) # family + expect_true(entry_applies_to_os(list("*"), "ubuntu-2604")) + expect_true(entry_applies_to_os(list("alpine", "ubuntu-2604"), "ubuntu-2604")) + expect_false(entry_applies_to_os(list("alpine-324"), "ubuntu-2604")) + expect_false(entry_applies_to_os(list("ubuntu-2404"), "ubuntu-2604")) # other codename +}) + +test_that("new_registry_packages returns only added entries for the platform", { + base <- list( + list( + package = "RcppParallel", + platforms = list("alpine", "ubuntu-2604"), + versions = "*" + ) + ) + current <- list( + base[[1L]], # unchanged -> not "new" + list(package = "BFpack", platforms = list("ubuntu-2604"), versions = "*"), + list( + package = "someAlpinePkg", + platforms = list("alpine-324"), + versions = "*" + ) + ) + # For ubuntu-2604: only the newly-added BFpack (RcppParallel is unchanged, + # someAlpinePkg does not apply to this OS). + expect_identical( + new_registry_packages(current, base, os = "ubuntu-2604"), + "BFpack" + ) + # For alpine-324: the alpine package is new and applies. + expect_identical( + new_registry_packages(current, base, os = "alpine-324"), + "someAlpinePkg" + ) + # Without an OS filter, both additions are returned. + expect_setequal( + new_registry_packages(current, base), + c("BFpack", "someAlpinePkg") + ) + # A changed platform set on the same package counts as a new entry. + widened <- list(list( + package = "RcppParallel", + platforms = list("*"), + versions = "*" + )) + expect_identical(new_registry_packages(widened, base), "RcppParallel") }) test_that("retirement_candidates flags entries whose package no longer fails", { diff --git a/local/tests/test-rebuild-missing.R b/local/tests/test-rebuild-missing.R new file mode 100644 index 0000000..a5e70b9 --- /dev/null +++ b/local/tests/test-rebuild-missing.R @@ -0,0 +1,94 @@ +source(file.path("..", "rebuild-missing-helpers.R")) + +test_that("shard_slice partitions the list without gaps or overlap", { + pkgs <- letters[1:10] + parts <- lapply(1:3, function(i) shard_slice(pkgs, 3, i)) + + expect_identical(parts[[1]], c("a", "d", "g", "j")) + expect_identical(parts[[2]], c("b", "e", "h")) + expect_identical(parts[[3]], c("c", "f", "i")) + + expect_identical(sort(unlist(parts)), sort(pkgs)) + expect_identical(anyDuplicated(unlist(parts)), 0L) +}) + +test_that("shard_slice is deterministic and survives short lists", { + expect_identical( + shard_slice(letters[1:10], 3, 2), + shard_slice(letters[1:10], 3, 2) + ) + expect_identical(shard_slice(character(0), 3, 1), character(0)) + # more shards than packages: the tail shards get nothing rather than erroring + expect_identical(shard_slice(c("a"), 3, 1), "a") + expect_identical(shard_slice(c("a"), 3, 2), character(0)) +}) + +test_that("shard_slice rejects an out-of-range index", { + expect_error(shard_slice(letters, 3, 4), "split_index") + expect_error(shard_slice(letters, 3, 0), "split_index") +}) + +test_that("outstanding_packages keeps source fallbacks and drops real binaries", { + cran_version <- c(httr = "1.4.8", R6 = "2.6.1", curl = "7.1.0") + cran_md5 <- c( + httr_1.4.8 = "8756015b94a9cff6f410ca4de8557f12", + R6_2.6.1 = "f01b1787f12797c29194d63c9afd5d70", + curl_7.1.0 = "8af2ccbf5d85dc18866f45f1f26f348d" + ) + etag <- c( + # byte-identical to CRAN: the build never happened + "httr_1.4.8.tar.gz" = "8756015b94a9cff6f410ca4de8557f12", + # a real binary was published + "R6_2.6.1.tar.gz" = "9d6087ee9adda3f0a3b8067cfc652c05" + # curl has no object at all + ) + + out <- outstanding_packages( + c("httr", "R6", "curl"), + etag, + cran_version, + cran_md5 + ) + expect_identical(out, c("httr", "curl")) +}) + +test_that("outstanding_packages treats unknowns as already built", { + cran_version <- c(a = "1.0", b = "1.0") + cran_md5 <- c(a_1.0 = "aaaa") + + # a multipart ETag carries no MD5, and `b` is missing from CRAN's index: + # neither may schedule a rebuild + etag <- c("a_1.0.tar.gz" = "abc-3", "b_1.0.tar.gz" = "bbbb") + + expect_identical( + outstanding_packages(c("a", "b"), etag, cran_version, cran_md5), + character(0) + ) +}) + +test_that("outstanding_packages keeps a package CRAN has no version for", { + out <- outstanding_packages( + "ghost", + c(), + c(other = "1.0"), + c(other_1.0 = "aaaa") + ) + expect_identical(out, "ghost") +}) + +test_that("outstanding_packages handles an empty list", { + expect_identical( + outstanding_packages(character(0), c(), c(), c()), + character(0) + ) +}) + +test_that("parse_rebuild_args defaults the budget", { + a <- parse_rebuild_args(c("3", "2")) + expect_identical(a$split_into, 3L) + expect_identical(a$split_index, 2L) + expect_identical(a$budget_hours, 20) + + b <- parse_rebuild_args(c("3", "2", "1.5")) + expect_identical(b$budget_hours, 1.5) +}) diff --git a/local/tests/test-trim-pkgcache.R b/local/tests/test-trim-pkgcache.R deleted file mode 100644 index 1a5b418..0000000 --- a/local/tests/test-trim-pkgcache.R +++ /dev/null @@ -1,68 +0,0 @@ -source(file.path("..", "r-minor-helpers.R")) - -# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is -# set to `age_secs` in the past so we can exercise the age gate deterministically. -make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) { - root <- tempfile("pkgcache-") - meta <- file.path(root, "R", "pkgcache", "_metadata") - dir.create(meta, recursive = TRUE) - old <- Sys.time() - age_secs - mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p } - mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p } - for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i))) - for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i))) - if (keep_repos) { - mk_dir(file.path(meta, "CRAN-075c426938")) - mk_dir(file.path(meta, "BioCsoft-1ac964ed6c")) - mk_file(file.path(meta, "bioc-sysreqs.dcf.gz")) - mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive - } - root -} - -n_churn <- function(root) { - meta <- file.path(root, "R", "pkgcache", "_metadata") - length(Sys.glob(file.path(meta, "patched-*"))) + - length(Sys.glob(file.path(meta, "pkgs-*.rds"))) -} - -test_that("empty cache_dir is a no-op", { - expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L) -}) - -test_that("missing _metadata dir is a no-op", { - expect_identical( - trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0), - 0L - ) -}) - -test_that("fewer than keep entries removes nothing", { - root <- make_meta(patched = 2L, pkgs = 2L) - expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L) - expect_identical(n_churn(root), 4L) -}) - -test_that("trims down to keep newest, leaving churn == keep", { - root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old - removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0) - expect_identical(removed, 40L) - expect_identical(n_churn(root), 20L) -}) - -test_that("entries younger than min_age_secs are protected", { - root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60) - # keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed - expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L) - expect_identical(n_churn(root), 30L) -}) - -test_that("stable repo dirs and pkg downloads are never touched", { - root <- make_meta(patched = 30L, pkgs = 30L) - trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0) - meta <- file.path(root, "R", "pkgcache", "_metadata") - expect_true(dir.exists(file.path(meta, "CRAN-075c426938"))) - expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c"))) - expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz"))) - expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg"))) -}) diff --git a/local/trial-build-patch.R b/local/trial-build-patch.R index 4e8030a..2229074 100644 --- a/local/trial-build-patch.R +++ b/local/trial-build-patch.R @@ -40,23 +40,26 @@ cat(sprintf( patches_dir )) -ok <- tryCatch( - { - bincraft::build_binary_package( - package, - tag_limit = 1L, - patches = patches_dir, - archive = FALSE, - upload = FALSE, - store_build_metadata = FALSE - ) - TRUE - }, +# build_binary_package() catches build failures internally and RETURNS "error" +# for the failed tag rather than throwing (bincraft >= v4.4.7), so inspect the +# return value -- checking only for a thrown exception reports a broken build as +# passing (false green). +res <- tryCatch( + bincraft::build_binary_package( + package, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ), error = function(e) { - cat(sprintf("Trial build FAILED: %s\n", conditionMessage(e))) - FALSE + cat(sprintf("Trial build FAILED (threw): %s\n", conditionMessage(e))) + "error" } ) +flat <- as.character(unlist(res)) +ok <- length(flat) > 0L && !("error" %in% flat) if (ok) { cat(sprintf("Trial build OK: %s builds with the proposed patch.\n", package)) diff --git a/local/trial-build-registry.R b/local/trial-build-registry.R new file mode 100644 index 0000000..9708408 --- /dev/null +++ b/local/trial-build-registry.R @@ -0,0 +1,163 @@ +#!/usr/bin/env Rscript + +# Merge gate for the auto-patch PR (issue #115, step 3): for every registry +# entry the PR ADDS that applies to this platform, trial-build the package with +# the registry applied, in this platform's own build-env image. Nothing is +# uploaded, archived, or written to the metadata DB. +# +# Exit 0 only if every new entry's package builds; exit 1 if any fails, so it +# gates the PR. A platform with no new entries is a fast no-op. +# +# Usage (inside a build-env image): +# PLATFORM=ubuntu-2604 Rscript local/trial-build-registry.R [base_ref] +# base_ref git ref to diff the registry against (default: origin/main) + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(jsonlite, quietly = TRUE) + library(bincraft, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +args <- commandArgs(trailingOnly = TRUE) +base_ref <- if (length(args) >= 1L) { + args[[1L]] +} else { + Sys.getenv("BASE_REF", "origin/main") +} +os <- Sys.getenv("PLATFORM", "") +if (!nzchar(os)) { + stop("PLATFORM env var is not set (e.g. ubuntu-2604).") +} + +patches_dir <- file.path(script_dir, "patches") +registry_file <- file.path(patches_dir, "registry.json") +current <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +# Read the registry at base_ref. Fail loud if the ref or file can't be read: +# silently treating the base as empty would trial-build the WHOLE registry +# instead of just the entries the branch adds. +registry_rel <- "local/patches/registry.json" +# system2() with captured output runs via /bin/sh, so shell-quote the git args +# (refs contain "^{}" and ":" that the shell would otherwise mangle). +ref_ok <- suppressWarnings(system2( + "git", + shQuote(c( + "rev-parse", + "--verify", + "--quiet", + sprintf("%s^{commit}", base_ref) + )), + stdout = TRUE, + stderr = FALSE +)) +if (!is.null(attr(ref_ok, "status"))) { + stop(sprintf("base ref %s does not resolve to a commit.", base_ref)) +} +in_base <- suppressWarnings(system2( + "git", + shQuote(c("ls-tree", base_ref, "--", registry_rel)), + stdout = TRUE, + stderr = FALSE +)) +file_in_base <- length(in_base) > 0L && any(nzchar(in_base)) +base_json <- suppressWarnings(system2( + "git", + shQuote(c("show", sprintf("%s:%s", base_ref, registry_rel))), + stdout = TRUE, + stderr = FALSE +)) +show_ok <- is.null(attr(base_json, "status")) +if (file_in_base && !show_ok) { + stop(sprintf( + "could not read %s at %s; refusing to build the whole registry.", + registry_rel, + base_ref + )) +} +base <- if (show_ok && length(base_json) > 0L) { + jsonlite::fromJSON(paste(base_json, collapse = "\n"), simplifyVector = FALSE) +} else { + list() # file genuinely absent at base -> every entry is new +} + +pkgs <- new_registry_packages(current, base, os = os) +if (length(pkgs) == 0L) { + cat(sprintf( + "No new registry entries apply to %s; nothing to trial-build.\n", + os + )) + q(status = 0) +} + +cat(sprintf( + "Trial-building %d new registry %s on %s (vs %s):\n %s\n", + length(pkgs), + if (length(pkgs) == 1L) "entry" else "entries", + os, + base_ref, + toString(pkgs) +)) + +results <- vapply( + pkgs, + function(pkg) { + cat(sprintf("\n=== trial build: %s ===\n", pkg)) + # bincraft::build_binary_package() catches build failures internally and + # RETURNS "error" for the failed tag rather than throwing, so a green gate + # must inspect the return value -- checking only for a thrown exception + # reports a broken build as passing. + res <- tryCatch( + bincraft::build_binary_package( + pkg, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ), + error = function(e) { + cat(sprintf("FAILED %s (threw): %s\n", pkg, conditionMessage(e))) + "error" + } + ) + flat <- as.character(unlist(res)) + ok <- length(flat) > 0L && !("error" %in% flat) + if (!ok) { + cat(sprintf( + "FAILED %s: build did not succeed (result: %s)\n", + pkg, + if (length(flat) > 0L) toString(flat) else "" + )) + } + ok + }, + logical(1L) +) + +failed <- pkgs[!results] +cat(sprintf( + "\n%d/%d passed on %s.%s\n", + sum(results), + length(results), + os, + if (length(failed) > 0L) sprintf(" Failed: %s", toString(failed)) else "" +)) +if (length(failed) > 0L) { + q(status = 1) +} +q(status = 0) diff --git a/local/uvr-install.sh b/local/uvr-install.sh new file mode 100755 index 0000000..3966e85 --- /dev/null +++ b/local/uvr-install.sh @@ -0,0 +1,120 @@ +#!/bin/sh +# Install R packages into the CI library with uvr (https://github.com/nbafrank/uvr). +# +# Usage: +# local/uvr-install.sh httr2 jsonlite +# local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft@v4.4.3 +# +# Replaces `pak::pak(...)`. uvr is project-scoped: `uvr add` refuses to run +# outside a project and always writes to `.uvr/library/`, and only +# `uvr sync --library` can target an existing library. The project is therefore +# minted in a scratch directory under TMPDIR and thrown away afterwards; that +# also keeps `uvr init`'s `.Rprofile` out of the repo checkout, where it would +# hijack `.libPaths()` for every other R call in the pipeline. +# +# Pruning is a no-op here: uvr disables it whenever `--library` is passed, +# precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds +# bincraft and its dependencies alongside whatever this script installs). +# +# System dependencies come from uvr's vendored r-system-requirements rules, so +# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed. +# +# Environment: +# UVR_R_BIN R interpreter to install for; set by install-bincraft.R so +# the per-R-minor passes target their own R, not the primary +# R_VERSION fallback interpreter selector (/opt/R//bin/R) +# UVR_TARGET_LIB target library; defaults to R_LIBS_USER, then to the +# active R's .libPaths()[1] (which is where pak wrote) +# UVR_INSTALL_DIR where the uvr binary lands (default /usr/local/bin) + +set -eu + +# renovate: datasource=github-releases depName=nbafrank/uvr +UVR_PIN="v0.4.6" + +if [ "$#" -eq 0 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi + +# The build images keep R under /opt/R/ and off PATH. uvr resolves the +# interpreter via PATH and never downloads one unless `uvr r install` is run, so +# put the requested R first. +r_bin="${UVR_R_BIN:-}" +if [ -z "$r_bin" ] && [ -n "${R_VERSION:-}" ] && [ -x "/opt/R/${R_VERSION}/bin/R" ]; then + r_bin="/opt/R/${R_VERSION}/bin/R" +fi +if [ -n "$r_bin" ]; then + PATH="$(dirname "$r_bin"):$PATH" + export PATH +else + r_bin="$(command -v R)" +fi + +target_lib="${UVR_TARGET_LIB:-${R_LIBS_USER:-}}" +if [ -z "$target_lib" ]; then + target_lib="$("$r_bin" --no-echo --no-save -e 'cat(.libPaths()[1])')" +fi +if [ -z "$target_lib" ]; then + echo "error: could not determine a target library; set UVR_TARGET_LIB" >&2 + exit 2 +fi +mkdir -p "$target_lib" + +# Pin the manifest to the active R so the lockfile's R stays in step with the +# library's R sentinel. Without that, uvr can decide the library is ABI-stale +# and wipe it -- and this target is shared with bincraft. uvr only discovers R +# via PATH/R_HOME (it does not scan /opt/R), so the R put on PATH above is the +# only candidate this constraint can resolve to. +# shellcheck disable=SC2016 # $major/$minor are R expressions, not shell vars +r_full="$("$r_bin" --no-echo --no-save -e 'cat(paste(R.version$major, R.version$minor, sep = "."))')" + +install_dir="${UVR_INSTALL_DIR:-/usr/local/bin}" +uvr_bin="${install_dir}/uvr" +if [ ! -x "$uvr_bin" ]; then + echo "Bootstrapping uvr ${UVR_PIN} into ${install_dir}" + UVR_INSTALL_DIR="$install_dir" UVR_VERSION="$UVR_PIN" \ + sh -c 'curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | sh' +fi + +project_dir="${TMPDIR:-/tmp}/uvr-ci-$$" +rm -rf "$project_dir" +mkdir -p "$project_dir" +trap 'rm -rf "$project_dir"' EXIT +cd "$project_dir" + +"$uvr_bin" init --here --r-version "$r_full" +# --no-install resolves and locks only; retry because concurrent shards can +# expose short-lived DNS or CRAN-index failures and uvr rolls the manifest back +# cleanly after an unsuccessful resolution. +add_attempt=1 +while ! "$uvr_bin" add --no-install "$@"; do + if [ "$add_attempt" -ge 4 ]; then + echo "error: uvr add failed after ${add_attempt} attempts" >&2 + exit 1 + fi + add_delay=$((add_attempt * 10)) + echo "warning: uvr add attempt ${add_attempt} failed; retrying in ${add_delay}s" >&2 + sleep "$add_delay" + add_attempt=$((add_attempt + 1)) +done + +# TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs +# `apt-get install` for every resolved system dependency without refreshing the +# index first, and the ubuntu build images end their apt layers with +# `rm -rf /var/lib/apt/lists/*`. With no index apt cannot resolve a package that +# exists and is enabled, so `igraph needs: libglpk-dev` fails the whole build +# with `E: Unable to locate package libglpk-dev` on ubuntu 26.04. +# +# Fixed upstream in `e491b2e`, tagged one day after v0.4.5 (nbafrank/uvr#250), +# and the images pick it up via build-env-images#23 — but only after an image +# rebuild is triggered, which is why this runs here too. +# +# apt only: `apk add` fetches its index implicitly and dnf refreshes expired +# metadata on its own. Non-fatal, since a refresh failure still leaves whatever +# index is already there, and the install's own error is the more actionable one. +if command -v apt-get >/dev/null 2>&1; then + apt-get update -qq || echo "warning: apt-get update failed; continuing" >&2 +fi + +"$uvr_bin" sync --library "$target_lib" --install-system-deps diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index 6b32037..e7ca776 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -7,7 +7,6 @@ options(error = function() { suppressPackageStartupMessages(library(data.table)) library(DBI, quietly = TRUE) library(RPostgres, quietly = TRUE) -library(s3fs, quietly = TRUE) library(jsonlite, quietly = TRUE) library(httr2, quietly = TRUE) @@ -50,8 +49,8 @@ cat(sprintf( )) # --------------------------------------------------------------------------- -# 1. Query PostgreSQL for known build failures (before s3fs init to avoid -# C++ pointer conflicts between s3fs/curl and RPostgres/libpq) +# 1. Query PostgreSQL for known build failures (before anything that uses curl, +# to avoid C++ pointer conflicts between curl and RPostgres/libpq) # --------------------------------------------------------------------------- cat("Connecting to PostgreSQL...\n") con <- DBI::dbConnect( @@ -97,62 +96,71 @@ cran_dt <- data.table( ) # --------------------------------------------------------------------------- -# 3. S3 tarballs +# 3. Published binaries # --------------------------------------------------------------------------- -cat("Connecting to S3...\n") -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE -) - -s3_path <- sprintf( - "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", +# Read the slot's own index rather than listing the bucket. An object being +# present does not mean a binary was built: when a build fails, bincraft +# publishes the CRAN source tarball in its place so the package stays +# installable, and a bucket listing cannot tell the two apart. That is how +# amd64/alpine324 came to hold 13,547 CRAN sources that this audit never +# reported. bincraft stamps `Built` only on records it actually built, so the +# index answers the question a listing cannot. +# +# A slot last indexed by a bincraft that predates the source-fallback fix +# stamps `Built` on every record, including the fallbacks, so this reports +# exactly what it used to until that slot is re-indexed. +index_url <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", arch, s3_codename ) -cat(sprintf("Listing S3 path: %s\n", s3_path)) +cat(sprintf("Reading package index: %s\n", index_url)) -s3_pkgs <- tryCatch( - s3fs::s3_dir_ls(s3_path, recurse = FALSE), +index <- tryCatch( + { + con <- gzcon(url(index_url, open = "rb")) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "Built")) + }, error = function(e) { cat(sprintf( - "WARNING: Could not list S3 path %s: %s\n", - s3_path, + "WARNING: Could not read %s: %s\n", + index_url, conditionMessage(e) )) - character(0) + NULL } ) -file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) -parts <- parts[sapply(parts, length) == 3] -if (length(parts) == 0) { - s3_dt <- data.table(Package = character(0), Version = character(0)) +if (is.null(index) || nrow(index) == 0) { + binary_dt <- data.table(Package = character(0), Version = character(0)) } else { - s3_dt <- data.table( - Package = sapply(parts, `[`, 2), - Version = sapply(parts, `[`, 3) + built <- !is.na(index[, "Built"]) + binary_dt <- data.table( + Package = as.character(index[built, "Package"]), + Version = as.character(index[built, "Version"]) ) + cat(sprintf( + "Index holds %d records, %d of them built binaries (%d served as CRAN source)\n", + nrow(index), + sum(built), + sum(!built) + )) } cat(sprintf( - "S3 contains %d tarballs for %s/%s\n", - nrow(s3_dt), + "S3 contains %d binaries for %s/%s\n", + nrow(binary_dt), arch, s3_codename )) # --------------------------------------------------------------------------- -# 4. Find missing packages (CRAN release version not in S3) +# 4. Find missing packages (CRAN release version without a binary in S3) # --------------------------------------------------------------------------- setkey(cran_dt, Package, Version) -setkey(s3_dt, Package, Version) -missing_dt <- cran_dt[!s3_dt] +setkey(binary_dt, Package, Version) +missing_dt <- cran_dt[!binary_dt] cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) # --------------------------------------------------------------------------- @@ -328,7 +336,6 @@ if (nchar(forgejo_token) == 0) { } plat_header <- sprintf("## %s", platform) - arch_header <- sprintf("### %s", arch) plat_idx <- which(lines == plat_header) @@ -359,31 +366,40 @@ if (nchar(forgejo_token) == 0) { } plat_lines <- lines[seq(pi, plat_end)] - arch_local_idx <- which(plat_lines == arch_header) - if (length(arch_local_idx) == 0) { - # Append arch subsection at end of platform block - lines <- c( - lines[seq_len(plat_end)], - "", - arch_lines, - lines[seq(plat_end + 1, length(lines))] - ) + # Arch subsection headers within the platform block (### arm64 / ### amd64). + # Match by prefix: headers carry a " (N missing, M to rebuild)" suffix, so + # exact-equality matching never found the existing block and silently + # appended a duplicate on every run. Remove *all* blocks for this arch + # (collapsing any previously accumulated duplicates), then write one fresh + # block, so the issue holds a single current subsection per arch. + sub_hdr <- which(grepl("^### ", plat_lines)) + arch_re <- sprintf("^### %s( |$)", arch) + + if (length(sub_hdr) == 0) { + # No arch subsections yet -- append after the platform header/preamble. + new_plat_lines <- c(plat_lines, "", arch_lines) } else { - ai <- pi + arch_local_idx[1] - 1 # absolute line index - - # End of arch subsection - next_arch <- which( - grepl("^### |^## |^---", lines) & seq_along(lines) > ai - ) - arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end - - lines <- c( - lines[seq_len(ai - 1)], - arch_lines, - lines[seq(arch_end + 1, length(lines))] - ) + preamble <- plat_lines[seq_len(sub_hdr[1] - 1)] + # Each subsection runs from its ### header to the line before the next + # ### header (#### known-failures stays inside its own block). + sub_end <- c(sub_hdr[-1] - 1, length(plat_lines)) + kept <- character(0) + for (k in seq_along(sub_hdr)) { + block <- plat_lines[seq(sub_hdr[k], sub_end[k])] + if (!grepl(arch_re, block[1])) { + kept <- c(kept, block) + } + } + new_plat_lines <- c(preamble, kept, "", arch_lines) } + + tail_lines <- if (plat_end < length(lines)) { + lines[seq(plat_end + 1, length(lines))] + } else { + character(0) + } + lines <- c(lines[seq_len(pi - 1)], new_plat_lines, tail_lines) } # Rebuild excluded footer diff --git a/plans/2026-08-07-per-minor-edge-routing.md b/plans/2026-08-07-per-minor-edge-routing.md new file mode 100644 index 0000000..3c3e320 --- /dev/null +++ b/plans/2026-08-07-per-minor-edge-routing.md @@ -0,0 +1,166 @@ +# Per-R-minor edge routing implementation plan + +Spec: `specs/2026-08-07-per-minor-edge-routing-design.md` + +**Goal:** let a stock `install.packages()` see the per-minor packages by routing `PACKAGES*` requests to `…/src/contrib//`, where `bincraft` publishes a union index. + +**Architecture:** the union is built in `bincraft`; the edge script only redirects index requests, gated on a `UNION_SLOTS` script variable; the script lives in this repo and is applied by OpenTofu. + +**Tech stack:** Deno / TypeScript (Bunny Edge Scripting, SDK 0.12), OpenTofu with `BunnyWay/bunnynet` 0.17, R (bincraft). + +## Global constraints + +- Redirect only `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds`; never a tarball, because the union index already carries the correct tarball URL for both classes of package. +- Every redirect carries `Cache-Control: no-store`; redirect targets stay UA-independent. +- `UNION_SLOTS` is empty by default, so deploying the script is a no-op until a slot is backfilled. +- A slot is `/`, e.g. `amd64/alpine324`. +- Verified prerequisites: `PACKAGES*` is served `cdn-cache: BYPASS`, so the script sees every index request; `Deno.env.get()` reads script variables; the SDK local server listens on `127.0.0.1:8080`. + +--- + +## Task 1: Edge script and its test matrix + +**Files:** + +- Create: `edge/rpkgs-router.ts` +- Create: `edge/rpkgs-router.test.ts` +- Modify: `justfile` (add `edge-test`) + +**Produces:** a single-file script deployable as `bunnynet_compute_script.content`, reading `UNION_SLOTS` from the environment. + +- [ ] **Step 1: write the test matrix first** + +`edge/rpkgs-router.test.ts` spawns `deno run -A edge/rpkgs-router.ts` with `UNION_SLOTS=amd64/alpine324`, waits for `127.0.0.1:8080`, and issues requests with `redirect: "manual"`. + +Cases, asserted on the `location` header (or its absence): + +| # | path | User-Agent | expectation | +| --- | ------------------------------------------------------ | --------------------------------------------- | ------------------------------------------------------------ | +| 1 | `/amd64/alpine324/latest/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)` | 302 → `…/src/contrib/4.5/PACKAGES.gz` | +| 2 | same | `R (4.6.0 …)` | 302 → `…/src/contrib/4.6/PACKAGES.gz` | +| 3 | same, but slot `amd64/noble` | `R (4.5.3 …)` | no redirect (slot not in `UNION_SLOTS`) | +| 4 | `…/src/contrib/curl_7.1.0.tar.gz` | `R (4.5.3 …)` | no redirect | +| 5 | `…/src/contrib/4.5/PACKAGES.gz` | `R (4.5.3 …)` | no redirect (loop guard) | +| 6 | `…/src/contrib/PACKAGES.gz` | `curl/8.0` | no redirect (no R minor) | +| 7 | `/src/contrib/PACKAGES.gz` | alpine UA with `Alpine Linux … 3.24` | 302 → `/amd64/alpine324/latest/src/contrib/4.5/PACKAGES.gz` | +| 8 | `/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)`, no distro | 302 → `cran.r-project.org`, **not** a `linux-musl` slot | +| 9 | `/src/contrib/foo_1.0.tar.gz` | `R (4.5.1 aarch64-apple-darwin20 …)` | 302 → `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | +| 10 | `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | any | 302 → `cran.r-project.org` | +| 11 | any redirect above | — | `cache-control: no-store` | + +- [ ] **Step 2: run the tests and watch them fail** + +`just edge-test` → every case fails, because `edge/rpkgs-router.ts` does not exist. + +- [ ] **Step 3: write `edge/rpkgs-router.ts`** + +Order of evaluation in `onOriginRequest`: + +1. normalise `//` runs in the path +2. darwin `/src/contrib/*` → `/bin/macosx//contrib//` +3. `/bin/macosx/**` → CRAN +4. `/{arch}/{os}/latest/src/contrib/`: pass through if `rest` already starts with `/`, or is not an index file, or the slot is not in `UNION_SLOTS`, or the UA has no R minor; otherwise redirect into `/` +5. `/`, `/src/contrib`, `/src/contrib/**`: resolve arch+os from the UA, redirect to CRAN when the distro is unidentifiable, otherwise redirect to the qualified path, adding `/` under the same index-file rule +6. anything else: pass through + +The R minor comes from either `R/4.5.3` or `R (4.5.3 …)`, so a stock UA is enough. The `linux-gnu` / `linux-musl` fallback in `parseUserAgent` is deleted: those are not slot names. + +- [ ] **Step 4: run the tests until they pass** + +`just edge-test` + +- [ ] **Step 5: commit** + +```bash +git add edge/rpkgs-router.ts edge/rpkgs-router.test.ts justfile +git commit -m "feat(edge): route PACKAGES requests to the per-R-minor slot" +``` + +--- + +## Task 2: Manage the script from OpenTofu + +**Files:** + +- Modify: `cdn.tf` + +**Consumes:** `edge/rpkgs-router.ts` from Task 1. + +- [ ] **Step 1: add the resources** + +```terraform +resource "bunnynet_compute_script" "rpkgs_router" { + type = "middleware" + name = "rpkgs-router" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + default_value = "" + required = false +} +``` + +and replace `middleware_script = 29277` with `middleware_script = bunnynet_compute_script.rpkgs_router.id`. + +- [ ] **Step 2: validate** + +`tofu init -backend=false && tofu validate` + +- [ ] **Step 3: import the existing script (needs `BUNNYNET_API_KEY`)** + +```bash +tofu import bunnynet_compute_script.rpkgs_router 29277 +tofu plan +``` + +The plan must show an in-place `content` update and **no** replacement of the pull zone. A replacement means the import did not take. + +- [ ] **Step 4: commit** + +```bash +git add cdn.tf +git commit -m "feat(cdn): manage the edge middleware script from this repo" +``` + +--- + +## Task 3: Union index writer in bincraft + +**Files (repo `codefloe.com/rpkgs/bincraft`):** + +- Modify: `R/package_index.R` +- Test: `tests/testthat/test-package_index.R` + +**Produces:** `write_union_index(flat_records, minor_records)` returning the merged records, called from `upload_package_index()` when `r_minor` is set. + +- [ ] **Step 1: write the failing tests** + +- a package present in both slots keeps the per-minor record, with `Path = "4.5"` +- a package only in the flat slot survives with no `Path` +- a package only in the per-minor slot survives with `Path = "4.5"` +- a union smaller than the flat input raises an error rather than returning + +- [ ] **Step 2: run them and watch them fail** + +`Rscript -e 'testthat::test_file("tests/testthat/test-package_index.R")'` + +- [ ] **Step 3: implement `write_union_index()` and call it from `upload_package_index()`** + +After `update_PACKAGES()` has written the per-minor index, read the flat slot's `PACKAGES.rds`, set `Path = ` on the per-minor records, drop the flat records for packages the per-minor slot already has, and rewrite `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` in the per-minor slot. + +- [ ] **Step 4: run the tests until they pass** + +- [ ] **Step 5: commit and open the PR against bincraft** + +--- + +## Task 4: Roll out slot by slot + +- [ ] Re-index one slot (`amd64/alpine324`, R 4.5) and confirm the union index lists both `curl` (per-minor, `Path: 4.5`) and `jsonlite` (flat, no `Path`). +- [ ] Set `UNION_SLOTS = "amd64/alpine324"` and confirm in `reg.devxy.io/r/r-alpine:4.5-3.24` that `available.packages()` returns the union count and `"curl" %in% rownames(...)`. +- [ ] Add `arm64/alpine324`, then the remaining slots. + +`install.packages("curl")` will still fail to build on `alpine324` until that slot's source tarballs are replaced with real binaries. That is tracked separately. diff --git a/provider.tf b/provider.tf index b267fcc..d4f2564 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.15" + version = "~> 0.18" } } } diff --git a/renovate.json b/renovate.json index 53c7306..7b2c3b3 100644 --- a/renovate.json +++ b/renovate.json @@ -1,7 +1,12 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": ["local>devxy/renovate-config"], - "ignorePaths": ["docker/**"], + "ignorePaths": [ + "docker/**", + ".crow/process-updates.yaml", + ".crow/build-all-versions.yaml", + ".crow/weekly-rebuild-missing.yaml" + ], "customManagers": [ { "customType": "regex", @@ -14,6 +19,13 @@ "packageNameTemplate": "https://codefloe.com/rpkgs/bincraft.git", "datasourceTemplate": "git-tags", "extractVersionTemplate": "^v?(?.+)$" + }, + { + "customType": "regex", + "fileMatch": ["^local/uvr-install\\.sh$"], + "matchStrings": [ + "# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+UVR_PIN=\"(?[^\"]+)\"" + ] } ] } diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh new file mode 100755 index 0000000..648dfc3 --- /dev/null +++ b/scripts/purge_cdn_zone.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Purge the entire BunnyCDN pull zone. +# +# `purge_cdn_cache.sh` purges the five index files by URL, which is right after +# a normal update: new packages arrive at new URLs, so only the index is stale. +# +# A rebuild is different. It replaces an object *in place*: a package whose +# build failed was published as its CRAN source, and the rebuilt binary takes +# exactly the same URL. The zone caches tarballs for ~370 days +# (`cache_expiration_time` in cdn.tf), so without a purge every client keeps +# receiving the source tarball for up to a year, and nothing about it looks +# wrong from the outside. +# +# Purging per URL would mean one API call per replaced package -- ~13.5k per +# arch against a rate-limited endpoint, where a single missed call leaves a +# silently stale package. One zone purge is a single call regardless of how many +# objects were replaced. The cost is a cold cache for everything else, which is +# why this is not used by the daily update path. +# +# The public hostnames currently use separate pull zones, so callers must pass +# every zone that serves the repository. A zone can be identified by its +# numeric ID or by one of its hostnames; hostname lookup avoids persisting IDs +# that change when a zone is recreated. +# +# Usage: +# purge_cdn_zone.sh [...] +# +set -euo pipefail + +if (($# < 2)); then + echo "usage: $0 [...]" >&2 + exit 2 +fi + +api_key="$1" +shift + +resolve_zone_id() { + local zone="$1" + local response_file + local zone_id + + if [[ "${zone}" =~ ^[0-9]+$ ]]; then + echo "${zone}" + return + fi + + response_file=$(mktemp) + curl -sS -o "${response_file}" \ + -H "AccessKey: ${api_key}" \ + "https://api.bunny.net/pullzone" + zone_id=$( + jq -r --arg hostname "${zone}" \ + '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \ + "${response_file}" + ) + rm -f "${response_file}" + + if [[ -z "${zone_id}" ]]; then + echo "Could not find BunnyCDN pull zone for hostname ${zone}" >&2 + exit 1 + fi + + echo "${zone_id}" +} + +for zone in "$@"; do + zone_id=$(resolve_zone_id "${zone}") + echo "Purging BunnyCDN pull zone ${zone_id}" + + response_file="/tmp/purge_zone_response_${zone_id}.txt" + status=$( + curl -sS -o "${response_file}" -w '%{http_code}' -X POST \ + -H "AccessKey: ${api_key}" \ + -H "Content-Length: 0" \ + "https://api.bunny.net/pullzone/${zone_id}/purgeCache" + ) + + if [[ "${status}" != "200" && "${status}" != "204" ]]; then + echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 + cat "${response_file}" >&2 + exit 1 + fi + + echo "Purged pull zone ${zone_id} (HTTP ${status})" +done diff --git a/specs/2026-08-07-per-minor-edge-routing-design.md b/specs/2026-08-07-per-minor-edge-routing-design.md new file mode 100644 index 0000000..b55812c --- /dev/null +++ b/specs/2026-08-07-per-minor-edge-routing-design.md @@ -0,0 +1,174 @@ +# Design: Routing clients to per-R-minor binary slots + +Date: 2026-08-07 +Status: Approved (pending spec review) + +## Problem + +`bincraft` routes ABI-"risky" packages to a per-minor slot `…/latest/src/contrib//` and indexes every directory independently (`upload_package_index()` calls `cranlike::update_PACKAGES()` on one prefix at a time). +Nothing unions those indices, and `contrib.url()` only ever yields `/src/contrib`, so no value of `options(repos)` can address a per-minor slot. +Only `uvr` resolves per-minor URLs, which means the per-minor slots are invisible to `install.packages()` by construction. + +Measured on 2026-08-07: + +| slot | flat `src/contrib` | `src/contrib/4.5` | unique packages only in the per-minor slot | +| ----------------- | ------------------ | ----------------- | ------------------------------------------ | +| `amd64/alpine324` | 21 640 | 3 310 | 2 886 | +| `amd64/noble` | 24 495 | 398 | 23 | + +This is what issue #63 records as "missing binaries" on `alpine324`. +The packages are not missing; they are in a directory base R cannot reach. +The user-visible symptom in `reg.devxy.io/r/r-alpine:4.5-3.24` is: + +``` +> install.packages("curl") +Warning message: +package 'curl' is not available for this version of R +``` + +A second, unrelated defect exists on the same slot and is **out of scope here**: many `alpine324` tarballs are byte-identical CRAN _source_ tarballs that the index nevertheless stamps `Built: R 4.5.3; …-linux-musl`. +Routing exposes `curl`; only a rebuild of that slot makes it install. + +## Goal + +Let a stock `install.packages()` see one complete package list for its own R minor, without duplicating tarballs and without an R-version-varying cache key anywhere in the CDN. + +## Key constraint that drives the design + +R resolves a package's download URL from the index, not from the request path, and it keeps the `contriburl` it _asked for_ rather than the one it was redirected to. +Measured with `options(repos = …/latest)` against a middleware that redirects the index into `4.5/`: + +``` +curl available: TRUE +curl repo: …/latest/src/contrib # the flat URL, not the 4.5 one it was served from +``` + +So the union index is always addressed relative to the **flat** directory, whatever path it was fetched from. +`available.packages()` honours a `Path:` field and folds it into the `Repository` column, which gives the whole routing for free: + +- a per-minor record carries `Path: `, so its tarball is fetched from `…/src/contrib//` +- a flat record carries no `Path`, so its tarball is fetched from `…/src/contrib/` + +Verified end to end against the live CDN with a locally built union index for `amd64/alpine324` (31 507 records): + +``` +curl: 7.1.0 -> …/latest/src/contrib/4.5 -> curl_7.1.0.tar.gz 717 725 B +jsonlite: 2.0.0 -> …/latest/src/contrib -> jsonlite_2.0.0.tar.gz 1 055 849 B +``` + +The corollary is that the edge script must **not** rewrite tarball requests: every tarball URL is already correct when it leaves the client, and redirecting one into `/` would break exactly the flat packages the union is meant to preserve. + +The complementary trick does not work: R's `gzcon()` reads only the first member of a concatenated gzip stream (10 291 of an expected 31 931 records), so an edge-side merge would have to fully decompress and recompress both indices and additionally 404 `PACKAGES.rds` to stop R preferring it. +That is why the union is produced in `bincraft`, not at the edge. + +## Approaches considered + +| Approach | Where the union lives | Verdict | +| ----------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- | +| **A. Union index written by `bincraft` (chosen)** | per-minor `PACKAGES*`, per-minor entries carry `Path: ` | Edge does one redirect; `PACKAGES.rds` stays correct; no duplication | +| B. Merge at the edge | middleware fetches both indices, recompresses | ~2 MB decompress/recompress per cache fill, cache key must include the R minor, breaks R's `.rds` fast path | +| C. Move the minor up the path (`latest//src/contrib/`) | addressable by `options(repos)` directly | No edge logic at all, but a full layout migration and breaks the published URL contract | + +Chosen: **A**. + +## Architecture + +### bincraft: union index (separate PR) + +After writing a per-minor index, republish it as a union of that slot and the flat slot: + +1. Read the flat slot's `PACKAGES.rds` and the per-minor slot's own records. +2. Set `Path: ` on every per-minor record, so its tarball resolves into the per-minor directory. +3. Drop every flat record whose package is already present in the per-minor slot, so the per-minor build always wins, and leave the survivors without a `Path`. +4. Write the merged `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib//`. + +Guard: refuse to publish a union with fewer records than the flat index it was built from. +A truncated union is worse than no union, because it silently removes packages from every client on that minor. + +### Edge script (this repo) + +The script routes `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` requests, and nothing else. + +``` +normalize path +parseClient(UA) -> { rMinor, arch, os } # rMinor from "R (4.5.3 …)" or "R/4.5.3" +darwin branches # unchanged +if path is /{arch}/{os}/latest/src/contrib/PACKAGES* + already under // ? pass through # loop guard + rMinor known && slot in UNION_SLOTS ? 302 -> …/src/contrib//PACKAGES* + else pass through # flat slot, today's behaviour +if path is /src/contrib/… # bare root + resolve arch + os; unknown -> 302 to CRAN + then apply the same PACKAGES* rule +else pass through +``` + +Redirects carry `Cache-Control: no-store`. +Every cacheable URL is therefore UA-independent, and no cache key has to vary by R version. + +### Repaired bare-root detection + +The bare `https://cran.rpkgs.com` form is currently broken for every Linux client that uses a stock R user agent. +`ALPINE_REGEX`, `UBUNTU_REGEX` and `RHEL_REGEX` only match a Posit-style user agent the user has to set by hand; stock R never carries the distro, so the script falls through to `extractOs()` and redirects to a slot that does not exist: + +``` +UA: R (4.5.3 x86_64-pc-linux-musl …) -> 302 /amd64/linux-musl/latest/… (404) +UA: R (4.5.3 x86_64-pc-linux-gnu …) -> 302 /amd64/linux-gnu/latest/… (404) +``` + +The fallback to a phantom `linux-musl` / `linux-gnu` slot is removed. +An unidentifiable distro redirects to CRAN, which is the existing behaviour for an unparseable user agent. +The R _minor_ is always present in a stock user agent, so per-minor routing itself does not depend on distro detection. + +### Rollout gate + +`UNION_SLOTS` is a `bunnynet_compute_script_variable` listing the slots whose per-minor index is already a union. +It is empty by default, so deploying the script changes nothing until `bincraft` has backfilled a slot, and a rollback is a variable edit rather than a code deploy. +All slots currently carry `4.4`, `4.5` and `4.6`; a client on any other minor falls through to the flat slot. + +### Deployment from this repo + +The script is a file in the repo, applied by the existing OpenTofu configuration: + +``` +edge/rpkgs-router.ts # the script +edge/rpkgs-router.test.ts # UA x path -> expected Location matrix +cdn.tf # bunnynet_compute_script + _variable +``` + +Provider `BunnyWay/bunnynet` v0.17.0 (already pinned) ships `bunnynet_compute_script` with `content` loadable via `file()`, plus `bunnynet_compute_script_variable`. +`middleware_script = bunnynet_compute_script.rpkgs_router.id` replaces the hard-coded `29277`, after a one-time `tofu import` of the existing script. + +## Error handling + +- Unknown R minor, or a slot not listed in `UNION_SLOTS`: pass through to the flat slot. + The client sees exactly today's behaviour. +- Unparseable distro on the bare-root form: redirect to CRAN. +- A request already under `…/src/contrib//`: pass through, so a redirect can never loop. +- A per-minor slot that does not exist for a listed minor: the client gets the origin's 404. + `UNION_SLOTS` is the operator's assertion that the slot is ready, so this is a configuration error, not a runtime condition to paper over. + +## Testing + +Local, before any apply: `deno run -A edge/rpkgs-router.ts` serves the middleware against the real origin, so `edge/rpkgs-router.test.ts` drives the whole matrix against that local server. + +- User agent matrix: R 4.4 / 4.5 / 4.6 on musl and gnu, both arches, Posit-style and stock forms, plus a darwin UA and a non-R UA. +- Path matrix: `PACKAGES`, `PACKAGES.gz`, `PACKAGES.rds`, a tarball, a path already under `4.5/`, and `/src/contrib/…` on the bare root. +- Assertion is the `Location` header (or its absence), not the body. + +After apply, a smoke test against `cran.rpkgs.com`: + +- `available.packages()` inside `reg.devxy.io/r/r-alpine:4.5-3.24` returns the union count, and `"curl" %in% rownames(...)` is `TRUE`. +- A flat-slot package still downloads from `…/src/contrib/`, and a per-minor package downloads from `…/src/contrib//`. + +## Out of scope + +- `Meta/archive.rds` stays flat-only, so `remotes::install_version()` does not see per-minor archives. +- The `alpine324` source-tarball defect: that slot serves CRAN sources stamped as binaries, and needs a rebuild independent of this work. +- Any change to how `uvr` resolves per-minor URLs; it already addresses the slots directly. + +## Split of work + +1. `bincraft`: union index writer plus its guard, and a re-index of one slot to validate. +2. This repo: `edge/rpkgs-router.ts`, its test matrix, and the `cdn.tf` resources with `UNION_SLOTS` empty. +3. Enable `UNION_SLOTS` slot by slot as `bincraft` backfills them. diff --git a/specs/2026-08-12-shard-weekly-rebuild-design.md b/specs/2026-08-12-shard-weekly-rebuild-design.md new file mode 100644 index 0000000..f4ccbaa --- /dev/null +++ b/specs/2026-08-12-shard-weekly-rebuild-design.md @@ -0,0 +1,167 @@ +# Design: Sharding and resuming the weekly rebuild + +Date: 2026-08-12 +Status: Approved (pending spec review) + +## Problem + +`weekly-rebuild-missing` runs one job per `-` and walks that slot's rebuild list serially in a single `R -q -e` invocation (`.crow/weekly-rebuild-missing.yaml:165`). +Until 2026-08-09 that was cheap, because every source fallback was skipped as "already built" and the list was effectively empty. +Since bincraft #105/#106/#107 and build-cran-binaries #159 the gate works, and the lists are now large. + +Share of records whose object is byte-identical to CRAN's source, measured against `cran.r-project.org` MD5s on 2026-08-12: + +| slot | records | source-served | share | +| ------------------ | ------: | ------------: | -----------: | +| `amd64/resolute` | 24 212 | 15 023 | 62.1% | +| `arm64/resolute` | 24 291 | 13 670 | 56.3% | +| `arm64/alpine324` | 24 328 | 9 514 | 39.2% | +| `amd64/alpine324` | 24 343 | 8 917 | 36.7% | +| `arm64/rhel10` | 24 695 | 5 384 | 21.9% | +| `amd64/rhel10` | 24 881 | 4 712 | 19.2% | +| 12 remaining slots | ~24 700 | 850 to 2 130 | 3.5% to 8.7% | + +A single serial job cannot absorb that. +Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) started on 2026-08-09, ran for roughly two days, reached `[8692/23885] cholera`, and was killed there. + +Two distinct failures follow from that shape. + +**No parallelism.** The work is embarrassingly parallel across packages, but one job does all of it. + +**No resumability, and no clean stopping point.** The loop has no terminating condition other than exhausting the list, so the only way to stop it is a kill. +A restarted run re-reads the same list and walks it from the first entry. +It skips completed packages via `check_s3_root_package()`, but that costs a CRAN version resolution and an S3 `HEAD` per package, thousands of times, before it reaches new work. +Worse, a kill is not a pipeline failure: the `Purge CDN cache` step is guarded by `when: status: [success, failure]` (`.crow/weekly-rebuild-missing.yaml:206-207`), and on 10910 it produced no output at all. +So the ~4 600 binaries that run did publish stayed hidden behind stale edge copies. + +## Goal + +Turn each slot's rebuild into bounded, parallel, restartable units, without introducing state that can disagree with the bucket. + +## Design + +### 1. Shard the matrix three ways + +Each of the 18 `OS`/`ARCH` rows in `.crow/weekly-rebuild-missing.yaml` gains `SPLIT_INTO: 3` and `SPLIT_INDEX: 1|2|3`, giving 54 rows. +This mirrors `.crow/build-all-versions.yaml:57-98`, which already shards its matrix four ways per arch. + +Routing needs no change. +The cron filter `cron: weekly-rebuild-missing-${OS}-${ARCH}` and the manual `evaluate: weekly_rebuild_missing == "${OS}-${ARCH}"` both match all three shards of a slot. +Placement stays on the `rpkgs-${ARCH}` group label, so shards queue against available capacity rather than oversubscribing it. + +### 2. Extract the loop into `local/rebuild-missing.R` + +The build is currently a single ~1 500-character `R -q -e` argument. +Shard arithmetic and resume logic do not belong in a YAML string, and none of it is testable there. +The loop moves to `local/rebuild-missing.R`, invoked as `Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX`, mirroring `local/build-all.R`. +Its body is unchanged in substance: read `/tmp/rebuild_pkgs.txt`, subtract `local/excluded-packages.json`, loop with `tryCatch` around `bincraft::build_binary_package()`. + +The slice is **interleaved**, not contiguous: + +```r +# the list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, +# rstan*), so contiguous thirds would be badly unbalanced +mine <- pkgs[seq(split_index, length(pkgs), by = split_into)] +``` + +`local/build-all.R:64` uses contiguous chunks via `cut()`. +That is fine there because its list is every CRAN package and version, so the chunks average out. +Here the list is a filtered backlog in which expensive families sit adjacent, so interleaving is the better default. +Interleaving also makes each shard's `[i/n]` progress representative of the slot as a whole. + +### 3. Resume by re-deriving state from the bucket + +Before the loop, the shard performs one `s3fs::s3_dir_info()` on `devxy-rpkgs-binaries///latest/src/contrib` and reads the `etag` column. +It fetches CRAN's `PACKAGES` once for the latest version and published `MD5sum` of every package. +A package is still outstanding if and only if the object at `_.tar.gz` has an ETag equal to CRAN's `MD5sum` for that version, which is the definition `check_s3_root_package()` already applies one package at a time. + +```r +# one paginated listing instead of ~2900 sequential HEAD requests per shard +info <- s3fs::s3_dir_info(slot_dir) +etag <- setNames(gsub('^"|"$', "", info$etag), basename(info$uri)) + +key <- sprintf("%s_%s.tar.gz", mine, cran_version[mine]) +# keep a package when no object exists yet, or when the object is still +# byte-identical to CRAN's source; drop it once a real binary is published +mine <- mine[is.na(etag[key]) | etag[key] == cran_md5[key]] +``` + +This is the whole resume mechanism. +There is no progress file, no volume, and no database cursor. +A restarted shard recomputes ground truth and continues where it stopped, and it is correct even when a sibling shard, a `process-updates` cron, or a manual `just rebuild` completed something in the meantime. + +Three properties make this the right source of truth: + +- **It is what the build itself checks.** Any other store can disagree with the bucket; this one cannot. +- **It is agent-independent.** `.crow/weekly-rebuild-missing.yaml` mounts no `volumes:`, unlike `.crow/build-all-versions.yaml:132-133`, so `/mnt/cache` is per-job and cannot carry progress anyway. +- **It costs one listing.** `cranlike`'s `s3` fork already does exactly this call against this bucket at ~24 000 objects, so the approach is proven at the required scale. + +It must read ETags rather than the slot index's `Built` field, which is how `local/packages-to-build.R:104-130` answers the same question. +Under this design the index is not rewritten until the dependent re-index pipeline runs (section 5), so mid-run it cannot reflect the current run's progress. + +Packages that genuinely fail to build re-publish their CRAN source, so they stay outstanding and would be retried on every restart. +That is already handled upstream: `bincraft::filter_packages_with_errors()` (`R/build_binaries.R:1018`, `:1143`) drops anything with `error_occurred = TRUE`, and `store_build_metadata = TRUE` is passed on every call. +No additional poison-pill filter is needed here. + +Only the flat `src/contrib` path is considered. +The rebuild call passes no `is_r_minor_sensitive`, so it defaults to `FALSE` and only ever targets the flat path; the resume filter matches that scope deliberately. + +### 4. Give each shard a wall-clock budget + +`local/rebuild-missing.R` takes a budget, defaulting to 20 hours, and breaks out of the loop once it is exceeded: + +```r +# exit cleanly rather than being killed, so the dependent re-index still runs +if (difftime(Sys.time(), started, units = "hours") > budget_hours) { + cat(sprintf("Budget of %sh reached after %d/%d packages; stopping cleanly\n", budget_hours, i, n)) + break +} +``` + +It exits 0 and reports how much of the slice it covered. +Every run then has a terminating condition, the re-index and purge always fire, and the remainder is picked up by the next run with no bookkeeping, because section 3 recomputes the outstanding set from scratch. + +### 5. Move the re-index and purge into `.crow/weekly-rebuild-reindex.yaml` + +Three shards per slot means three concurrent `upload_package_index()` calls on the same S3 prefix. +`cranlike::update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work. +The re-index steps (`.crow/weekly-rebuild-missing.yaml:171-176`) and the purge step (`:187-207`) therefore leave that file entirely. + +The new file carries: + +```yaml +depends_on: + - weekly-rebuild-missing +runs_on: [success, failure] +``` + +`runs_on: [success, failure]` validates as a workflow-level key under `crow lint`, so a failing shard no longer withholds the re-index. +The file uses the same 18-row matrix and the same `when:` gating as `weekly-rebuild-missing`, so it only re-indexes slots that actually ran. +Each row re-indexes the flat slot and every per-minor slot. +`scripts/purge_cdn_zone.sh` runs once on a single row, because all hostnames share pull zone `3857050` and 18 identical zone purges would be waste. + +## Failure behaviour + +| case | today | after | +| -------------------------- | ----------------------------------- | ----------------------------------------------------- | +| one package errors | `tryCatch` logs, loop continues | unchanged | +| a shard fails outright | purge runs, re-index does not | re-index and purge run via `runs_on` | +| a shard exceeds its budget | cannot happen, runs until killed | exits 0, re-index and purge run | +| a shard is killed | nothing runs | still nothing; trigger the re-index pipeline alone | +| a shard restarts | re-walks the list, HEAD per package | one listing, resumes at the first outstanding package | + +The known cost of `depends_on` being file-level rather than row-level: on the weekly cron no slot is re-indexed until the slowest of all 54 jobs finishes. +The 20-hour budget bounds that at roughly one day. + +## Out of scope + +- `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row for the platform and arch, which is precisely the source-fallback set. That is a separate change. +- Bunny Perma-Cache eviction. `scripts/purge_cdn_zone.sh` purges the regular edge cache only; see the note in `CLAUDE.md` and issue history. +- The audit that produces the rebuild list is unchanged. + +## Verification + +- `crow lint .crow/` passes for both pipeline files. +- `local/rebuild-missing.R` gets unit coverage in `local/tests/` for the two pure pieces: the interleaved slice (disjoint, covering, deterministic) and the outstanding-set filter (source-served ETag kept, binary ETag dropped, absent object kept). +- A single-slot manual run of `alpine-324-amd64` shard 1 confirms the listing shortcut against the live bucket, and that the reported outstanding count is close to the 8 917 measured above divided by three. +- Restarting that shard mid-run confirms it resumes rather than replaying, by comparing the outstanding count it reports on the second start.