diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index bd127d6..71f808b 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -28,6 +28,10 @@ matrix: ARCH: amd64 - CODENAME: redhat-10 ARCH: arm64 + - CODENAME: alpine321 + ARCH: amd64 + - CODENAME: alpine321 + ARCH: arm64 - CODENAME: alpine322 ARCH: amd64 - CODENAME: alpine322 diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index 5d6e6fe..2ac6886 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -9,27 +9,15 @@ # FORGEJO_TOKEN is used for both the branch push and opening the PR (no separate # write-scoped secret needed). Register the `auto-apply-patches` cron in the crow # UI, or run manually: -# crow pipeline create --branch main \ -# --var auto_apply_patches=true devxy/build-cran-binaries -# -# The gate variable is `auto_apply_patches`, named after the pipeline: a manual -# run instantiates every pipeline in `.crow/`, so one without its own gate runs -# on *any* manual trigger in this repo. This one pushes a branch and opens a PR, -# so it must stay off unless it is what was asked for. +# woodpecker-cli pipeline create --var task=auto-apply-patches --branch=main 7 variables: - auto_apply_patches: - description: 'Run the auto-patch proposer. Also gates this pipeline.' - options: - - 'true' - - 'false' - default: 'false' patch_limit: description: 'Max candidates to propose per run (top by failure volume).' default: '10' when: - event: manual - evaluate: 'auto_apply_patches == "true"' + evaluate: 'task == "auto-apply-patches"' - event: cron cron: auto-apply-patches @@ -58,7 +46,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite + - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT backend_options: kubernetes: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 2a6d9c2..e0cca86 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -3,33 +3,29 @@ # Variables are declared so the manual-run form exposes them (crow #1165); # they are merged with build-all-versions' identical declarations. variables: - # Gates this pipeline. A manual pipeline creation instantiates every file in - # .crow/, and a declared default is applied even when the run never passed - # this variable, so the default must be a value that matches no matrix row. target_arch: - description: 'Architecture to build, or "none" to run nothing.' + description: 'Architecture to build.' options: - - none - amd64 - arm64 - default: none + default: amd64 OS: - description: 'Base OS image name.' + description: "Base OS image name." options: - alpine - redhat - ubuntu default: alpine OS_VERSION: - description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' + description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." options: - - '3.24' - - '8' - - '9' - - '10' - - 'jammy' - - 'noble' - default: '3.24' + - "3.24" + - "8" + - "9" + - "10" + - "jammy" + - "noble" + default: "3.24" R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -70,24 +66,23 @@ steps: from_secret: B2_S3_SECRET_KEY PGPASS: from_secret: PGPASS + R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs - # Keep uvr's downloads and extracted-package entries on the persistent - # volume instead of the container-local ~/.uvr default. - UVR_CACHE_DIR: /mnt/cache/uvr/cache - UVR_PACKAGES_DIR: /mnt/cache/uvr/packages CCACHE_DIR: /mnt/cache/ccache volumes: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: # one-time full wipe to fix corrupted .so files from previous failed builds # - rm -rf /mnt/cache/R-pkgs - - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # Clear churny pkgcache metadata left by a prior crashed run (the "patched" + # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). + # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. + - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index fb1229d..3932811 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -4,36 +4,32 @@ # image and cache volume. Placement is via the group label (rpkgs-amd64/rpkgs-arm64). # Skip list lives in local/excluded-packages.json (read by local/build-all.R). variables: - # Gates this pipeline. A manual pipeline creation instantiates every file in - # .crow/, and a declared default is applied even when the run never passed - # this variable, so the default must be a value that matches no matrix row. target_arch: - description: 'Architecture to build, or "none" to run nothing.' + description: 'Architecture to build.' options: - - none - amd64 - arm64 - default: none + default: amd64 OS: - description: 'Base OS image name.' + description: "Base OS image name." options: - alpine - redhat - ubuntu default: alpine OS_VERSION: - description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' + description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." options: - - '3.22' - - '3.23' - - '3.24' - - '8' - - '9' - - '10' - - 'jammy' - - 'noble' - - 'resolute' - default: '3.24' + - "3.22" + - "3.23" + - "3.24" + - "8" + - "9" + - "10" + - "jammy" + - "noble" + - "resolute" + default: "3.24" R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -51,49 +47,38 @@ labels: platform: linux/${ARCH} group: rpkgs-${ARCH} -# Empty UVR_CACHE_DIR/UVR_PACKAGES_DIR fall back to uvr's container-local -# ~/.uvr defaults; amd64 deliberately does not persist them (as with the -# pkgcache dir it replaces), arm64 does. matrix: include: - ARCH: amd64 - UVR_CACHE_DIR: '' - UVR_PACKAGES_DIR: '' + R_PKG_CACHE_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: amd64 - UVR_CACHE_DIR: '' - UVR_PACKAGES_DIR: '' + R_PKG_CACHE_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: amd64 - UVR_CACHE_DIR: '' - UVR_PACKAGES_DIR: '' + R_PKG_CACHE_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: amd64 - UVR_CACHE_DIR: '' - UVR_PACKAGES_DIR: '' + R_PKG_CACHE_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 4 - ARCH: arm64 - UVR_CACHE_DIR: /mnt/cache/uvr/cache - UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + R_PKG_CACHE_DIR: /mnt/cache/pkgcache SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: arm64 - UVR_CACHE_DIR: /mnt/cache/uvr/cache - UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + R_PKG_CACHE_DIR: /mnt/cache/pkgcache SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: arm64 - UVR_CACHE_DIR: /mnt/cache/uvr/cache - UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + R_PKG_CACHE_DIR: /mnt/cache/pkgcache SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: arm64 - UVR_CACHE_DIR: /mnt/cache/uvr/cache - UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + R_PKG_CACHE_DIR: /mnt/cache/pkgcache SPLIT_INTO: 4 SPLIT_INDEX: 4 @@ -122,10 +107,8 @@ steps: from_secret: GITHUB_PAT # normal env vars GIT_USER: pat-s - # set the location of uvr's caches, which persist the R package - # dependencies needed to install the packages themselves - UVR_CACHE_DIR: ${UVR_CACHE_DIR} - UVR_PACKAGES_DIR: ${UVR_PACKAGES_DIR} + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR} R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache NCPUS: 2 @@ -133,7 +116,12 @@ steps: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # Clear churny pkgcache metadata left by a prior crashed run (the "patched" + # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). + # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. Within-run + # growth is bounded separately by trim_pkgcache_metadata() in build-all.R. + - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # The primary pass must not rely on build-all-versions-install-deps having # run on *this* agent: depends_on only orders the steps, but the cache # volume is per-agent, so a job landing on an agent where install-deps did diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 73bfff7..3da41fd 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -3,17 +3,13 @@ # Routing is preserved 1:1: # - cron: each existing `process-cran-updates--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `process_cran_updates` dropdown; -# "all" fans out every os/arch as parallel matrix workflows. +# - manual: pick a target from the `process_cran_updates` dropdown +# ("all" = every os/arch). # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: - # Gates this pipeline. A manual pipeline creation instantiates every file in - # .crow/, and a declared default is applied even when the run never passed - # this variable, so the default must be a value that matches no matrix row. process_cran_updates: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + description: "Manual run target: a specific -, or 'all' for every os/arch." options: - - none - all - alpine-322-amd64 - alpine-322-arm64 @@ -33,7 +29,7 @@ variables: - ubuntu-2404-arm64 - ubuntu-2604-amd64 - ubuntu-2604-arm64 - default: none + default: all when: - event: cron @@ -51,111 +47,111 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.22 + IMG: alpine:3.24 OS_ID: alpine322 - PROCESS_NEW: 'FALSE' + PROCESS_NEW: "FALSE" - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.23 + IMG: alpine:3.24 OS_ID: alpine322 - PROCESS_NEW: 'FALSE' + PROCESS_NEW: "FALSE" - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.23 + IMG: alpine:3.24 OS_ID: alpine323 - PROCESS_NEW: 'FALSE' + PROCESS_NEW: "FALSE" - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.23 + IMG: alpine:3.24 OS_ID: alpine323 - PROCESS_NEW: 'FALSE' + PROCESS_NEW: "FALSE" - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine324 - PROCESS_NEW: 'FALSE' + PROCESS_NEW: "FALSE" - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine324 - PROCESS_NEW: 'FALSE' + PROCESS_NEW: "FALSE" - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: redhat-8 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: redhat-9 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: redhat-9 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: redhat-10 ARCH: amd64 R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: redhat-10 ARCH: arm64 R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: ubuntu-2204 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: ubuntu-2204 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: ubuntu-2404 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: ubuntu-2404 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: ubuntu-2604 ARCH: amd64 R_VERSION: 4.5.3 IMG: ubuntu:resolute OS_ID: resolute - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" - OS: ubuntu-2604 ARCH: arm64 R_VERSION: 4.5.3 IMG: ubuntu:resolute OS_ID: resolute - PROCESS_NEW: 'TRUE' + PROCESS_NEW: "TRUE" steps: - name: 'Processing Updates' @@ -186,10 +182,8 @@ steps: NTFY_AUTH: TRUE NTFY_PASSWORD: from_secret: ntfy_token - # set the location of uvr's caches, which persist the R package - # dependencies needed to install the packages themselves - UVR_CACHE_DIR: /mnt/cache/uvr/cache - UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache @@ -200,11 +194,10 @@ steps: INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft - - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise @@ -219,7 +212,6 @@ steps: LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true - R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml deleted file mode 100644 index e56e1e9..0000000 --- a/.crow/repair-built-stamp.yaml +++ /dev/null @@ -1,110 +0,0 @@ -### Manual repair of a slot whose PACKAGES index advertises a broken `Built` -### stamp (e.g. `Built: R 4.5.0; NA; ...`). -# -# uvr matches the stamp's platform triple plus R minor to decide binary vs -# source, so an unusable triple turns a whole slot source-only. See -# local/repair-built-stamp.R for why this patches PACKAGES.db in place instead -# of forcing a full reparse. -# -# Run with `dry_run: true` first: it reports how many entries are broken per -# slot and changes nothing. Pick the R version the slot should advertise, which -# is the R_VERSION its entry in .crow/process-updates.yaml uses. -# -# The gate variable is `repair_built_stamp`, not `target_arch`: `target_arch` is -# what build-all-versions and build-all-versions-install-deps gate on, so a -# manual run passing it would start a full rebuild alongside this repair. Every -# pipeline here gates on a variable named after itself for exactly that reason. -# -# crow pipeline create --branch main \ -# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \ -# --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries -variables: - # Gates this pipeline. A manual pipeline creation instantiates every file in - # .crow/, and a declared default is applied even when the run never passed - # this variable, so the default must be a value that matches no matrix row. - repair_built_stamp: - description: 'Architecture of the slot to repair, or "none" to run nothing.' - options: - - none - - amd64 - - arm64 - default: none - OS: - description: 'Base OS image name.' - options: - - alpine - - redhat - - ubuntu - default: alpine - OS_VERSION: - description: 'OS image tag. Must match OS (alpine: 3.22/3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).' - options: - - '3.22' - - '3.23' - - '3.24' - - '8' - - '9' - - '10' - - 'jammy' - - 'noble' - - 'resolute' - default: '3.22' - R_VERSION: - description: 'R version whose stamp the slot should advertise.' - options: - - 4.5.3 - - 4.4.3 - default: 4.5.3 - dry_run: - description: 'Report what would change without writing anything.' - options: - - 'true' - - 'false' - default: 'true' - -when: - - event: manual - evaluate: 'repair_built_stamp == "${ARCH}"' - -skip_clone: true - -labels: - platform: linux/${ARCH} - group: rpkgs-${ARCH} - -matrix: - include: - - ARCH: amd64 - - ARCH: arm64 - -steps: - - name: 'Repair Built stamp' - image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GIT_USER: pat-s - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - | - if [ "$dry_run" = "false" ]; then - /opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" --apply - else - /opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" - fi - backend_options: - docker: - resources: - requests: - memory: 2Gi - cpu: 1000m - limits: - memory: 8Gi - cpu: 2000m diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 2195423..1ac71e8 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -7,27 +7,16 @@ # # The repo uses no `pull_request` triggers, so this runs manually against the # branch (or on a cron); point it at the auto-patch branch via `patch_branch`: -# crow pipeline create --branch main --var trial_build_registry=true \ -# --var patch_branch=auto/registry-patch-proposals devxy/build-cran-binaries -# -# The gate variable is `trial_build_registry`, named after the pipeline: a -# manual run instantiates every pipeline in `.crow/`, so one without its own -# gate runs on *any* manual trigger in this repo. This one starts a build per -# matrix row on both arches, which is far too expensive to fire by accident. +# woodpecker-cli pipeline create --var task=trial-build-registry \ +# --var patch_branch=auto/registry-patch-proposals --branch=main 7 variables: - trial_build_registry: - description: 'Trial-build the branch new registry entries. Also gates this pipeline.' - options: - - 'true' - - 'false' - default: 'false' patch_branch: description: 'Branch whose new registry entries to trial-build.' default: auto/registry-patch-proposals when: - event: manual - evaluate: 'trial_build_registry == "true"' + evaluate: 'task == "trial-build-registry"' - event: cron cron: trial-build-registry @@ -41,19 +30,19 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.22 + IMG: alpine:3.24 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.22 + IMG: alpine:3.24 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.23 + IMG: alpine:3.24 - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.23 + IMG: alpine:3.24 - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 @@ -128,18 +117,9 @@ steps: ARCH: ${ARCH} R_VERSION: ${R_VERSION} R_LIBS_USER: /mnt/cache/R-pkgs - # Surface the real compiler error when an isolated patched build fails, - # instead of bincraft's opaque "System command 'R' failed" (needs bincraft - # with BINCRAFT_VERBOSE_PATCH_BUILD support; harmless on older versions). - BINCRAFT_VERBOSE_PATCH_BUILD: 'TRUE' commands: - # Clone main, then check out the auto-patch branch if it exists. When the - # proposer had no candidates it never (re)creates that branch, so a missing - # branch means "nothing to verify" -- no-op cleanly instead of failing the - # clone. - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - git clone -q --branch ${patch_branch} https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - git fetch -q origin main - - 'if git ls-remote --exit-code --heads origin ${patch_branch} >/dev/null 2>&1; then git fetch -q origin ${patch_branch} && git checkout -q FETCH_HEAD; else echo "No ${patch_branch} branch; no pending auto-patch proposals to verify."; exit 0; fi' - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 25875da..6ff71e3 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -3,24 +3,20 @@ # Routing is preserved 1:1: # - cron: each existing `weekly-audit-missing--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `weekly_audit_missing` dropdown; -# "all" fans out every os/arch as parallel matrix workflows. +# - manual: pick a target from the `weekly_audit_missing` dropdown +# ("all" = every os/arch). # Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64). variables: - # Gates this pipeline. A manual pipeline creation instantiates every file in - # .crow/, and a declared default is applied even when the run never passed - # this variable, so the default must be a value that matches no matrix row. weekly_audit_missing: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + description: "Manual run target: a specific -, or 'all' for every os/arch." options: - - none - all + - alpine-321-amd64 + - alpine-321-arm64 - alpine-322-amd64 - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -31,9 +27,7 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none + default: all when: - event: cron @@ -48,27 +42,27 @@ labels: matrix: include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-323 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-323 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-324 + - OS: alpine-321 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.24 - - OS: alpine-324 + - OS: alpine-321 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 @@ -112,14 +106,6 @@ matrix: ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: ubuntu:resolute steps: - name: 'Audit missing binaries' @@ -146,8 +132,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite + - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' backend_options: docker: diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 7ea6c97..0284e93 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -8,26 +8,8 @@ # Global across platforms (the classifier groups over all of single_builds), so # a single job -- no matrix. Clones read-only; the only writes are the two # Forgejo issues via FORGEJO_TOKEN. -# -# Run manually with: -# crow pipeline create --branch main \ -# --var weekly_patch_proposals=true devxy/build-cran-binaries -# -# The gate variable is `weekly_patch_proposals`, named after the pipeline: a -# manual run instantiates every pipeline in `.crow/`, so one without its own -# gate runs on *any* manual trigger in this repo. This one posts and edits -# Forgejo issues, so an unrelated manual run must not fire it. -variables: - weekly_patch_proposals: - description: 'Run the weekly failure triage. Also gates this pipeline.' - options: - - 'true' - - 'false' - default: 'false' - when: - event: manual - evaluate: 'weekly_patch_proposals == "true"' - event: cron cron: weekly-patch-proposals @@ -53,7 +35,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite + - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 9639792..934c518 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -1,36 +1,21 @@ # Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). -# Three matrix rows per OS/arch, one per shard of that slot's rebuild list. +# One matrix row per OS/arch replaces the former per-platform files. # Routing is preserved 1:1: # - cron: each existing `weekly-rebuild-missing--` cron fires only -# its matching matrix rows (via the per-row `cron:` name filter), -# which is now all three shards of that slot. -# - manual: pick a target from the `weekly_rebuild_missing` dropdown; -# "all" fans out every os/arch and shard as parallel matrix -# workflows, while a single - runs its three shards. +# its matching matrix row (via the per-row `cron:` name filter). +# - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the +# previous bare manual trigger that ran every os/arch); pick a +# single - to run just one. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). -# -# The shard picks up its own slice and re-derives what is still outstanding -# from the bucket, so a restart resumes rather than replaying; see -# local/rebuild-missing.R. -# -# Re-indexing and the CDN purge deliberately do NOT live here. Three shards -# writing one slot's PACKAGES concurrently would race, so they moved to -# .crow/weekly-rebuild-reindex.yaml, which depends on this pipeline. variables: - # Gates this pipeline. A manual pipeline creation instantiates every file in - # .crow/, and a declared default is applied even when the run never passed - # this variable, so the default must be a value that matches no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + description: "Manual run target: a specific -, or 'all' for every os/arch." options: - - none - all - alpine-322-amd64 - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - redhat-8-amd64 - redhat-8-arm64 - redhat-9-amd64 @@ -41,9 +26,7 @@ variables: - ubuntu-2204-arm64 - ubuntu-2404-amd64 - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none + default: all when: - event: cron @@ -61,327 +44,59 @@ matrix: - OS: alpine-322 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 + IMG: alpine:3.24 - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 + IMG: alpine:3.24 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 - IMG: alpine:3.23 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: alpine-323 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: alpine-323 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 + IMG: alpine:3.24 - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 - IMG: alpine:3.23 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: alpine-323 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: alpine-323 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 IMG: alpine:3.24 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:8 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: redhat-8 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:8 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: redhat-8 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:8 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: redhat-8 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:8 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: redhat-8 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:8 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: redhat-8 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:8 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: redhat-9 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:9 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: redhat-9 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:9 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: redhat-9 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:9 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: redhat-9 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:9 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: redhat-9 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:9 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: redhat-9 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:9 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: redhat-10 ARCH: amd64 R_VERSION: 4.5.3 IMG: redhat:10 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: redhat-10 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: redhat:10 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: redhat-10 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: redhat:10 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: redhat-10 ARCH: arm64 R_VERSION: 4.5.3 IMG: redhat:10 - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: redhat-10 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: redhat:10 - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: redhat-10 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: redhat:10 - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: ubuntu-2204 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:jammy - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2204 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2204 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: ubuntu-2204 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:jammy - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2204 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2204 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: ubuntu-2404 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:noble - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2404 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2404 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - OS: ubuntu-2404 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2404 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2404 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 3 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 1 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 2 - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - SPLIT_INTO: 3 - SPLIT_INDEX: 3 steps: - name: 'Rebuild missing binaries' @@ -406,30 +121,23 @@ steps: FORGEJO_TOKEN: from_secret: FORGEJO_TOKEN GIT_USER: pat-s - UVR_CACHE_DIR: /mnt/cache/uvr/cache - UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache PLATFORM: ${OS} ARCH: ${ARCH} NCPUS: 2 - SPLIT_INTO: ${SPLIT_INTO} - SPLIT_INDEX: ${SPLIT_INDEX} - # Wall clock after which the shard stops cleanly instead of having to be - # killed. A kill matches neither `success` nor `failure`, so it would skip - # the dependent re-index and leave rebuilt binaries behind a stale edge. - REBUILD_BUDGET_HOURS: 20 commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 + - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: docker: resources: diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml deleted file mode 100644 index 2e0f2a0..0000000 --- a/.crow/weekly-rebuild-reindex.yaml +++ /dev/null @@ -1,191 +0,0 @@ -# Re-index and purge after weekly-rebuild-missing. -# -# weekly-rebuild-missing runs three shards per slot. Each of them replaces -# objects in place, so the slot's index still advertises the old MD5 and, for -# anything that had been served from source, no Built stamp. Re-indexing from -# inside a shard would mean three concurrent `upload_package_index()` calls on -# one prefix: `cranlike::update_PACKAGES()` lists the live bucket, so an early -# lister that uploads last publishes an index missing its siblings' work. -# -# So it happens exactly once per slot, here, after every shard has finished. -# `runs_on: [success, failure]` keeps that true when a shard fails; only an -# explicit cancel skips it, and this pipeline can then be triggered on its own. - -variables: - # Mirrors the gate on weekly-rebuild-missing so a manual run re-indexes - # exactly the slots it rebuilt. A manual pipeline creation instantiates every - # file in .crow/, so the default must match no matrix row. - weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." - options: - - none - - all - - alpine-322-amd64 - - alpine-322-arm64 - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - -when: - - event: cron - cron: weekly-rebuild-missing-${OS}-${ARCH} - - event: manual - evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' - -depends_on: - - weekly-rebuild-missing - -runs_on: [success, failure] - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-323 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-323 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: redhat-8 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-8 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-9 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-9 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-10 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: redhat-10 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: ubuntu-2204 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2204 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2404 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2404 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - -steps: - - name: 'Re-index the slot' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GIT_USER: pat-s - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: ${R_VERSION} - PLATFORM: ${OS} - ARCH: ${ARCH} - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - # The codename is detected from the image's /etc/os-release. - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate - # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' - commands: - - apk add --no-cache -q bash curl jq - # Crow carries the checkout from the re-index step into this step. - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES - # Runs on every row rather than on one designated slot: a cron fires only - # its own slot's row, so gating on a named slot would leave every other - # slot unpurged. A manual "all" run therefore purges the zone 18 times, - # which is a cheap API call and rare. - # - # Run it even when the re-index above failed: the objects were still - # replaced, and a stale edge is exactly what keeps them hidden. - when: - - status: [success, failure] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2bf38b9..a0f4422 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,39 +7,30 @@ exclude: | benchmark/| docker/reprex/ ) -# The `^local/patches/.*\.patch$` excludes below keep unified diffs byte-exact: -# a context line for a blank line is a single space, and stripping it (or -# appending a newline) makes `git apply` reject the patch, which surfaces as -# "patch did not apply cleanly" at build time rather than as a lint failure -# here. The exclusions are per-hook, not global, so `validate patch registry` -# still runs when a patch changes. repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: end-of-file-fixer - exclude: ^local/patches/.*\.patch$ - id: trailing-whitespace - exclude: ^local/patches/.*\.patch$ args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.23.2 + rev: v0.23.0 hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.6 + rev: v3.9.4 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit - rev: 0.11.0 + rev: 0.10.0 hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v3.11.2 + rev: v3.8.0 hooks: - id: editorconfig-checker - exclude: ^local/patches/.*\.patch$ - repo: https://github.com/adrienverge/yamllint.git rev: v1.38.0 hooks: diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 9591b83..6ef10fa 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,79 +2,79 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/http" { - version = "3.6.1" + version = "3.6.0" hashes = [ - "h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=", - "h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=", - "h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=", - "h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=", - "h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=", - "h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=", - "h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=", - "h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=", - "h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=", - "h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=", - "h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=", - "h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=", - "h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=", - "h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=", - "h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=", - "zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9", - "zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6", - "zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab", - "zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b", - "zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c", - "zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4", - "zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502", - "zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69", - "zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6", - "zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279", - "zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6", - "zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7", - "zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df", - "zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621", - "zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424", + "h1:0n4RBz9zNw6TTddh5+x7E8L2+qzPXNwKhK4uoZ/DUwE=", + "h1:22Ob7lpzMBSqdrCvoFN5EgmhGPHPBovV/9qo0c/Cd+A=", + "h1:2IRBvmWOYrq/ooaYYn2i86jZb7iIUvlg0KlmOMfDHoQ=", + "h1:5mucXikk4OcW3un3u94QnMx4AB4Wfih+sXeMd5QxSNk=", + "h1:5oU7Zm+2gAVGmxqtJ9E8uTudUkYy/DEn/y3IWphdv4k=", + "h1:5w0R4b1/VSzpqQF1tXXPr/qmaQLPVRXamOmPKWFcTk4=", + "h1:AEVeJr8xGmwad+JUUQ833C3x5d4W+W2szF5DfwxYppw=", + "h1:CPHJ+0zQbS/cX1m55Y90jIOgf1jV3ocUUnqsXAh+9Eg=", + "h1:JPewnGDOJudNer5+ghqwXoaJkfot3QRq9uiEYvo+JHU=", + "h1:QzbluV2vQLxsJYxjpziQCmPndIoJ/UGS4/UHH/GpwUM=", + "h1:TjUNbUdqweRBq/ycQ4ixpNkx5qaYwpXEOn9QCpqNZP8=", + "h1:XNbcODP60ajj21N/OO7af8bBg1ltIsYkq9egn7BYbiY=", + "h1:tgrbgmX7WYQz9G9ncgu7TkpVB+RlLjJA/Rvp9KPlZH8=", + "h1:vLxthX/ZWsOZ+aHKbAMqmNKqD0K5f4nJ8ppy0Ioyup0=", + "h1:wZOdGBAZkY8OKEPjKz82j1HloAKOmmvtjWyTxM+I110=", + "zh:0f719fa5426bc883e9fa6abf7f6498e48025edafbc29015e2f5c028f1cca3b9d", + "zh:1b4d7dafefd6c61764b2f9ed6943ceb9a200dee3590d18747e3a5f6b20ce85e0", + "zh:1d23a712984866d29f7b07028a4e99c783c71f1a5dddf08bc3d4e7da9d91a1fa", + "zh:257d23d58c3bb024b6bc8eb88736eaf912e934ad47c639d0c3c742bddda849a1", + "zh:479860e1a5468f5e04013b9364c9496d7ed0804bf9a1acd8e07558d57609993d", + "zh:4cb5e681bf599b411b27c4a2c4066a5fb2ed79aaa3a1a3cb5a30002fec062ce9", + "zh:4fb35c3f643dae9f3670d719397a415f815a0b95f8ed7bd8a72f27a94ba78092", + "zh:59ba40825ab38db5b4a0989a2db0df35cc15d8984f898176011ba352f27d77b7", + "zh:61fc1252eb88088638f4c69ea4e2171cde2e5089fa632ac1e943b13787348f73", + "zh:7c5d6dd5f7cbc460e95d368be35c29b4e0402069b8912dbd5d1cd7fa9acef216", + "zh:7f76d756240d4284642f359ad470226e5378670239aadc366ef54d9d914d4d2e", + "zh:8133ad0814098177e0d067c816ccf1bf48bbadacd18f6f2c808c90447505723b", + "zh:c93be06269bb728f1968f8c50506de56c887017ac1d6e4be1f925651d8437eb6", + "zh:ef47b78a10a82e6cf53344a6a85a94041c28286c10a70541c564d762f1cfede0", + "zh:f5796a53a74999135bd9087aff50fddda59129d09b2f9b1902ff8c0c1e047e48", ] } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.18.2" - constraints = "~> 0.18" + version = "0.15.1" + constraints = "~> 0.15" hashes = [ - "h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=", - "h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=", - "h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=", - "h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=", - "h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=", - "h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=", - "h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=", - "h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=", - "h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=", - "h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=", - "h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=", - "h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=", - "h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=", - "h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=", - "h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=", - "h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=", - "h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=", - "zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c", - "zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178", - "zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd", - "zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef", - "zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738", - "zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc", - "zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c", + "h1:/2NUpbtjkc+w6n5V3kGP0rSzGjN0K2Wdfe2K+CZdmhU=", + "h1:1TOrpmCR0aT5xX1sjt70zqlYkMJnVe6r0nx2B0DS/mE=", + "h1:4uC8r8ILr+vZJ230uGqIUaWQ7uORCrIzoEuYrAq2JuE=", + "h1:6b+2osJUfwcaYZ6mathLPf/58sr/4XkLXQrcSufnkMk=", + "h1:CPldfjf79QS8mIP+GWoS0FVhrFlyjvN2+39zfyP76Ik=", + "h1:EvXICHyGIKpoYlDeKHOPzfmpvdRdhgsOMcR2nb6+tKY=", + "h1:GZLq+nDxS1CyBH0ELGTSQj9X7ozemJ1jpPA4KwbtR+c=", + "h1:ISNFqL745IQgZ6yMLy8ofV8ixbYqZYa9JKdi2W3pmNk=", + "h1:IrNrEuvFd0nYDGQefwmT8d1CSJb9e8LN5w9vw1ODp7E=", + "h1:Ms79slY9bZ94+n4cwIHvI9+/cvbucwo6S1+z5KAiznw=", + "h1:NWA9XSEBcpSkgwwIvl6tHrxGQY3uYhqNS4Vnb9RLyLQ=", + "h1:UjvxxxggicLtiE3yTe1Gx0oLUTeZpWmgIfXuHzWHn1c=", + "h1:VgJjo14DGkU4Jwo4D3GT4/5sq1tdjiZscKS5l3cb890=", + "h1:dBu3AW5YNLIvbBIMNk3wUHKw4TW+BDbj34a+mCqYhWE=", + "h1:i5oGD06nQ3JRsBIa2u4wCej+ETgp970CFl75dOKkHno=", + "h1:mTqR+vD1AWPx+mu7S0/pzBy71z7WOrjH7arOP77PXh4=", + "h1:nmTM61G8vYjpofeEqspMORpsNvTGCNZySGfjdXardL0=", + "zh:0f9bf5aaa47164a4d6ae4433d5e285a9456a5053401b2bad4ed68622f574ddee", + "zh:3039bee421fb8855a919f449fc731d145254371f5f0c39cc4660f3aeed6a8b10", + "zh:36664b08186e0c194747b18dee24ed97327c6e704133d4cf0df27abef1652f86", + "zh:3c7eae99d8c5ff65dfb99c8b9c1980147282d68971e1ef1ff0f126a6bff59d8f", + "zh:5293cc21abf54f4e5745437ca2d40d206aae323b2e1d41cf45dcc63a8868cbb3", + "zh:5994e5145e616e7e881010717e4c7def2945eb6d933f62db4ec3167732ccac84", + "zh:7994db9ed3fdb6cbf21f2154ea962cf82f04e425988bedc6657de2497d6cb6c3", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef", - "zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3", - "zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb", - "zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e", - "zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11", - "zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7", - "zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9", - "zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1", - "zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff", - "zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339", + "zh:93aa863e536ba9376ccf9e614e9edc9b214a2ce8c4316d416a8e249b436f52d2", + "zh:ab5cb4baeda57559686a0ccf0e09158aa64624ee6ba0ef32b769f13b11a43068", + "zh:ae9388b62eede8fd9272407bf75f8241a965bd489d45ec9dd3f9fac696d500e1", + "zh:caa5befd16960e2f69c7ec483e228e5ff43ab0979c17f1b874c9ffaa1c7c0e43", + "zh:cddd3e1067defa06a4e4ad5cb3940c7943e29c42417de57236aa7d3e2aeaae13", + "zh:cdfa44d591d0805116159556947904d70f534c6816188c45cf3a7544d2722ac9", + "zh:d607e9f1f3e09f13404f219e1893e3b3c77aece4afb54e999f934c021f41f576", + "zh:d8a397aca95125c6a0c0c78d2ded5843b9204effa9f5cf7419f017b500ad9228", + "zh:f5499eaff0d221725ad209d27d87c5b46d5c554caad7dc42947c760377abe3b0", + "zh:fc5f5cf433abc83e5169fa222992ec521c0c802075970251e3dd2c1d50c4f5c1", ] } diff --git a/README.md b/README.md index 989069e..28541a1 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ For every package+tag combination: ## Patching packages Some CRAN packages fail to compile on specific platforms due to compiler- or OS-specific issues unrelated to the package itself. -The canonical example is `RcppParallel`, whose bundled TBB is linked in a way that lets a system TBB on the build host shadow it, so the published binary depends on a library the consumer does not have. +The canonical example is `RcppParallel`, whose bundled TBB sources fail on musl (Alpine) and newer compiler/OS combinations. Because such packages are often transitive dependencies of many others, a single failure cascades: all dependents fail even though nothing is wrong with the dependent itself. To address this, frequently-failing packages can be "patched" before they are installed — whether as a direct build target or a transitive dependency pulled in by `pak`. @@ -112,9 +112,9 @@ Processing updates with an existing database file takes around 5 minutes. ### Inferring System Dependencies -R package dependencies and their system dependencies are installed through [uvr](https://github.com/nbafrank/uvr). -`uvr` allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. -Additionally, it automatically infers package dependencies using JSON rules vendored from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). +R package dependencies and their system dependencies are installed through {pak}. +{pak} allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. +Additionally, it automatically infers package dependencies using JSON rules from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). Not all R packages specify required system dependencies in their DESCRIPTION file, and not all listed dependencies have existing rules in `rstudio/r-system-requirements`. For Alpine, no rules existed until recently, establishing a foundation for semi-automated package installation on Alpine Linux. @@ -152,13 +152,12 @@ A Shiny dashboard providing a search functionality of the database and grouped s Is supported by writing `Meta/archive.rds` during each package index update, listing all available archived packages. -### `uvr add package@version` +### `pak::pak(package@version)` -Clients that resolve archived versions search for `Archive/` and can install all versions they find. -This holds for `uvr` as well as for older `pak`-based clients. +`pak` searches for `Archive/` and can install all versions it finds. Ensure to use a clean cache if other repositories have been used previously. -If in doubt or when testing, call `uvr cache clean`. +If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. ## Lessons Learned @@ -181,7 +180,7 @@ If in doubt or when testing, call `uvr cache clean`. ## URL Composition and Platform Identifiers -Platform identifiers have been aligned with those used in , which `uvr` vendors and uses for its automatic syslib dependency installer (`uvr sync --install-system-deps`): +Platform identifiers have been aligned with those used in to ensure proper recognition by the automatic syslib dependency installer of `pak`, specifically via the environment variable `PKG_SYSREQS_PLATFORM`: - redhat-9 - redhat-8 @@ -238,8 +237,8 @@ internal error 1 in memDecompress Solution: ```sh -uvr cache clean -rm -rf /mnt/cache/uvr /root/.cache/R/ +rm -rf /mnt/cache/R-pkgs/pak /mnt/cache/pkgcache/ /root/.cache/R/ +R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' ``` diff --git a/build-all-versions-install-deps.yaml b/build-all-versions-install-deps.yaml index dd02e8a..926d550 100644 --- a/build-all-versions-install-deps.yaml +++ b/build-all-versions-install-deps.yaml @@ -30,9 +30,8 @@ # from_secret: HETZNER_S3_SECRET_KEY_K3S # # normal env vars # GIT_USER: pat-s -# # set the location of uvr's caches, which persist the R package dependencies needed to install the packages themselves -# # UVR_CACHE_DIR: /mnt/cache/uvr/cache -# # UVR_PACKAGES_DIR: /mnt/cache/uvr/packages +# # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves +# # R_PKG_CACHE_DIR: /mnt/cache/pkgcache # R_LIBS_USER: /mnt/cache/R-pkgs # CCACHE_DIR: /mnt/cache/ccache # volumes: @@ -40,8 +39,9 @@ # commands: # - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # - mkdir -p /mnt/cache/R-pkgs -# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/uvr -# - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft +# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/pkgcache +# - /opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +# - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git", dependencies = TRUE)' # - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine322', 'amd64'); saveRDS(pkgs, '/mnt/cache/pkgs_amd64.rds')" # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine323', 'arm64'); saveRDS(pkgs, '/mnt/cache/pkgs_arm64.rds')" diff --git a/cdn.tf b/cdn.tf index 349486e..8a950f5 100644 --- a/cdn.tf +++ b/cdn.tf @@ -32,7 +32,7 @@ # cache_stale = ["offline", "updating"] # use_background_update = true -# block_ips = var.cdn_block_ips + # block_ips = var.cdn_block_ips # # 50 TB # limit_bandwidth = 50000000000000 @@ -52,56 +52,6 @@ ### cran.rpkgs.com -locals { - rpkgs_slots = [ - for pair in setproduct( - ["amd64", "arm64"], - ["resolute", "noble", "jammy", "rhel8", "rhel9", "rhel10", "alpine323", "alpine324"] - ) : "${pair[0]}/${pair[1]}" - ] - - # The supported R minors: the current one plus the two previous, which is - # exactly what build-env-images installs as R_VERSION_LATEST / PREV1 / PREV2. - # These must stay in step. A minor listed here without a published index - # sends those clients to a 404; a published minor missing from this list - # sends them to CRAN for sources instead of serving the binaries we built. - rpkgs_supported_minors = ["4.4", "4.5", "4.6"] - - # bunny.net serves every pull zone on .b-cdn.net, so staging needs no - # DNS record and is never advertised. - rpkgs_test_hostname = "cran-rpkgs-test.b-cdn.net" -} - -# The edge middleware that resolves the bare cran.rpkgs.com form to an -# / slot and routes PACKAGES* to the per-R-minor slot. The source of -# truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release. -# -# The script pre-dates this configuration, so it is adopted rather than created: -# tofu import bunnynet_compute_script.rpkgs_router 29277 -resource "bunnynet_compute_script" "rpkgs_router" { - type = "middleware" - name = "rpkgs-router" - content = file("${path.module}/edge/rpkgs-router.ts") -} - -# Slots ("/", comma separated) whose per-minor index bincraft has -# already republished as a union of the per-minor and flat slots. Routing to a -# slot that is not listed here would hide every package the per-minor index does -# not carry, so this stays empty until a slot has been backfilled. -resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { - script = bunnynet_compute_script.rpkgs_router.id - name = "UNION_SLOTS" - default_value = "" - required = false -} - -resource "bunnynet_compute_script_variable" "rpkgs_router_known_minors" { - script = bunnynet_compute_script.rpkgs_router.id - name = "KNOWN_MINORS" - default_value = join(",", local.rpkgs_supported_minors) - required = false -} - resource "bunnynet_pullzone" "cran_rpkgs_com" { name = "cran-rpkgs" @@ -109,12 +59,12 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { cache_expiration_time = 31919000 websockets_enabled = false - errorpage_whitelabel = true + errorpage_whitelabel = true origin { type = "OriginUrl" url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" - middleware_script = bunnynet_compute_script.rpkgs_router.id + middleware_script = 29277 } routing { @@ -132,21 +82,6 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { request_coalescing_enabled = true block_post_requests = true - # Set on the zone since before this configuration existed; declared here so - # `tofu apply` stops silently removing it. - # - # The router makes it redundant on paper: the only UA-dependent responses it - # produces are redirects, and those carry `Cache-Control: no-store`, while - # their targets are concrete per-slot, per-minor URLs whose content depends - # only on the path. Dropping it would also be a real win, because otherwise - # every distinct R version string keys its own copy of every tarball. - # - # It stays for now anyway: it is the second line of defence against the one - # failure that would be quiet and confusing (an R 4.6 client served the 4.5 - # index), and removing it is worth doing on its own once per-minor routing is - # confirmed live, not as a side effect of enabling that routing. - cache_vary_headers = ["User-Agent"] - limit_requests = 5000 limit_connections = 1000 @@ -174,151 +109,6 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } -### Staging zone for edge-router changes - -# Every published / slot. The staging zone enables per-minor routing -# for all of them at once; production adopts the same list only after -# `scripts/verify-r-minor-routing.sh --live` passes against staging. - -# A second copy of the same router, bound to the same B2 origin, so UNION_SLOTS -# can be exercised end to end before production is touched. -resource "bunnynet_compute_script" "rpkgs_router_test" { - type = "middleware" - name = "rpkgs-router-test" - content = file("${path.module}/edge/rpkgs-router.ts") -} - -resource "bunnynet_compute_script_variable" "rpkgs_router_test_union_slots" { - script = bunnynet_compute_script.rpkgs_router_test.id - name = "UNION_SLOTS" - default_value = join(",", local.rpkgs_slots) - required = false -} - -# Without this the staging zone rewrites to PUBLIC_CDN_ORIGIN, so its redirects -# land on production and the test silently measures the wrong system. -resource "bunnynet_compute_script_variable" "rpkgs_router_test_extra_hosts" { - script = bunnynet_compute_script.rpkgs_router_test.id - name = "EXTRA_PUBLIC_HOSTS" - default_value = local.rpkgs_test_hostname - required = false -} - -resource "bunnynet_compute_script_variable" "rpkgs_router_test_known_minors" { - script = bunnynet_compute_script.rpkgs_router_test.id - name = "KNOWN_MINORS" - default_value = join(",", local.rpkgs_supported_minors) - required = false -} - -resource "bunnynet_pullzone" "cran_rpkgs_test" { - name = "cran-rpkgs-test" - - cache_errors = false - - cache_expiration_time = 31919000 - websockets_enabled = false - errorpage_whitelabel = true - - origin { - type = "OriginUrl" - url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" - middleware_script = bunnynet_compute_script.rpkgs_router_test.id - } - - routing { - filters = [ - "scripting", - ] - } - - s3_auth_enabled = true - s3_auth_key = var.B2_S3_ACCESS_KEY - s3_auth_secret = var.B2_S3_SECRET_KEY - s3_auth_region = "eu-central-003" - - cache_enabled = true - request_coalescing_enabled = true - block_post_requests = true - - cache_vary_headers = ["User-Agent"] - - # Staging carries only synthetic verification traffic, so the production - # ceilings would be pure headroom. - limit_requests = 500 - limit_connections = 100 - - safehop_enabled = true - add_canonical_header = true - cache_stale = ["offline", "updating"] - block_ips = var.cdn_block_ips - - # 1 TB - limit_bandwidth = 1000000000000 - - block_root_path = true -} - - -# Alliance SwissPass historically used a separate, manually configured pull -# zone. Adopt it so both public repositories use the same B2 origin, middleware -# release and cache behavior. -import { - to = bunnynet_pullzone.cran_allianceswisspass - id = "3265648" -} - -resource "bunnynet_pullzone" "cran_allianceswisspass" { - name = "cran-allianceswisspass" - - cache_errors = false - cache_expiration_time = 31919000 - websockets_enabled = false - errorpage_whitelabel = true - - origin { - type = "OriginUrl" - url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" - middleware_script = bunnynet_compute_script.rpkgs_router.id - } - - routing { - filters = [ - "scripting", - ] - } - - s3_auth_enabled = true - s3_auth_key = var.B2_S3_ACCESS_KEY - s3_auth_secret = var.B2_S3_SECRET_KEY - s3_auth_region = "eu-central-003" - - cache_enabled = true - request_coalescing_enabled = true - block_post_requests = true - cache_vary_headers = ["User-Agent"] - - limit_requests = 5000 - limit_connections = 1000 - - safehop_enabled = true - add_canonical_header = true - cache_stale = ["offline", "updating"] - block_ips = var.cdn_block_ips - - # 50 TB - limit_bandwidth = 50000000000000 - - block_root_path = true -} - -resource "bunnynet_pullzone_hostname" "cran_allianceswisspass" { - pullzone = bunnynet_pullzone.cran_allianceswisspass.id - name = "cran.allianceswisspass.devxy.io" - force_ssl = true - tls_enabled = true -} - # resource "bunnynet_storage_zone" "devxy-r-binaries" { # name = "devxy-r-binaries-storage" # region = "DE" diff --git a/docker/Containerfile-shiny-app b/docker/Containerfile-shiny-app index 4b0bd6c..7c7994f 100644 --- a/docker/Containerfile-shiny-app +++ b/docker/Containerfile-shiny-app @@ -1,15 +1,7 @@ FROM devxygmbh/r-alpine:4.4-3.20 AS build # ARG GITHUB_PAT -RUN apk add --no-cache curl ca-certificates \ - && curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh \ - | UVR_INSTALL_DIR=/usr/local/bin sh - -# One uvr project drives both dependency installs below. UVR_LIBRARY points the -# syncs at the image's R library instead of the project-local .uvr/library/, so -# `Rscript app.R` finds the packages without a uvr-aware .Rprofile. -ENV UVR_LIBRARY=/usr/lib/R/library -RUN mkdir -p /uvr && cd /uvr && uvr init --here +RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' COPY --link ./DESCRIPTION . COPY --link ./R ./R @@ -19,19 +11,13 @@ COPY --link ./shiny/app.R /app/app.R RUN ls -la -# install R package deps. uvr has no equivalent of bare `pak::pak()`, which -# reads the DESCRIPTION in the working directory, so extract the dependency -# names and hand them to `uvr add`. -RUN R -q --no-echo -e "d <- read.dcf('DESCRIPTION'); f <- intersect(colnames(d), c('Depends', 'Imports', 'LinkingTo')); p <- trimws(sub('[(].*', '', unlist(strsplit(paste(d[, f], collapse = ','), ',')))); writeLines(setdiff(p[nzchar(p)], c('R', rownames(installed.packages()))), '/tmp/deps.txt')" \ - && cd /uvr && xargs -r uvr add --no-install < /tmp/deps.txt \ - && uvr sync --install-system-deps +# install R package deps +RUN R -q -e 'pak::pak()' RUN R CMD INSTALL --no-docs --without-keep.source . # install shiny app deps -RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); writeLines(pkgs, '/tmp/app-deps.txt')" \ - && cd /uvr && xargs -r uvr add --no-install < /tmp/app-deps.txt \ - && uvr sync --install-system-deps +RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); pak::pak(pkgs)" ENV PGPASS="" diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 8e61ebd..5bc107a 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -17,10 +17,7 @@ ARG CACHEBUST WORKDIR /work COPY build-one.R /work/build-one.R # Resolve and install the latest bincraft release dynamically (no hardcoded pin). -# uvr-install.sh lands under /work/local/ because install-bincraft.R looks for it -# there when the working directory is not a repo checkout. COPY install-bincraft.R /work/install-bincraft.R -COPY uvr-install.sh /work/local/uvr-install.sh # Ship the patch registry so build-one.R's `patches = "local/patches"` resolves # (build context is `local/`, CWD is /work). COPY patches /work/local/patches diff --git a/docker/reprex/alpine.sh b/docker/reprex/alpine.sh index 851a9cc..94806e5 100644 --- a/docker/reprex/alpine.sh +++ b/docker/reprex/alpine.sh @@ -1,14 +1,10 @@ docker run --rm -it --platform linux/arm64 alpine sh -apk add --no-cache R R-dev g++ curl ca-certificates +apk add --no-cache R R-dev g++ -curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | UVR_INSTALL_DIR=/usr/local/bin sh +R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' -# `uvr add` always writes to .uvr/library/; only `uvr sync` honours UVR_LIBRARY, -# so add without installing and let the sync place the packages. -export UVR_LIBRARY=/usr/lib/R/library -mkdir -p /uvr && cd /uvr && uvr init --here -uvr add --no-install gert purrr && uvr sync --install-system-deps +R -q -e 'pak::pak(c("gert", "purrr"))' R @@ -26,20 +22,14 @@ unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE) tag <- all_tags$name package_name <- rep(package_name, length(tag)) -# uvr has no `pak::local_install_deps()`; read the DESCRIPTION of the checkout -# and `uvr add` the dependency names instead. purrr::walk2(package_name[1], tag, \(x, y) { print(y) system("git config --global advice.detachedHead false") - src <- sprintf("/tmp/%s_%s", x[1], y) system2("git", args = c( "clone", "-q", sprintf("--branch=%s", tail(y, 1)), - sprintf("https://github.com/cran/%s", x[1]), src + sprintf("https://github.com/cran/%s", x[1]), sprintf("/tmp/%s_%s", x[1], y) )) - d <- read.dcf(file.path(src, "DESCRIPTION")) - f <- intersect(colnames(d), c("Depends", "Imports", "LinkingTo")) - deps <- trimws(sub("[(].*", "", unlist(strsplit(paste(d[, f], collapse = ","), ",")))) - deps <- setdiff(deps[nzchar(deps)], "R") - system2("uvr", c("add", "--no-install", deps)) - system2("uvr", c("sync", "--install-system-deps")) + pak::local_install_deps(sprintf("/tmp/%s_%s", x[1], y)) }) + + diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts deleted file mode 100644 index 3e2fe8b..0000000 --- a/edge/rpkgs-router.test.ts +++ /dev/null @@ -1,209 +0,0 @@ -/** - * Routing matrix for `edge/rpkgs-router.ts`. - * - * The script is exercised through the SDK's local server rather than by - * importing its internals, so what is tested is the artifact that gets - * deployed. Requests that the script passes through are proxied to the real - * origin, which keeps the "no redirect" cases honest: they assert that the - * client reached the flat slot, not merely that no `Location` was set. - * - * Run with `just edge-test`. - */ -import { assertEquals } from 'jsr:@std/assert@1'; - -const SCRIPT = new URL('./rpkgs-router.ts', import.meta.url).pathname; -const BASE = 'http://127.0.0.1:8080'; -const UNION_SLOTS = 'amd64/alpine324'; - -const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; -const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; -const UA_R43_MUSL = 'R (4.3.3 x86_64-pc-linux-musl x86_64 linux-musl)'; -const UA_R47_MUSL = 'R (4.7.0 x86_64-pc-linux-musl x86_64 linux-musl)'; -const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; -const UA_R45_RESOLUTE = 'R/4.5.3 (Ubuntu 26.04) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; -const UA_R45_FUTURE_UBUNTU = - 'R/4.5.3 (Ubuntu 28.04; codename=dynamic-dugong) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; -const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)'; -const UA_CURL = 'curl/8.0.1'; - -const SLOT = '/amd64/alpine324/latest/src/contrib'; -const OTHER_SLOT = '/amd64/noble/latest/src/contrib'; - -interface Probe { - status: number; - location: string | null; - cacheControl: string | null; -} - -async function probe(path: string, userAgent: string): Promise { - const res = await fetch(BASE + path, { - headers: { 'User-Agent': userAgent }, - redirect: 'manual', - }); - await res.body?.cancel(); - return { - status: res.status, - location: res.headers.get('location'), - cacheControl: res.headers.get('cache-control'), - }; -} - -/** Kill tolerantly: the child has already exited if the script failed to load. */ -async function stopServer(child: Deno.ChildProcess): Promise { - try { - child.kill(); - } catch { - // already gone - } - await child.status; -} - -async function startServer(): Promise { - const child = new Deno.Command(Deno.execPath(), { - args: ['run', '-A', SCRIPT], - env: { UNION_SLOTS }, - stdout: 'null', - stderr: 'inherit', - }).spawn(); - - for (let attempt = 0; attempt < 150; attempt++) { - try { - const res = await fetch(`${BASE}/`, { - headers: { 'User-Agent': UA_CURL }, - redirect: 'manual', - }); - await res.body?.cancel(); - return child; - } catch { - await new Promise((resolve) => setTimeout(resolve, 200)); - } - } - - await stopServer(child); - throw new Error('edge script did not start listening on ' + BASE); -} - -Deno.test('rpkgs-router', async (t) => { - const server = await startServer(); - - try { - await t.step("routes an index request to the client's R minor", async () => { - const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL); - assertEquals(res.status, 302); - assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); - }); - - await t.step('routes R 4.6 to its own slot', async () => { - const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R46_MUSL); - assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`); - }); - - // We publish binaries only for the supported window. An excluded minor has - // no slot we can serve safely, so it goes to CRAN for sources rather than - // to a 404 or to binaries built under another minor. - await t.step('sends an excluded R minor to CRAN for the index', async () => { - const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R43_MUSL); - assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); - }); - - await t.step('sends a future R minor to CRAN too', async () => { - const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R47_MUSL); - assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); - }); - - // The index and the tarballs R resolves against it have to come from the - // same place. Serving one from CRAN and the other from here would hand R a - // binary where it expects a source tarball. - await t.step('sends an excluded minor to CRAN for tarballs as well', async () => { - const res = await probe(`${SLOT}/foo_1.0.tar.gz`, UA_R43_MUSL); - assertEquals(res.location, 'https://cran.r-project.org/src/contrib/foo_1.0.tar.gz'); - }); - - await t.step('leaves an excluded minor alone on a slot outside UNION_SLOTS', async () => { - const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R43_MUSL); - assertEquals(res.location, null); - assertEquals(res.status, 200); - }); - - await t.step('routes PACKAGES and PACKAGES.rds too', async () => { - for (const file of ['PACKAGES', 'PACKAGES.rds']) { - const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL); - assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/${file}`, `expected ${file} to be routed`); - } - }); - - await t.step('marks the redirect uncacheable', async () => { - const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL); - assertEquals(res.cacheControl, 'no-store'); - }); - - await t.step('leaves a slot outside UNION_SLOTS alone', async () => { - const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R45_MUSL); - assertEquals(res.location, null); - assertEquals(res.status, 200); - }); - - await t.step('never routes a tarball', async () => { - const res = await probe(`${SLOT}/jsonlite_2.0.0.tar.gz`, UA_R45_MUSL); - assertEquals(res.location, null); - assertEquals(res.status, 200); - }); - - await t.step('serves an archived binary when it exists', async () => { - const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`; - const res = await probe(path, UA_R45_MUSL); - assertEquals(res.status, 200); - assertEquals(res.location, null); - }); - - await t.step('does not redirect a path already under a minor', async () => { - const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); - assertEquals(res.location, null); - assertEquals(res.status, 200); - }); - - await t.step('leaves a client without an R version alone', async () => { - const res = await probe(`${SLOT}/PACKAGES.gz`, UA_CURL); - assertEquals(res.location, null); - assertEquals(res.status, 200); - }); - - await t.step('resolves the bare root to slot and minor', async () => { - const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_ALPINE); - assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); - }); - - await t.step('resolves Ubuntu 26.04 to the resolute slot', async () => { - const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_RESOLUTE); - assertEquals(res.location, 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz'); - }); - - await t.step('resolves a future Ubuntu release from its codename', async () => { - const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_FUTURE_UBUNTU); - assertEquals(res.location, 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz'); - }); - - await t.step('sends an unidentifiable distro to CRAN', async () => { - const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL); - assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); - }); - - await t.step('keeps the macOS rewrite', async () => { - const res = await probe('/src/contrib/foo_1.0.tar.gz', UA_R45_DARWIN); - assertEquals(res.location, 'https://cran.rpkgs.com/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz'); - }); - - await t.step('keeps the macOS binary passthrough to CRAN', async () => { - const path = '/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz'; - const res = await probe(path, UA_R45_DARWIN); - assertEquals(res.location, `https://cran.r-project.org${path}`); - }); - - await t.step('collapses duplicate slashes before matching', async () => { - const res = await probe(`/amd64/alpine324//latest/src/contrib//PACKAGES.gz`, UA_R45_MUSL); - assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); - }); - } finally { - await stopServer(server); - } -}); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts deleted file mode 100644 index 05b6e18..0000000 --- a/edge/rpkgs-router.ts +++ /dev/null @@ -1,316 +0,0 @@ -/** - * Edge middleware for cran.rpkgs.com. - * - * Two jobs: - * - * 1. Resolve the bare `https://cran.rpkgs.com` form to a concrete - * `/` slot from the User-Agent, or send the client to CRAN when - * the distro cannot be identified. - * 2. Route `PACKAGES*` requests to the per-R-minor slot - * (`…/latest/src/contrib//`), so a stock `install.packages()` sees the - * packages that only exist there. - * - * Only index files are routed. Tarballs are deliberately left alone: R keeps - * the contrib URL it asked for, not the one it was redirected to, so every - * tarball URL is resolved against the flat directory and the union index steers - * the per-minor ones with a `Path: ` field. Rewriting a tarball request - * here would send flat-slot packages into a directory that does not hold them. - * - * Routing is gated on UNION_SLOTS. The raw per-minor index holds only the - * ABI-sensitive subset of a slot; it is safe to route to it only once bincraft - * has republished it as a union of the per-minor and flat slots. - * - * Deployed by OpenTofu from this file (`bunnynet_compute_script.rpkgs_router`). - * Test with `just edge-test`. - */ -import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; - -const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; -const CRAN_ORIGIN = 'https://cran.r-project.org'; -const PUBLIC_CDN_HOSTS = new Set([ - 'cran.rpkgs.com', - 'cran.allianceswisspass.devxy.io', - // Staging hostnames, so the identical script can run on a test pull zone and - // redirect within itself. Without this a test zone rewrites to - // PUBLIC_CDN_ORIGIN, quietly exercising production instead of itself. - ...(Deno.env.get('EXTRA_PUBLIC_HOSTS') ?? '') - .split(',') - .map((host) => host.trim()) - .filter((host) => host.length > 0), -]); - -/** Slots ("/", comma separated) whose per-minor index is a union. */ -const UNION_SLOTS = new Set( - (Deno.env.get('UNION_SLOTS') ?? '') - .split(',') - .map((slot) => slot.trim()) - .filter((slot) => slot.length > 0), -); - -/** - * R minors for which a per-minor index is actually published. - * - * contribPath() has no way to probe the origin, so a minor that is not - * published here must fall back to the flat index. Routing an unlisted minor - * would send that client to a 404 and it would see no packages at all - a - * silent, total failure rather than a degraded one. - */ -const KNOWN_MINORS = new Set( - (Deno.env.get('KNOWN_MINORS') ?? '4.4,4.5,4.6') - .split(',') - .map((minor) => minor.trim()) - .filter((minor) => minor.length > 0), -); - -/** `///latest/src/contrib[/]` */ -const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/; - -/** A path that already sits in a per-minor slot, e.g. `4.5/PACKAGES.gz`. */ -const MINOR_DIR_REGEX = /^\d+\.\d+\//; - -/** The only files this script routes. */ -const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; - -const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; - -/** A binary archive URL whose upstream source counterpart CRAN can serve. */ -const ARCHIVE_TARBALL_REGEX = - /^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/; - -const MACOS_BIN_REGEX = - /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; - -const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i; - -const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i; -const UBUNTU_CODENAME_REGEX = /Ubuntu [\d.]+;\s*codename=([a-z][a-z0-9-]*)/i; -const UBUNTU_CODENAMES: Record = { - '26.04': 'resolute', - '24.04': 'noble', - '22.04': 'jammy', -}; - -const ALPINE_REGEX = /(?:Alpine Linux(?:\s+VERSION_ID=)?|alpine-)\s*(\d+)\.(\d+)/i; - -/** - * R's own User-Agent is `R (4.5.3 x86_64-pc-linux-musl …)`; the Posit-style one - * some sites configure is `R/4.5.3 R (…)`. Both carry the minor, which is why - * per-minor routing works without the distro being identifiable. - */ -const R_MINOR_REGEXES = [/\bR\/(\d+)\.(\d+)/, /\bR \((\d+)\.(\d+)/]; - -function normalizePathname(pathname: string): string { - return pathname.replace(/\/{2,}/g, '/'); -} - -function redirectTo(location: string, status = 302): Response { - return new Response(null, { - status, - headers: { - Location: location, - // The target depends on the User-Agent, so the redirect itself must - // never be cached; only its target is a cacheable, UA-independent URL. - 'Cache-Control': 'no-store', - 'X-Via': 'MyMiddleware', - 'X-Rewritten-By': 'rpkgs-edge-middleware', - }, - }); -} - -function publicCdnOrigin(url: URL): string { - return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; -} - -/** - * True when the client reports an R minor that we deliberately do not serve. - * - * A client that reports no minor at all is not "unsupported": non-R fetchers - * (mirror scripts, image builds) must keep getting the flat slot. Only a - * known-and-excluded minor falls through to CRAN. - */ -function isExcludedMinor(userAgent: string): boolean { - const rMinor = extractRMinor(userAgent); - return rMinor !== null && !KNOWN_MINORS.has(rMinor); -} - -function extractRMinor(userAgent: string): string | null { - for (const regex of R_MINOR_REGEXES) { - const match = userAgent.match(regex); - if (match) { - return `${match[1]}.${match[2]}`; - } - } - return null; -} - -function mapArch(arch: string): string { - if (arch === 'aarch64') return 'arm64'; - if (arch === 'x86_64') return 'amd64'; - return arch; -} - -function extractArch(userAgent: string): string { - const match = userAgent.match(/(x86_64|aarch64|arm64|i386|i686)/); - return match ? mapArch(match[1]) : ''; -} - -/** - * Identify the `/` slot from the User-Agent, or null. - * - * A stock R User-Agent carries only `linux-gnu` / `linux-musl`, which are not - * slot names: returning them produced redirects into slots that do not exist - * (`/amd64/linux-musl/latest/…`, a guaranteed 404). An unidentifiable distro - * is reported as such so the caller can fall back to CRAN. - */ -function parseSlot(userAgent: string): string | null { - const arch = extractArch(userAgent); - if (!arch) { - return null; - } - - const rhel = userAgent.match(RHEL_REGEX); - if (rhel) { - return `${arch}/rhel${rhel[2]}`; - } - - const ubuntu = userAgent.match(UBUNTU_REGEX); - if (ubuntu) { - const codenameMatch = userAgent.match(UBUNTU_CODENAME_REGEX); - if (codenameMatch) { - return `${arch}/${codenameMatch[1].toLowerCase()}`; - } - - const codename = UBUNTU_CODENAMES[ubuntu[1]]; - if (codename) { - return `${arch}/${codename}`; - } - } - - const alpine = userAgent.match(ALPINE_REGEX); - if (alpine) { - return `${arch}/alpine${alpine[1]}${alpine[2]}`; - } - - return null; -} - -function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: string } | null { - const rverMatch = userAgent.match(/R \((\d+)\.(\d+)/); - const archMatch = userAgent.match(/(aarch64|arm64|x86_64)/); - const osMatch = userAgent.match(/darwin(\d+)/); - - if (!rverMatch || !archMatch || !osMatch) { - return null; - } - - const arch = archMatch[1] === 'aarch64' ? 'arm64' : archMatch[1]; - const darwinVer = parseInt(osMatch[1], 10); - const os = darwinVer >= 21 && darwinVer < 22 ? `monterey-${arch}` : `big-sur-${arch}`; - - return { os, arch, rver: `${rverMatch[1]}.${rverMatch[2]}` }; -} - -/** - * The contrib path a request should be served from, relative to the slot. - * - * Returns the per-minor path for an index file when the slot is known to carry - * a union index and the client's R minor is one we publish; otherwise the flat - * path, which is what every client sees today. - */ -function contribPath(slot: string, rest: string, userAgent: string): string { - const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`; - - if (!INDEX_FILE_REGEX.test(rest) || !UNION_SLOTS.has(slot)) { - return flat; - } - - const rMinor = extractRMinor(userAgent); - return rMinor && KNOWN_MINORS.has(rMinor) ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; -} - -BunnySDK.net.http - .servePullZone({ url: 'https://cran.rpkgs.com/' }) - .onOriginRequest((ctx) => { - const url = new URL(ctx.request.url); - const path = normalizePathname(url.pathname); - const userAgent = ctx.request.headers.get('User-Agent') || ''; - const publicOrigin = publicCdnOrigin(url); - - // macOS clients are served from CRAN's own binary tree. - const srcContrib = path.match(SRC_CONTRIB_REGEX); - if (srcContrib && /darwin/.test(userAgent)) { - const mac = parseMacUserAgent(userAgent); - if (mac) { - return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`)); - } - } - - if (MACOS_BIN_REGEX.test(path)) { - return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); - } - - // Already-qualified slot URLs: what the runtime images have baked in. - const slotPath = path.match(SLOT_PATH_REGEX); - if (slotPath) { - const slot = `${slotPath[1]}/${slotPath[2]}`; - const rest = slotPath[3]; - - // Never rewrite a request that is already in a per-minor slot, or the - // redirect would chase its own tail. - if (MINOR_DIR_REGEX.test(rest)) { - return Promise.resolve(ctx.request); - } - - // An R minor outside the supported window has no binaries we can safely - // serve, so the whole interaction goes to CRAN: the index and the - // tarballs R will resolve against it. Serving the index from CRAN but - // tarballs from here would hand R a binary where it expects a source - // tarball, which fails in a far more confusing way than not being - // served at all. - if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) { - return Promise.resolve(redirectTo(`${CRAN_ORIGIN}/src/contrib/${rest}`)); - } - - const target = contribPath(slot, rest, userAgent); - if (target === path) { - return Promise.resolve(ctx.request); - } - return Promise.resolve(redirectTo(`${publicOrigin}${target}`)); - } - - // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. - if (path === '/' || path === '/src/contrib' || path.startsWith('/src/contrib/')) { - const slot = parseSlot(userAgent); - if (!slot) { - return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); - } - - if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) { - return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); - } - - const rest = srcContrib ? srcContrib[1] : ''; - return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); - } - - return Promise.resolve(ctx.request); - }) - .onOriginResponse(async (ctx) => { - const path = normalizePathname(new URL(ctx.request.url).pathname); - const archive = path.match(ARCHIVE_TARBALL_REGEX); - - // Binary archives can be incomplete when an older build never succeeded. - // Preserve renv/remotes version restores by falling back to CRAN's source - // package only for an absent archived tarball. A requested version can be - // either archived upstream or still current, so probe the archive first. - // Other 404s remain visible. - if (ctx.response.status === 404 && archive) { - const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`; - const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' }); - const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`; - return redirectTo(sourceUrl); - } - - ctx.response.headers.append('X-Via', 'MyMiddleware'); - return Promise.resolve(ctx.response); - }); diff --git a/justfile b/justfile index 6c75b13..a9a705e 100644 --- a/justfile +++ b/justfile @@ -73,17 +73,3 @@ rebuild os tag arch package *versions: --build-arg CACHEBUST="$(date +%s)" \ -f docker/build-one.Dockerfile \ local - -# run the edge middleware routing matrix (uses a local deno, else the deno image) -edge-test: - #!/usr/bin/env bash - set -euo pipefail - if command -v deno >/dev/null 2>&1; then - deno test -A edge/rpkgs-router.test.ts - else - docker run --rm \ - -v "$PWD:/w" -w /w \ - -v deno-cache:/deno-dir \ - denoland/deno:latest \ - deno test -A edge/rpkgs-router.test.ts - fi diff --git a/local/build-all.R b/local/build-all.R index 37fc593..1df1f83 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -27,9 +27,7 @@ package_cache_files <- c( "/mnt/cache/packages/s3_cache.rds" ) if (!all(file.exists(package_cache_files))) { - message( - "Package snapshot missing from cache; recomputing via packages-to-build.R" - ) + message("Package snapshot missing from cache; recomputing via packages-to-build.R") dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { tmp <- paste0(path, ".tmp.", Sys.getpid()) @@ -38,10 +36,7 @@ if (!all(file.exists(package_cache_files))) { } source(file.path("local", "packages-to-build.R")) save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds") - save_rds_atomic( - pkgs[r_minor_sensitive == TRUE], - "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" - ) + save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds") message("Package snapshot recomputed.") } @@ -117,22 +112,20 @@ built <- DBI::dbGetQuery( ) DBI::dbDisconnect(con) before <- nrow(chunk) -chunk <- chunk[ - !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), -] -sprintf( - "Skipped %d already-attempted package versions; %d remaining for this job", - before - nrow(chunk), - nrow(chunk) -) +chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] +sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, -# saving significant memory for the dependency-installer subprocesses +# saving significant memory for pak subprocess forks s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") sprintf("S3 cache: %s files", length(s3_cache)) n <- nrow(chunk) +# Every `trim_every` packages, bound the pkgcache _metadata dir so a full-platform +# run does not accumulate thousands of ~70 MB snapshots and fill the host disk. +# No-op on amd64 (R_PKG_CACHE_DIR is empty / cache not persisted). +trim_every <- 25L mapply( function(pkg, ver, sens, i) { cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) @@ -158,6 +151,12 @@ mapply( upload = TRUE, store_build_metadata = TRUE ) + if (i %% trim_every == 0L) { + removed <- trim_pkgcache_metadata() + if (removed > 0L) { + cat(sprintf(" [pkgcache trim] removed %d stale _metadata entries\n", removed)) + } + } }, chunk$Package, chunk$Version, diff --git a/local/dedupe-audit-issue.R b/local/dedupe-audit-issue.R deleted file mode 100644 index c78015a..0000000 --- a/local/dedupe-audit-issue.R +++ /dev/null @@ -1,150 +0,0 @@ -# One-off maintenance: collapse the duplicate arch subsections that accumulated -# in the "Missing package binaries for latest version ()" issues. -# -# A bug in weekly-missing-binaries-audit.R matched the existing "### " -# subsection by its bare header while writing headers with a -# " (N missing, M to rebuild)" suffix, so every audit run appended a fresh block -# instead of replacing it. This script rewrites each "## " section to -# keep only the *last* (freshest) block per arch. The audit fix prevents further -# accumulation; this cleans up what is already there. -# -# Env: FORGEJO_TOKEN (required). DRY_RUN=1 to preview counts without patching. - -library(httr2, quietly = TRUE) - -forgejo_base <- "https://git.devxy.io/api/v1" -repo <- "devxy/build-cran-binaries" -token <- Sys.getenv("FORGEJO_TOKEN") -dry_run <- nchar(Sys.getenv("DRY_RUN")) > 0 - -if (nchar(token) == 0) { - stop("FORGEJO_TOKEN env var is not set") -} - -issue_titles <- c( - "Missing package binaries for latest version (Alpine)", - "Missing package binaries for latest version (Ubuntu)", - "Missing package binaries for latest version (Red Hat)" -) - -# Collapse one "## " block: keep only the last block per arch, -# emitted in order of first appearance. `pl[1]` is the "## " header. -dedupe_platform <- function(pl) { - sub_hdr <- which(grepl("^### ", pl)) - if (length(sub_hdr) == 0) { - return(pl) - } - preamble <- pl[seq_len(sub_hdr[1] - 1)] - sub_end <- c(sub_hdr[-1] - 1, length(pl)) - blocks <- lapply(seq_along(sub_hdr), function(k) { - pl[seq(sub_hdr[k], sub_end[k])] - }) - arches <- vapply( - blocks, - function(b) sub("^### (\\S+).*", "\\1", b[1]), - character(1) - ) - # Index of the last block for each arch, kept in first-appearance order. - last_idx <- vapply( - unique(arches), - function(a) max(which(arches == a)), - integer(1) - ) - keep <- sort(last_idx) - out <- preamble - for (i in keep) { - out <- c(out, blocks[[i]]) - } - out -} - -process_issue <- function(title) { - search_url <- sprintf( - "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", - forgejo_base, - repo, - utils::URLencode(title, reserved = TRUE) - ) - resp <- request(search_url) |> - req_headers(Authorization = paste("token", token)) |> - req_perform() - issues <- resp_body_json(resp, simplifyVector = FALSE) - match_idx <- which(vapply(issues, function(x) x$title, character(1)) == title) - if (length(match_idx) == 0) { - cat(sprintf("[skip] No issue found: %s\n", title)) - return(invisible()) - } - - issue_number <- issues[[match_idx[1]]]$number - body <- issues[[match_idx[1]]]$body - if (is.null(body) || nchar(body) == 0) { - cat(sprintf("[skip] Empty body: #%d %s\n", issue_number, title)) - return(invisible()) - } - - lines <- strsplit(body, "\n", fixed = TRUE)[[1]] - before <- sum(grepl("^### ", lines)) - - # Split off the "## Excluded packages" footer so it is preserved verbatim. - excl_idx <- which(lines == "## Excluded packages") - footer <- character(0) - if (length(excl_idx) > 0) { - pre_dash <- which(lines == "---" & seq_along(lines) < excl_idx[1]) - cut <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_idx[1] - footer <- lines[seq(cut, length(lines))] - lines <- lines[seq_len(cut - 1)] - } - - # Platform headers ("## "); everything before the first is preamble. - plat_idx <- which(grepl("^## ", lines)) - if (length(plat_idx) == 0) { - cat(sprintf("[skip] No platform sections: #%d %s\n", issue_number, title)) - return(invisible()) - } - top <- lines[seq_len(plat_idx[1] - 1)] - plat_end <- c(plat_idx[-1] - 1, length(lines)) - - new_lines <- top - for (j in seq_along(plat_idx)) { - pl <- lines[seq(plat_idx[j], plat_end[j])] - new_lines <- c(new_lines, dedupe_platform(pl)) - } - if (length(footer) > 0) { - new_lines <- c(new_lines, footer) - } - - after <- sum(grepl("^### ", new_lines)) - cat(sprintf( - "#%d %s: %d -> %d arch subsections%s\n", - issue_number, - title, - before, - after, - if (dry_run) " (dry run, not patched)" else "" - )) - - if (dry_run) { - return(invisible()) - } - - patch_url <- sprintf( - "%s/repos/%s/issues/%d", - forgejo_base, - repo, - issue_number - ) - request(patch_url) |> - req_headers( - Authorization = paste("token", token), - `Content-Type` = "application/json" - ) |> - req_body_json(list(body = paste(new_lines, collapse = "\n"))) |> - req_method("PATCH") |> - req_perform() - cat(sprintf(" patched #%d\n", issue_number)) -} - -for (t in issue_titles) { - process_issue(t) -} -cat("Done.\n") diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R index 73922b3..4ce30cf 100644 --- a/local/failing-builds-classify.R +++ b/local/failing-builds-classify.R @@ -155,44 +155,6 @@ fingerprint_error <- function(error_text, package = NULL, max_chars = 200L) { fp } -# --------------------------------------------------------------------------- -# Dependency-cascade detection -# --------------------------------------------------------------------------- -# If a build's error_text shows the failure was actually in a DIFFERENT package -# (a dependency that would not compile), return that dependency's name; -# otherwise NA. Used to avoid proposing a per-package fix for a package that -# only fails because a shared dependency does not build (e.g. the ~73 Stan -# packages that fail while building `rstan`). Generalises the RcppParallel -# `applies_to` guard to any dependency named in the log. -failing_dependency <- function(error_text, package) { - if (length(error_text) == 0L || is.na(error_text) || !nzchar(error_text)) { - return(NA_character_) - } - x <- as.character(error_text) - # optional opening quote before the package name: apostrophe, double-quote, - # backtick, or curly quotes -- written as \u escapes so the pattern stays - # valid UTF-8 regardless of source encoding. - q <- "[\u0027\u0022\u0060\u2018\u2019]?" - name <- "([A-Za-z][A-Za-z0-9._]+)" - # Markers R/pak emit naming the package that actually failed to compile. - pats <- c( - paste0("compilation failed for package ", q, name), - paste0("Failed to build source package ", q, name), - paste0("Error in building package ", q, name), - paste0("dependenc(?:y|ies) ", q, name, q, "?[^\\n]*not available") - ) - deps <- character(0L) - for (p in pats) { - hits <- regmatches(x, gregexpr(p, x, perl = TRUE))[[1L]] - if (length(hits) > 0L) { - deps <- c(deps, sub(p, "\\1", hits, perl = TRUE)) - } - } - deps <- sub("[._]+$", "", deps) # drop a trailing sentence period (e.g. "rstan.") - deps <- setdiff(unique(deps), package) # a package failing on its OWN code is not a cascade - if (length(deps) == 0L) NA_character_ else deps[[1L]] -} - # --------------------------------------------------------------------------- # Classification # --------------------------------------------------------------------------- @@ -329,38 +291,18 @@ build_triage_report <- function( unregistered <- setdiff(pkgs, registered_pkgs) auto_proposable <- isTRUE(sig$auto) && sig$matched - # Decide, per package, whether it is genuinely fixable or merely blocked on - # a dependency (so a per-package entry would be useless). Two blocking modes: - # 1. `applies_to`: a package-specific patch (e.g. RcppParallel's) is only - # valid for its own package; other matches are downstream of it. - # 2. data-driven cascade: the package's error_text shows a *different* - # package failed to compile (e.g. the ~73 Stan packages blocked on rstan). - # `blocked_map` maps a blocked package -> the dependency it waits on. - g$blocked_dep <- vapply( - seq_len(nrow(g)), - function(i) failing_dependency(g$error_text[[i]], g$name[[i]]), - character(1L) - ) + # A signature whose fix is package-specific (`applies_to`) may only be + # proposed for that package. Everything else matching it is a downstream + # failure blocked on that package (e.g. RcppParallel dependents carrying + # RcppParallel's own error text) -- never propose those a bogus entry. proposable <- unregistered - blocked_map <- list() + blocked_on <- NULL if (!is.null(sig$applies_to)) { - for (p in setdiff(pkgs, sig$applies_to)) { - blocked_map[[p]] <- sig$applies_to + proposable <- intersect(unregistered, sig$applies_to) + downstream <- setdiff(pkgs, sig$applies_to) + if (length(downstream) > 0L) { + blocked_on <- sig$applies_to } - proposable <- intersect(proposable, sig$applies_to) - } - for (p in proposable) { - deps <- g$blocked_dep[g$name == p] - if (!any(is.na(deps))) { - # every failing build of p is a cascade -> blocked, not fixable here - blocked_map[[p]] <- unique(deps) - } - } - proposable <- setdiff(proposable, names(blocked_map)) - blocked_packages <- names(blocked_map) - blocked_on <- unique(unlist(blocked_map, use.names = FALSE)) - if (length(blocked_on) == 0L) { - blocked_on <- NULL } proposed <- list() @@ -384,8 +326,6 @@ build_triage_report <- function( suggested_fix = sig$fix, applies_to = sig$applies_to, blocked_on = blocked_on, - blocked_packages = blocked_packages, - blocked_map = if (length(blocked_map) > 0L) blocked_map else NULL, fingerprint = names(fp_tab)[[1L]], fingerprint_variants = length(fp_tab), build_count = nrow(g), diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R index ce7b683..51f5bc7 100644 --- a/local/failing-builds-report.R +++ b/local/failing-builds-report.R @@ -194,17 +194,15 @@ for (r in report) { ) cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "") } - } else if (r$auto_proposable && length(r$blocked_packages) == 0L) { - cat(" (all affected packages already have a registry entry)\n") - } - # Blocked packages are shown even when the group also has proposals. - if (length(r$blocked_packages) > 0L) { + } else if (!is.null(r$blocked_on)) { cat(sprintf( - " (blocked on %s -- %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", + " (blocked on %s -- these %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", toString(r$blocked_on), - length(r$blocked_packages), + length(r$packages), toString(r$blocked_on) )) + } else if (r$auto_proposable) { + cat(" (all affected packages already have a registry entry)\n") } } diff --git a/local/fetch-rebuild-packages-from-issue.R b/local/fetch-rebuild-packages-from-issue.R index 9d8043c..26e544d 100644 --- a/local/fetch-rebuild-packages-from-issue.R +++ b/local/fetch-rebuild-packages-from-issue.R @@ -4,6 +4,7 @@ forgejo_base <- "https://git.devxy.io/api/v1" repo <- "devxy/build-cran-binaries" platform <- Sys.getenv("PLATFORM") arch <- Sys.getenv("ARCH") +token <- Sys.getenv("FORGEJO_TOKEN") output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") if (nchar(platform) == 0) { @@ -12,36 +13,6 @@ if (nchar(platform) == 0) { if (nchar(arch) == 0) { stop("ARCH env var is not set") } - -# Prefer the audit's freshly-written RDS. The audit overwrites it each run -# (saveRDS), so unlike the Forgejo issue body it is never subject to the -# duplicate-subsection accumulation bug. Fall back to parsing the issue when the -# RDS is absent (e.g. a fresh runner with no shared cache). -rds_file <- file.path( - Sys.getenv("REBUILD_PKG_RDS_DIR", "/mnt/cache/packages"), - sprintf("weekly_rebuild_%s_%s.rds", platform, arch) -) -if (file.exists(rds_file)) { - pkgs <- tryCatch(as.character(readRDS(rds_file)), error = function(e) NULL) - if (!is.null(pkgs) && length(pkgs) > 0) { - cat(sprintf( - "Using audit RDS %s: %d rebuildable packages for %s/%s\n", - rds_file, - length(pkgs), - platform, - arch - )) - writeLines(pkgs, output_file) - cat(sprintf("Wrote package list to %s\n", output_file)) - q("no") - } - cat(sprintf( - "RDS %s present but empty/unreadable -- falling back to issue\n", - rds_file - )) -} - -token <- Sys.getenv("FORGEJO_TOKEN") if (nchar(token) == 0) { stop("FORGEJO_TOKEN env var is not set") } diff --git a/local/install-bincraft.R b/local/install-bincraft.R index e39cb4f..4665b3b 100644 --- a/local/install-bincraft.R +++ b/local/install-bincraft.R @@ -11,11 +11,10 @@ # # How it works: list the remote tags with `git ls-remote` (no token needed for # the public repo), keep the `vX.Y.Z` release tags, pick the highest version, -# and install it with uvr via `local/uvr-install.sh`. uvr is idempotent on the -# git ref, so re-running keeps the package when it is already current and only -# updates when a newer tag ships. Filtering/sorting is done in R (not via git's -# `--sort`/refspec) so behaviour is identical across git versions and -# `system2()` argument handling. +# and install it with pak. pak is idempotent on the git ref, so re-running keeps +# the package when it is already current and only updates when a newer tag ships. +# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is +# identical across git versions and `system2()` argument handling. repo_url <- Sys.getenv( "BINCRAFT_GIT_URL", @@ -45,43 +44,7 @@ latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][ ] message(sprintf("Installing latest bincraft release: %s", latest)) - -# uvr addresses Forgejo repos as `forgejo::host/owner/repo@ref` rather than as a -# git URL, so drop the scheme and the trailing `.git` from `repo_url`. -spec <- sprintf( - "forgejo::%s@%s", - sub("\\.git$", "", sub("^[a-z]+://", "", repo_url)), - latest -) - -# `local/uvr-install.sh` when run from the repo root, `/work/local/` in the -# build-one image, which copies the two scripts into a flatter layout. -helper <- Sys.getenv("UVR_INSTALL_SH", unset = "") -if (!nzchar(helper)) { - candidates <- c("local/uvr-install.sh", "/work/local/uvr-install.sh") - found <- candidates[file.exists(candidates)] - if (length(found) == 0L) { - stop("Could not locate uvr-install.sh; set UVR_INSTALL_SH", call. = FALSE) - } - helper <- found[1L] -} - -# Point uvr at the R running this script and at the library it would install -# into, so the per-R-minor passes in the build pipelines (which call a different -# Rscript with R_LIBS_USER pointed elsewhere) target their own R and library. -Sys.setenv( - UVR_R_BIN = file.path(R.home("bin"), "R"), - UVR_TARGET_LIB = .libPaths()[1L] -) - -status <- system2(helper, shQuote(spec)) -if (!identical(status, 0L)) { - stop( - sprintf("uvr failed to install %s (exit %s)", spec, status), - call. = FALSE - ) -} - +pak::pak(sprintf("git::%s@%s", repo_url, latest)) message(sprintf( "bincraft %s installed (%s)", as.character(utils::packageVersion("bincraft")), diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 3b508fb..7a3652d 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -15,15 +15,15 @@ suppressPackageStartupMessages(library(data.table)) # Sys.setenv("OS_VERSION" = "3.22") # Sys.setenv("ARCH" = "arm64") -arch <- Sys.getenv("ARCH") +arch = Sys.getenv("ARCH") # target: alpine-322, ubuntu-2404, redhat-9, etc. -platform <- paste( +platform = paste( Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-" ) # Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10") -codename <- bincraft::set_codename(NULL) +codename = bincraft::set_codename(NULL) con <- DBI::dbConnect( RPostgres::Postgres(), @@ -35,8 +35,8 @@ con <- DBI::dbConnect( sslmode = "require" ) -cran_archive <- tools::CRAN_archive_db() -cran_release <- tools::CRAN_package_db() +cran_archive = tools::CRAN_archive_db() +cran_release = tools::CRAN_package_db() # Subset cran_archive to only those packages cran_archive_in_release <- cran_archive[ names(cran_archive) %in% cran_release$Package @@ -84,66 +84,18 @@ s3fs::s3_file_system( region_name = "eu-central-003", refresh = TRUE ) -s3_pkgs <- s3fs::s3_dir_ls( +s3_pkgs = s3fs::s3_dir_ls( sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename), recurse = TRUE ) +# Save the raw S3 file listing for the build step to use as s3_package_cache +# This avoids loading s3fs/reticulate in the build container, saving memory for pak forks +saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds") + file_names <- basename(s3_pkgs) - -# An object occupying a key is not proof a binary was built: a package whose -# build failed has its CRAN source published under exactly that name. Left in -# the cache, `build_binary_package()` reads it as "already built" and skips the -# package forever, which is how alpine324 accumulated ~13.5k source tarballs. -# -# bincraft stamps `Built` only on records it actually built, so the slot's own -# index distinguishes them. A slot last indexed by a bincraft that predates that -# fix stamps `Built` on everything, so the cache is then unchanged from before. -# Archived objects have no index record and are kept: unknown means binary, -# never "rebuild it". -index_url <- sprintf( - "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", - arch, - codename -) -source_served <- tryCatch( - { - con_idx <- gzcon(url(index_url, open = "rb")) - on.exit(close(con_idx), add = TRUE) - idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) - sprintf( - "%s_%s.tar.gz", - idx[is.na(idx[, "Built"]), "Package"], - idx[is.na(idx[, "Built"]), "Version"] - ) - }, - error = function(e) { - cat(sprintf( - "WARNING: could not read %s (%s); keeping the full S3 cache\n", - index_url, - conditionMessage(e) - )) - character(0) - } -) - -binary_cache <- setdiff(file_names, source_served) -cat(sprintf( - "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", - length(file_names), - length(file_names) - length(binary_cache), - length(binary_cache) -)) - -# Save the S3 file listing for the build step to use as s3_package_cache. -# This avoids loading s3fs/reticulate in the build container, saving memory for -# the dependency-installer subprocesses -saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") -# Built from the filtered listing, not the raw one: `s3_dt` is subtracted from -# the build list below, so a source fallback left in here would exclude the very -# package that needs building. -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", binary_cache) -parts <- regmatches(binary_cache, matches) +matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) +parts <- regmatches(file_names, matches) parts <- parts[sapply(parts, length) == 3] s3_dt <- data.table( Package = sapply(parts, `[`, 2), diff --git a/local/patches/RcppParallel/disable-tbb.patch b/local/patches/RcppParallel/disable-tbb.patch new file mode 100644 index 0000000..0fe8ad9 --- /dev/null +++ b/local/patches/RcppParallel/disable-tbb.patch @@ -0,0 +1,16 @@ +diff --git a/src/Makevars.in b/src/Makevars.in +index be8445f..faee771 100644 +--- a/src/Makevars.in ++++ b/src/Makevars.in +@@ -60,7 +60,10 @@ else + endif + + ifeq ($(UNAME), Linux) +- USE_TBB=Linux ++ # bincraft patch: the bundled Intel TBB build hangs/fails on musl (Alpine) ++ # and newer toolchains (g++ 15). Skip it (leave USE_TBB unset) and force the ++ # TinyThread backend so RcppParallel still builds. ++ PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=0 + endif + + ifeq ($(UNAME), SunOS) diff --git a/local/patches/RcppParallel/force-bundled-tbb.patch b/local/patches/RcppParallel/force-bundled-tbb.patch deleted file mode 100644 index 5c37fa6..0000000 --- a/local/patches/RcppParallel/force-bundled-tbb.patch +++ /dev/null @@ -1,150 +0,0 @@ -diff --git a/R/aaa.R b/R/aaa.R -index 568a2aa..bfdab4c 100644 ---- a/R/aaa.R -+++ b/R/aaa.R -@@ -5,4 +5,21 @@ TBB_LIB <- "" - TBB_INC <- "" - - TBB_NAME <- "tbb" --TBB_MALLOC_NAME <- "tbbmalloc" -\ No newline at end of file -+TBB_MALLOC_NAME <- "tbbmalloc" -+ -+# bincraft patch: our build images (and plenty of user environments) export -+# TBB_ROOT / TBB_LIB / TBB_INC. A binary we publish always carries its own -+# oneTBB in RcppParallel/lib -- see the companion changes in -+# tools/config/configure.R and src/install.libs.R -- so honouring those -+# variables at run time is actively harmful: .onLoad() would dyn.load() a -+# second, unrelated TBB into the process next to the bundled one (two copies -+# of the same symbols in the global scope, which segfaults R on load), and -+# RcppParallelLibs() / CxxFlags() would hand that system TBB to dependents -+# such as rstan, putting NEEDED libtbb.so.12 back into their binaries. Read -+# the variables only when explicitly opted back in. -+bincraftGetenv <- function(name, unset = "") { -+ if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") -+ Sys.getenv(name, unset = unset) -+ else -+ unset -+} -\ No newline at end of file -diff --git a/R/tbb.R b/R/tbb.R -index 6f6a745..e407986 100644 ---- a/R/tbb.R -+++ b/R/tbb.R -@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { - sysname <- Sys.info()[["sysname"]] - - # find root for TBB install -- tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) -+ tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) - if (is.null(name)) - return(tbbRoot) - -@@ -58,3 +58,3 @@ tbbCxxFlags <- function() { - # if TBB_INC is set, apply those library paths -- tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) -+ tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) - if (!file.exists(tbbInc)) { -@@ -117,7 +117,7 @@ tbbLdFlags <- function() { - } - - # shortcut if TBB_LIB defined -- tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) -+ tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) - if (nzchar(tbbLib)) { - if (R.version$os == "emscripten") { - fmt <- "-L%1$s -l%2$s" -diff --git a/src/install.libs.R b/src/install.libs.R -index 3b3cfda..c0e6f3e 100644 ---- a/src/install.libs.R -+++ b/src/install.libs.R -@@ -477,6 +477,18 @@ prependFlags <- function(prependFlags, toFlags) { - tbbLib <- Sys.getenv("TBB_LIB") - tbbInc <- Sys.getenv("TBB_INC") - -+# bincraft patch: the companion change in tools/config/configure.R stops an -+# ambient TBB_LIB / TBB_INC from selecting a system TBB, but this script is -+# also run directly by `R CMD INSTALL` (not only through the `tbb` rule in -+# src/Makevars, which passes the configured values), so at install time it -+# still sees the image's environment and would symlink the system libraries -+# into RcppParallel/lib. Drop them here for the same reason, under the same -+# opt-out. -+if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { -+ tbbLib <- "" -+ tbbInc <- "" -+} -+ - args <- commandArgs(trailingOnly = TRUE) - if (identical(args, "build")) { - if (nzchar(tbbLib) && nzchar(tbbInc)) { -diff --git a/tools/config/configure.R b/tools/config/configure.R -index 6293fe1..eae4aaa 100644 ---- a/tools/config/configure.R -+++ b/tools/config/configure.R -@@ -40,6 +40,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) - tbbLib <- Sys.getenv("TBB_LIB", unset = NA) - tbbInc <- Sys.getenv("TBB_INC", unset = NA) - -+# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Several of -+# our build images export these (a leftover from RcppParallel 5.x, whose -+# bundled Intel TBB would not build on musl or with modern g++), and any of -+# them switches the branches below to a system TBB. The published binary then -+# records NEEDED libtbb.so.12 (or libtbb.so.2 for the classic Intel TBB) and -+# gets a RcppParallel/lib full of absolute symlinks into the image's library -+# dir, so it cannot dyn.load on a consumer machine without that exact TBB. -+# 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, -+# so the bundled copy is always the right choice here; forcing it in the -+# package rather than relying on the image environment keeps the binary -+# correct whichever image version CI happens to pull. Set -+# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. -+if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { -+ tbbRoot <- NA -+ tbbLib <- NA -+ tbbInc <- NA -+} -+ - tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") - tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") - -@@ -186,12 +204,23 @@ define( - ) - - # set PKG_LIBS -+# -+# bincraft patch: the library directories below are passed as plain '-L', not -+# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) -+# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', -+# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build -+# host with a distro TBB installed, '-ltbb' resolves to that library and -+# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the -+# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads -+# the system TBB rather than the copy shipped in RcppParallel/lib, and fails -+# outright on a machine that has no system TBB. gcc places a plain '-L' before -+# its built-in dirs, so the intended library is found first. - pkgLibs <- if (!is.na(tbbLib)) { - - # a TBB supplied via TBB_LIB / TBB_ROOT. an rpath is meaningless on Windows, - # where the loader has no equivalent -- see R/zzz.R for how we resolve there - c( -- "-Wl,-L\"$(TBB_LIB)\"", -+ "-L\"$(TBB_LIB)\"", - if (.Platform$OS.type != "windows") - sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), - "-l$(TBB_NAME)", -@@ -201,14 +230,14 @@ pkgLibs <- if (!is.na(tbbLib)) { - } else if (R.version$os == "emscripten") { - - c( -- "-Wl,-Ltbb/build/lib_release", -+ "-Ltbb/build/lib_release", - "-l$(TBB_NAME)" - ) - - } else { - - c( -- "-Wl,-Ltbb/build/lib_release", -+ "-Ltbb/build/lib_release", - "-l$(TBB_NAME)", - "-l$(TBB_MALLOC_NAME)" - ) diff --git a/local/patches/registry.json b/local/patches/registry.json index 2457304..e53f1c9 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -1,13 +1,13 @@ [ { "package": "RcppParallel", - "versions": "6.2.1", - "platforms": ["*"], + "versions": "*", + "platforms": ["alpine", "ubuntu-2604"], "env": {}, "configure_args": [], "makevars": {}, - "patch": "RcppParallel/force-bundled-tbb.patch", - "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, so the system-TBB workaround needed for 5.x is gone, but two things still steer the build back to a system TBB. (1) Ambient TBB_ROOT/TBB_LIB/TBB_INC: several build images still export these (build-env-images dropped them, but the images are rebuilt only by cron/manual runs, so a stale image keeps them), and R reads ~/.Renviron *after* the process environment, so no pipeline-side env override can undo it. configure.R then takes the system-TBB branch, install.libs.R symlinks the image's libraries into RcppParallel/lib as absolute paths, and the binary records NEEDED libtbb.so.12 (libtbb.so.2 for the classic Intel TBB on el8/el9) -- it cannot dyn.load on a consumer machine without that exact TBB. Both files are patched to ignore those variables (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE); install.libs.R needs it separately because R CMD INSTALL runs it outside the src/Makevars rule that passes the configured values. (2) Link order: the bundled branch passes its build dir as '-Wl,-Ltbb/build/lib_release', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,', so '-ltbb' would still resolve to a distro TBB when one is installed; a plain '-L' puts the bundled dir first. Verified on the current build-env-ubuntu:jammy image (which still exports TBB_INC/TBB_LIB): NEEDED libtbb.so, RUNPATH $ORIGIN/../lib, real libtbb.so.2 in RcppParallel/lib, and the package loads with every system libtbb moved away." + "patch": "RcppParallel/disable-tbb.patch", + "reason": "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains (g++ 15 on ubuntu-2604); patch unsets USE_TBB and forces -DRCPP_PARALLEL_USE_TBB=0 so RcppParallel skips the bundled build and uses the TinyThread backend" }, { "package": "fs", @@ -18,5 +18,17 @@ "makevars": {}, "patch": "fs/force-vendored-libuv.patch", "reason": "fs 2.x configure links system libuv whenever pkg-config finds libuv-devel (installed as a build-time sysreq), producing an fs.so with NEEDED libuv.so.1. That binary fails to dyn.load on consumer machines lacking runtime libuv, because install.packages/renv do not install SystemRequirements (only pak does, and only in the build container). The patch short-circuits configure to copy src/Makevars.vendor and build the bundled static libuv (needs cmake) so the binary is self-contained on every platform. An env/pkg-config override was tried first but the rebuilt binary still linked libuv.so.1, so a source patch is used instead." + }, + { + "package": "rstan", + "versions": "*", + "platforms": ["*"], + "env": {}, + "configure_args": [], + "makevars": { + "CPPFLAGS": "-DTBB_INTERFACE_NEW -I/usr/local/include" + }, + "patch": null, + "reason": "StanHeaders' init_threadpool_tbb.hpp unconditionally includes the legacy (removed in oneTBB 2021+) for version detection, breaking compilation of Module.cpp against the bundled oneTBB. Pre-defining TBB_INTERFACE_NEW skips that include and selects the modern tbb/global_control.h + tbb/task_arena.h path that the bundled TBB provides (-I/usr/local/include preserves the default CPPFLAGS the override replaces)" } ] diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 28f65db..80c4ab2 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -160,38 +160,19 @@ unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) { ) } -# Aggregate blocked packages across ALL groups by the dependency they wait on, -# so one dependency (RcppParallel, rstan, sf, ...) is a single line -- deduped -# and ranked by how many distinct dependents it blocks -- instead of repeating -# once per fingerprint group. Reads each group's `blocked_map` (package -> the -# dependency it is blocked on). Returns records sorted by dependent count desc, -# each with up to `max_pkgs` example dependents. -blocked_by_dependency <- function(report, max_pkgs = 15L) { - acc <- list() # dependency -> character vector of dependent packages - for (g in report) { - bm <- g$blocked_map - if (is.null(bm) || length(bm) == 0L) { - next - } - for (pkg in names(bm)) { - for (dep in as.character(unlist(bm[[pkg]]))) { - acc[[dep]] <- unique(c(acc[[dep]], pkg)) - } - } - } - if (length(acc) == 0L) { - return(list()) - } - out <- lapply(names(acc), function(dep) { - pkgs <- acc[[dep]] +# Groups blocked on a dependency (a package-specific fix pinned via `applies_to` +# whose dependents merely carry its error): report the dependency + how many +# dependents wait on it, so fixing it once is recognised as clearing the batch. +blocked_summary <- function(report, max_pkgs = 15L) { + bl <- Filter(function(g) !is.null(g$blocked_on), report) + lapply(bl, function(g) { list( - dependency = dep, - n_packages = length(pkgs), - packages = utils::head(pkgs, max_pkgs), - packages_truncated = length(pkgs) > max_pkgs + blocked_on = g$blocked_on, + n_packages = length(g$packages), + packages = utils::head(g$packages, max_pkgs), + packages_truncated = length(g$packages) > max_pkgs ) }) - out[order(-vapply(out, function(x) x$n_packages, integer(1L)))] } # Does a registry entry's `platforms` apply to a build on `os` (e.g. diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R index f8e1284..4d82a11 100644 --- a/local/proposal-tracking.R +++ b/local/proposal-tracking.R @@ -127,13 +127,13 @@ if (length(retire) > 0L) { # --------------------------------------------------------------------------- # Blind spots: failures the classifier could not auto-propose. # --------------------------------------------------------------------------- -blocked <- blocked_by_dependency(report) +blocked <- blocked_summary(report) unmatched <- unclassified_summary(report) cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") if (length(blocked) > 0L) { for (b in blocked) { - cat(sprintf(" %-20s %5d dependent(s)\n", b$dependency, b$n_packages)) + cat(sprintf(" %s: %d dependent(s) waiting\n", b$blocked_on, b$n_packages)) } } else { cat(" (none)\n") @@ -207,8 +207,8 @@ if (do_issue) { body_lines <- c( body_lines, sprintf( - "- **%s**: %d dependent(s) (e.g. %s%s)", - b$dependency, + "- **%s**: %d dependent(s) waiting (e.g. %s%s)", + b$blocked_on, b$n_packages, toString(b$packages), if (isTRUE(b$packages_truncated)) ", ..." else "" diff --git a/local/propose-patches.R b/local/propose-patches.R index 360df9f..11085f0 100644 --- a/local/propose-patches.R +++ b/local/propose-patches.R @@ -148,20 +148,18 @@ for (r in report) { } } -# Packages blocked on a dependency are reported (aggregated by dependency, -# ranked by impact), not proposed: fixing the named dependency clears the batch. -blocked <- blocked_by_dependency(report) +# Groups blocked on a dependency (e.g. RcppParallel dependents) are reported, +# not proposed: fixing the named dependency clears them all at once. +blocked <- Filter(function(r) !is.null(r$blocked_on), report) if (length(blocked) > 0L) { - n_blocked_pkgs <- length(unique(unlist( - lapply(report, function(r) r$blocked_packages) - ))) - cat(sprintf( - "\nBlocked on a dependency (%d dependencies block %d dependents; fix the dependency, not each dependent):\n", - length(blocked), - n_blocked_pkgs - )) - for (b in blocked) { - cat(sprintf(" %-20s %5d dependent(s)\n", b$dependency, b$n_packages)) + cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") + for (r in blocked) { + cat(sprintf( + " %s: %d package(s) fail because %s does not build\n", + toString(r$blocked_on), + length(r$packages), + toString(r$blocked_on) + )) } } @@ -437,9 +435,7 @@ if (do_write) { save_ledger(merge_ledger(load_ledger(), new_ledger_records)) git <- function(...) { - # system2() with captured output runs via /bin/sh, so shell-quote every arg - # (commit messages contain "()", refs contain "^{}", etc.). - st <- system2("git", shQuote(c(...)), stdout = TRUE, stderr = TRUE) + st <- system2("git", c(...), stdout = TRUE, stderr = TRUE) if (!identical(attr(st, "status"), NULL)) { stop(sprintf( "git %s failed:\n%s", diff --git a/local/r-minor-helpers.R b/local/r-minor-helpers.R index 65aafdf..6f01c29 100644 --- a/local/r-minor-helpers.R +++ b/local/r-minor-helpers.R @@ -31,3 +31,42 @@ parse_build_args <- function(args) { ncpus = as.integer(pos[3L]) ) } + +# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the +# "patched" repo mints a new content hash on every PACKAGES change, so each build +# writes a fresh ~70 MB _metadata/pkgs-.rds (+ patched-/) that is +# never reused. Keep the `keep` newest entries by mtime; only remove entries +# older than `min_age_secs`, so a concurrent split-job's in-flight files are +# never deleted (each build uses a unique hash, so aged entries are +# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are +# not matched and thus preserved. Returns the number of entries removed. +trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"), + keep = 20L, + min_age_secs = 600) { + meta <- file.path(cache_dir, "R", "pkgcache", "_metadata") + if (!nzchar(cache_dir) || !dir.exists(meta)) { + return(0L) + } + entries <- c( + Sys.glob(file.path(meta, "patched-*")), + Sys.glob(file.path(meta, "pkgs-*.rds")) + ) + if (length(entries) == 0L) { + return(0L) + } + info <- file.info(entries) + order_new_first <- order(info$mtime, decreasing = TRUE) + ranked <- entries[order_new_first] + ranked_mtime <- info$mtime[order_new_first] + if (length(ranked) <= keep) { + return(0L) + } + candidates <- ranked[(keep + 1L):length(ranked)] + candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)]) + removable <- candidates[candidate_age >= min_age_secs] + if (length(removable) == 0L) { + return(0L) + } + unlink(removable, recursive = TRUE, force = TRUE) + length(removable) +} diff --git a/local/rebuild-missing-helpers.R b/local/rebuild-missing-helpers.R deleted file mode 100644 index 16a588e..0000000 --- a/local/rebuild-missing-helpers.R +++ /dev/null @@ -1,87 +0,0 @@ -# Pure helpers for local/rebuild-missing.R, kept separate so local/tests can -# source them without executing a rebuild. - -# Interleaved slice of the rebuild list. -# -# The list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, -# rstan*), so contiguous thirds would be badly unbalanced. Interleaving also -# makes each shard's progress counter representative of the slot as a whole. -shard_slice <- function(pkgs, split_into, split_index) { - split_into <- as.integer(split_into) - split_index <- as.integer(split_index) - if (is.na(split_into) || is.na(split_index)) { - stop("shard_slice(): split_into and split_index must be integers") - } - if (split_into < 1L || split_index < 1L || split_index > split_into) { - stop(sprintf( - "shard_slice(): need 1 <= split_index <= split_into, got %s of %s", - split_index, - split_into - )) - } - # seq() errors on a descending range, which is what an empty list or a shard - # index past the end would produce. - if (length(pkgs) < split_index) { - return(pkgs[0L]) - } - pkgs[seq.int(split_index, length(pkgs), by = split_into)] -} - -# Packages that still need building, decided from the bucket rather than from -# remembered progress. -# -# This is bincraft's `check_s3_root_package()` evaluated in bulk: an object -# whose ETag equals CRAN's published MD5sum is byte-identical to CRAN's source, -# so the build that was supposed to replace it has not happened yet. -# -# `etag_by_file` named by `_.tar.gz`, values are unquoted ETags -# `cran_version` named by package -# `cran_md5` named by `_` -# -# Unknown always means "already a binary", never "rebuild it", so an unreadable -# CRAN index or a multipart ETag can never mass-schedule work. -outstanding_packages <- function(pkgs, etag_by_file, cran_version, cran_md5) { - if (length(pkgs) == 0L) { - return(pkgs) - } - - # An empty table indexes to zero length rather than to NA, which would - # recycle the whole result away and silently report "nothing to build". - lookup <- function(table, key) { - if (length(table) == 0L) { - return(rep(NA_character_, length(key))) - } - unname(as.character(table[key])) - } - - version <- lookup(cran_version, pkgs) - file <- sprintf("%s_%s.tar.gz", pkgs, version) - etag <- lookup(etag_by_file, file) - md5 <- lookup(cran_md5, paste(pkgs, version, sep = "_")) - - # No CRAN version means the package cannot be resolved to a tarball at all; - # leave it in and let bincraft report why. - unresolved <- is.na(version) - # No object at the key: never built, so it is outstanding by definition. - absent <- !unresolved & is.na(etag) - # A multipart upload carries a compound ETag rather than an MD5. - unknown <- !is.na(etag) & grepl("-", etag, fixed = TRUE) - - is_source <- !unresolved & - !is.na(etag) & - !unknown & - !is.na(md5) & - etag == md5 - - pkgs[unresolved | absent | is_source] -} - -parse_rebuild_args <- function(args) { - pos <- args[!startsWith(args, "--")] - budget <- as.numeric(pos[3L]) - list( - split_into = as.integer(pos[1L]), - split_index = as.integer(pos[2L]), - budget_hours = if (is.na(budget)) 20 else budget - ) -} diff --git a/local/rebuild-missing.R b/local/rebuild-missing.R deleted file mode 100644 index 369024e..0000000 --- a/local/rebuild-missing.R +++ /dev/null @@ -1,194 +0,0 @@ -### Rebuild one shard of a slot's missing-binary list. -# -# Usage: Rscript local/rebuild-missing.R [budget_hours] -# -# The list itself comes from local/fetch-rebuild-packages-from-issue.R, which -# writes $REBUILD_PKG_LIST (default /tmp/rebuild_pkgs.txt). -# -# Two properties matter here and are the reason this is a script rather than an -# `R -q -e` argument in the pipeline: -# -# * it is restartable. The outstanding set is re-derived from the bucket on -# every start, so a shard that died resumes where it stopped without any -# progress file, and without replaying thousands of per-package HEADs. -# * it terminates. A wall-clock budget stops the loop cleanly instead of the -# run having to be killed, which is what previously skipped the re-index and -# CDN purge and left rebuilt binaries hidden behind stale edge copies. - -options(error = function() { - cat("ERROR:", geterrmessage(), "\n", file = stdout()) - traceback(2) - q(status = 1) -}) - -library(bincraft, quietly = TRUE) - -source(file.path("local", "rebuild-missing-helpers.R")) - -args <- parse_rebuild_args(commandArgs(trailingOnly = TRUE)) -if (is.na(args$split_into) || is.na(args$split_index)) { - stop("usage: rebuild-missing.R [budget_hours]") -} - -list_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") -pkgs <- if (file.exists(list_file)) readLines(list_file) else character(0) -pkgs <- pkgs[nzchar(pkgs)] -if (length(pkgs) == 0L) { - cat("Nothing to rebuild\n") - q("no") -} - -excluded <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] -pkgs <- setdiff(pkgs, excluded) - -mine <- shard_slice(pkgs, args$split_into, args$split_index) -cat(sprintf( - "Shard %s/%s: %s of %s listed packages\n", - args$split_index, - args$split_into, - length(mine), - length(pkgs) -)) - -### Resume: ask the bucket what is still outstanding - -codename <- bincraft::set_codename(NULL) -local_machine <- Sys.info()[["machine"]] -arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" -slot_dir <- sprintf( - "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", - arch, - codename -) - -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE -) - -# One paginated listing instead of a HEAD per package. Not recursed: the -# rebuild passes no `is_r_minor_sensitive`, so it only ever targets the flat -# path, and the resume filter matches that scope deliberately. -info <- tryCatch(s3fs::s3_dir_info(slot_dir), error = function(e) NULL) -etag_by_file <- if (is.null(info) || nrow(info) == 0L) { - cat(sprintf( - "WARNING: could not list %s; building the whole shard\n", - slot_dir - )) - stats::setNames(character(), character()) -} else { - stats::setNames( - gsub('^"|"$', "", as.character(info$etag)), - basename(as.character(info$uri)) - ) -} - -cran <- tryCatch( - { - con <- gzcon(url( - "https://cloud.r-project.org/src/contrib/PACKAGES.gz", - open = "rb" - )) - on.exit(close(con), add = TRUE) - read.dcf(con, fields = c("Package", "Version", "MD5sum")) - }, - error = function(e) { - cat(sprintf( - "WARNING: could not read CRAN's index (%s)\n", - conditionMessage(e) - )) - NULL - } -) -cran_version <- stats::setNames(character(), character()) -cran_md5 <- stats::setNames(character(), character()) -if (!is.null(cran)) { - cran_version <- stats::setNames( - as.character(cran[, "Version"]), - as.character(cran[, "Package"]) - ) - keep <- !is.na(cran[, "MD5sum"]) - cran_md5 <- stats::setNames( - as.character(cran[keep, "MD5sum"]), - paste(cran[keep, "Package"], cran[keep, "Version"], sep = "_") - ) -} - -before <- length(mine) -mine <- outstanding_packages(mine, etag_by_file, cran_version, cran_md5) -cat(sprintf( - "Resume: %s of %s already carry a binary; %s outstanding\n", - before - length(mine), - before, - length(mine) -)) - -if (length(mine) == 0L) { - cat("Nothing outstanding for this shard\n") - q("no") -} - -### Build - -options( - crayon.enabled = TRUE, - Ncpus = as.integer(Sys.getenv("NCPUS", "2")), - future.globals.onReference = NULL -) - -started <- Sys.time() -n <- length(mine) -completed <- 0L -for (i in seq_along(mine)) { - elapsed <- as.numeric(difftime(Sys.time(), started, units = "hours")) - if (elapsed > args$budget_hours) { - cat(sprintf( - "Budget of %sh reached after %d/%d packages; stopping cleanly. The next run resumes from the bucket.\n", - args$budget_hours, - completed, - n - )) - break - } - - x <- mine[i] - cat(sprintf("[%d/%d] %s\n", i, n, x)) - tryCatch( - bincraft::build_binary_package( - x, - tag_limit = 1L, - patches = "local/patches", - s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", - s3_region = "eu-central-003", - s3_bucket = "devxy-rpkgs-binaries", - s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - metadata_db_host = "r-binaries.devxy.io", - metadata_db_name = "build_metadata", - metadata_db_table = "single_builds", - metadata_db_user = "rpkgs", - metadata_db_password = Sys.getenv("PGPASS"), - metadata_db_sslmode = "require", - metadata_db_port = 15432, - archive = TRUE, - upload = TRUE, - store_build_metadata = TRUE - ), - error = function(e) { - cat(sprintf("ERROR building %s - %s\n", x, conditionMessage(e))) - } - ) - completed <- completed + 1L -} - -cat(sprintf( - "Shard %s/%s finished: %d/%d packages processed in %.1fh\n", - args$split_index, - args$split_into, - completed, - n, - as.numeric(difftime(Sys.time(), started, units = "hours")) -)) diff --git a/local/repair-built-stamp.R b/local/repair-built-stamp.R deleted file mode 100644 index 06f73be..0000000 --- a/local/repair-built-stamp.R +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env Rscript - -### Rewrite a broken `Built` stamp across one arch/codename slot. -### -### A slot's index can end up advertising a stamp whose platform triple is -### unusable, e.g. `Built: R 4.5.0; NA; ...`. uvr picks binary vs source by -### matching that triple plus the R minor, so no client matches it and the whole -### slot silently reverts to source-only, which makes uvr compile everything and -### fail wherever a system `-dev` library is missing. -### -### Rewriting it is not a matter of re-running the normal index update. -### `upload_package_index()` reuses the slot's remote `PACKAGES.db`, and -### cranlike's `update_db()` only reparses files whose md5 changed, so entries -### already in the database keep the stamp they were written with. Dropping -### `PACKAGES.db` to force a full reparse does work, but for an S3 repo cranlike -### reads each package's metadata from the CRAN *source* mirror on GitHub, so a -### 25k-entry slot means 25k requests to raw.githubusercontent.com and a real -### risk of being rate-limited part-way through. -### -### Only the `Built` column is wrong, so correct it in place instead: patch the -### column in `PACKAGES.db`, put the database back, and let -### `upload_package_index()` re-emit `PACKAGES*` from it. `update_db()` always -### rewrites the index files even when nothing was reparsed, so no tarball is -### re-read and nothing is fetched from GitHub. -### -### The replacement comes from `bincraft::built_stamp()` under the R running -### this script, so run it under the R version the slot should advertise (the -### `R_VERSION` its entry in `.crow/process-updates.yaml` uses). That is what a -### healthy `upload_package_index()` run would have written. -### -### Usage, inside the platform's build image: -### Rscript local/repair-built-stamp.R [--apply] -### -### Without `--apply` it reports what it would change and touches nothing. - -suppressPackageStartupMessages({ - library(bincraft) -}) - -args <- commandArgs(trailingOnly = TRUE) -arch <- args[1L] -apply_changes <- "--apply" %in% args - -if (is.na(arch) || !nzchar(arch)) { - stop( - "Usage: Rscript local/repair-built-stamp.R [--apply]", - call. = FALSE - ) -} - -bucket <- "devxy-rpkgs-binaries" -endpoint <- "https://s3.eu-central-003.backblazeb2.com" -region <- "eu-central-003" - -codename <- bincraft::set_codename(NULL) -if (is.null(codename) || is.na(codename) || !nzchar(codename)) { - stop( - "Could not detect a codename from /etc/os-release; run this in a build image.", - call. = FALSE - ) -} - -# built_stamp() refuses an unusable platform, so a broken build image fails here -# rather than writing a second bad stamp over the first one. -stamp <- bincraft::built_stamp() -message(sprintf("Slot: %s/%s", arch, codename)) -message(sprintf("New stamp: %s", stamp)) - -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = endpoint, - region_name = region, - refresh = TRUE -) - -base_dir <- file.path(bucket, arch, codename, "latest", "src", "contrib") - -# A stamp is broken when its platform component is absent or literally "NA". -broken_stamp_where <- paste( - "Built IS NULL", - "OR Built LIKE '%; NA;%'", - "OR Built LIKE '%; ;%'" -) - -# The generic slot plus every per-minor sub-slot, which carry the same stamp and -# are poisoned by the same run. -r_minors <- sub( - "^.*/R/([0-9]+\\.[0-9]+)\\.[0-9]+$", - "\\1", - list.dirs("/opt/R", recursive = FALSE) -) -r_minors <- unique(grep("^[0-9]+\\.[0-9]+$", r_minors, value = TRUE)) -slots <- c(base_dir, file.path(base_dir, r_minors)) - -repair_slot <- function(slot) { - db_remote <- file.path(slot, "PACKAGES.db") - if (!s3fs::s3_file_exists(db_remote)) { - message(sprintf(" %s: no PACKAGES.db, skipping", slot)) - return(invisible(NULL)) - } - - db_local <- tempfile(fileext = ".db") - s3fs::s3_file_download(db_remote, db_local, overwrite = TRUE) - - con <- DBI::dbConnect(RSQLite::SQLite(), db_local) - on.exit(DBI::dbDisconnect(con), add = TRUE) - - total <- DBI::dbGetQuery(con, "SELECT COUNT(*) AS n FROM packages")$n - broken <- DBI::dbGetQuery( - con, - sprintf("SELECT COUNT(*) AS n FROM packages WHERE %s", broken_stamp_where) - )$n - - message(sprintf(" %s: %s entries, %s broken", slot, total, broken)) - if (broken == 0L) { - return(invisible(NULL)) - } - if (!apply_changes) { - message(" (dry run, pass --apply to rewrite)") - return(invisible(NULL)) - } - - DBI::dbExecute( - con, - sprintf("UPDATE packages SET Built = ? WHERE %s", broken_stamp_where), - params = list(stamp) - ) - DBI::dbDisconnect(con) - on.exit() - - s3fs::s3_file_upload(db_local, db_remote, overwrite = TRUE) - message(sprintf(" rewrote %s entries and uploaded PACKAGES.db", broken)) - invisible(NULL) -} - -invisible(lapply(slots, repair_slot)) - -if (!apply_changes) { - message("Dry run complete; nothing was changed.") - quit(save = "no") -} - -# Re-emit PACKAGES/PACKAGES.gz/PACKAGES.rds from the corrected database. Nothing -# is reparsed, because no tarball's md5 changed. -message("Re-emitting index files from the corrected database...") -bincraft::upload_package_index( - codename = codename, - s3_endpoint = endpoint, - s3_region = region, - s3_bucket = bucket, - s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") -) -for (minor in r_minors) { - try( - bincraft::upload_package_index( - codename = codename, - r_minor = minor, - s3_endpoint = endpoint, - s3_region = region, - s3_bucket = bucket, - s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") - ), - silent = FALSE - ) -} - -message("Done. Verify with:") -message(sprintf( - " curl -sS https://cran.devxy.io/%s/%s/latest/src/contrib/PACKAGES | grep '^Built:' | sort | uniq -c", - arch, - codename -)) diff --git a/local/test-package-loading.R b/local/test-package-loading.R index a3b7a28..26a2e82 100644 --- a/local/test-package-loading.R +++ b/local/test-package-loading.R @@ -1,28 +1,22 @@ -# Installs every CRAN package one by one and checks that it loads. Dependencies -# go through uvr via local/uvr-install.sh, which bootstraps the uvr binary on -# first use and installs into .libPaths()[1]. -uvr_install <- function(pkg) { - Sys.setenv( - UVR_R_BIN = file.path(R.home("bin"), "R"), - UVR_TARGET_LIB = .libPaths()[1L] +install.packages( + "pak", + repos = sprintf( + "https://r-lib.github.io/p/pak/stable/%s/%s/%s", + .Platform$pkgType, + R.Version()$os, + R.Version()$arch ) - status <- system2("local/uvr-install.sh", shQuote(pkg)) - if (!identical(status, 0L)) { - stop( - sprintf("uvr failed to install %s (exit %s)", pkg, status), - call. = FALSE - ) - } -} +) +Sys.setenv(PKG_SYSREQS = TRUE) all_pkgs <- rownames(available.packages()) -to_skip <- c("ABRSQOL", "ACA", "ACE.CoCo") -all_pkgs <- setdiff(all_pkgs, to_skip) +to_skip = c("ABRSQOL", "ACA", "ACE.CoCo") +all_pkgs = setdiff(all_pkgs, to_skip) for (i in all_pkgs) { message(sprintf("\nInstalling %s", i)) - uvr_install(i) + pak::pkg_install(i) library(i, character.only = TRUE) } @@ -329,7 +323,7 @@ if (length(to_process) == 0) { } else { for (i in to_process) { message(sprintf("\nInstalling %s", i)) - uvr_install(i) + pak::pkg_install(i) library(i, character.only = TRUE) } # Update to_skip to include all up to the last processed diff --git a/local/tests/test-failing-builds-classify.R b/local/tests/test-failing-builds-classify.R index d4df9f5..a365e14 100644 --- a/local/tests/test-failing-builds-classify.R +++ b/local/tests/test-failing-builds-classify.R @@ -105,49 +105,6 @@ test_that("RcppParallel dependents are blocked, not proposed a per-package patch expect_identical(grp$blocked_on, "RcppParallel") }) -test_that("failing_dependency names the dependency that actually failed", { - # A leaf package (BFpack) that fails building its rstan dependency. - txt <- paste( - "Error in installing dependencies for package BFpack with tag 1.6.1", - "Failed to build source package rstan.", - "ERROR: compilation failed for package ‘rstan’", - sep = "\n" - ) - expect_identical(failing_dependency(txt, "BFpack"), "rstan") - # A package failing in its OWN compilation is not a cascade. - own <- "ERROR: compilation failed for package ‘BFpack’" - expect_true(is.na(failing_dependency(own, "BFpack"))) - expect_true(is.na(failing_dependency(NA_character_, "x"))) - expect_true(is.na(failing_dependency("some unrelated error", "x"))) -}) - -test_that("Stan packages blocked on rstan are not proposed a per-package entry", { - # BFpack/GMLTM fail building rstan; the tbb error is in rstan's compile. - cascade <- paste( - "Failed to build source package rstan.", - "fatal error: tbb/tbb_stddef.h: No such file or directory", - sep = "\n" - ) - failures <- data.frame( - name = c("BFpack", "GMLTM", "someOwnPkg"), - platform = "ubuntu-2604", - arch = "amd64", - error_text = c( - cascade, - cascade, - # someOwnPkg fails in its OWN compile on the same header -> fixable. - "someOwnPkg.cpp: fatal error: tbb/tbb_stddef.h: No such file or directory" - ), - stringsAsFactors = FALSE - ) - report <- build_triage_report(failures, registered_pkgs = character(0L)) - grp <- Filter(function(g) g$signature == "tbb-stddef-removed", report)[[1L]] - # Only the own-compile package is proposed; the rstan cascades are blocked. - expect_identical(names(grp$proposed_entries), "someOwnPkg") - expect_setequal(grp$blocked_packages, c("BFpack", "GMLTM")) - expect_identical(grp$blocked_on, "rstan") -}) - test_that("RcppParallel itself is still proposed when it is the failing package", { failures <- data.frame( name = "RcppParallel", diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index 869d2b3..05cd0e9 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -146,35 +146,20 @@ test_that("unclassified_summary ranks unknown groups and caps output", { expect_identical(capped$dropped_groups, 1L) }) -test_that("blocked_by_dependency aggregates across groups, deduped and ranked", { - # RcppParallel dependents split across platforms/fingerprints -> separate - # groups, but one aggregated line; a dependent seen twice is counted once. +test_that("blocked_summary lists each dependency and its dependent count", { failures <- data.frame( - name = c("ACEsimFit", "AovBay", "AdaptGauss", "ACEsimFit", "loner"), - platform = c( - "ubuntu-2604", - "ubuntu-2604", - "alpine-324", - "alpine-324", - "ubuntu-2604" - ), + name = c("ACEsimFit", "AovBay", "AdaptGauss"), + platform = "ubuntu-2604", arch = "amd64", - error_text = c( - rep("Error: USE_TBB=Linux is not supported on this toolchain", 4L), - "Failed to build source package rstan.\nfatal error: tbb/tbb_stddef.h" - ), + error_text = "Error: USE_TBB=Linux is not supported on this toolchain", stringsAsFactors = FALSE ) report <- build_triage_report(failures, registered_pkgs = character(0L)) - agg <- blocked_by_dependency(report, max_pkgs = 2L) - deps <- vapply(agg, function(b) b$dependency, character(1L)) - expect_true("RcppParallel" %in% deps && "rstan" %in% deps) - rcpp <- Filter(function(b) b$dependency == "RcppParallel", agg)[[1L]] - # ACEsimFit appears in two groups -> counted once (3 distinct dependents). - expect_identical(rcpp$n_packages, 3L) - expect_true(rcpp$packages_truncated) # capped at max_pkgs = 2 - # Ranked by dependent count: RcppParallel (3) before rstan (1). - expect_identical(deps[[1L]], "RcppParallel") + b <- blocked_summary(report, max_pkgs = 2L) + expect_length(b, 1L) + expect_identical(b[[1L]]$blocked_on, "RcppParallel") + expect_identical(b[[1L]]$n_packages, 3L) + expect_true(b[[1L]]$packages_truncated) }) test_that("entry_applies_to_os matches codename, family, and wildcard", { diff --git a/local/tests/test-rebuild-missing.R b/local/tests/test-rebuild-missing.R deleted file mode 100644 index a5e70b9..0000000 --- a/local/tests/test-rebuild-missing.R +++ /dev/null @@ -1,94 +0,0 @@ -source(file.path("..", "rebuild-missing-helpers.R")) - -test_that("shard_slice partitions the list without gaps or overlap", { - pkgs <- letters[1:10] - parts <- lapply(1:3, function(i) shard_slice(pkgs, 3, i)) - - expect_identical(parts[[1]], c("a", "d", "g", "j")) - expect_identical(parts[[2]], c("b", "e", "h")) - expect_identical(parts[[3]], c("c", "f", "i")) - - expect_identical(sort(unlist(parts)), sort(pkgs)) - expect_identical(anyDuplicated(unlist(parts)), 0L) -}) - -test_that("shard_slice is deterministic and survives short lists", { - expect_identical( - shard_slice(letters[1:10], 3, 2), - shard_slice(letters[1:10], 3, 2) - ) - expect_identical(shard_slice(character(0), 3, 1), character(0)) - # more shards than packages: the tail shards get nothing rather than erroring - expect_identical(shard_slice(c("a"), 3, 1), "a") - expect_identical(shard_slice(c("a"), 3, 2), character(0)) -}) - -test_that("shard_slice rejects an out-of-range index", { - expect_error(shard_slice(letters, 3, 4), "split_index") - expect_error(shard_slice(letters, 3, 0), "split_index") -}) - -test_that("outstanding_packages keeps source fallbacks and drops real binaries", { - cran_version <- c(httr = "1.4.8", R6 = "2.6.1", curl = "7.1.0") - cran_md5 <- c( - httr_1.4.8 = "8756015b94a9cff6f410ca4de8557f12", - R6_2.6.1 = "f01b1787f12797c29194d63c9afd5d70", - curl_7.1.0 = "8af2ccbf5d85dc18866f45f1f26f348d" - ) - etag <- c( - # byte-identical to CRAN: the build never happened - "httr_1.4.8.tar.gz" = "8756015b94a9cff6f410ca4de8557f12", - # a real binary was published - "R6_2.6.1.tar.gz" = "9d6087ee9adda3f0a3b8067cfc652c05" - # curl has no object at all - ) - - out <- outstanding_packages( - c("httr", "R6", "curl"), - etag, - cran_version, - cran_md5 - ) - expect_identical(out, c("httr", "curl")) -}) - -test_that("outstanding_packages treats unknowns as already built", { - cran_version <- c(a = "1.0", b = "1.0") - cran_md5 <- c(a_1.0 = "aaaa") - - # a multipart ETag carries no MD5, and `b` is missing from CRAN's index: - # neither may schedule a rebuild - etag <- c("a_1.0.tar.gz" = "abc-3", "b_1.0.tar.gz" = "bbbb") - - expect_identical( - outstanding_packages(c("a", "b"), etag, cran_version, cran_md5), - character(0) - ) -}) - -test_that("outstanding_packages keeps a package CRAN has no version for", { - out <- outstanding_packages( - "ghost", - c(), - c(other = "1.0"), - c(other_1.0 = "aaaa") - ) - expect_identical(out, "ghost") -}) - -test_that("outstanding_packages handles an empty list", { - expect_identical( - outstanding_packages(character(0), c(), c(), c()), - character(0) - ) -}) - -test_that("parse_rebuild_args defaults the budget", { - a <- parse_rebuild_args(c("3", "2")) - expect_identical(a$split_into, 3L) - expect_identical(a$split_index, 2L) - expect_identical(a$budget_hours, 20) - - b <- parse_rebuild_args(c("3", "2", "1.5")) - expect_identical(b$budget_hours, 1.5) -}) diff --git a/local/tests/test-trim-pkgcache.R b/local/tests/test-trim-pkgcache.R new file mode 100644 index 0000000..1a5b418 --- /dev/null +++ b/local/tests/test-trim-pkgcache.R @@ -0,0 +1,68 @@ +source(file.path("..", "r-minor-helpers.R")) + +# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is +# set to `age_secs` in the past so we can exercise the age gate deterministically. +make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) { + root <- tempfile("pkgcache-") + meta <- file.path(root, "R", "pkgcache", "_metadata") + dir.create(meta, recursive = TRUE) + old <- Sys.time() - age_secs + mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p } + mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p } + for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i))) + for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i))) + if (keep_repos) { + mk_dir(file.path(meta, "CRAN-075c426938")) + mk_dir(file.path(meta, "BioCsoft-1ac964ed6c")) + mk_file(file.path(meta, "bioc-sysreqs.dcf.gz")) + mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive + } + root +} + +n_churn <- function(root) { + meta <- file.path(root, "R", "pkgcache", "_metadata") + length(Sys.glob(file.path(meta, "patched-*"))) + + length(Sys.glob(file.path(meta, "pkgs-*.rds"))) +} + +test_that("empty cache_dir is a no-op", { + expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L) +}) + +test_that("missing _metadata dir is a no-op", { + expect_identical( + trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0), + 0L + ) +}) + +test_that("fewer than keep entries removes nothing", { + root <- make_meta(patched = 2L, pkgs = 2L) + expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L) + expect_identical(n_churn(root), 4L) +}) + +test_that("trims down to keep newest, leaving churn == keep", { + root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old + removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0) + expect_identical(removed, 40L) + expect_identical(n_churn(root), 20L) +}) + +test_that("entries younger than min_age_secs are protected", { + root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60) + # keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed + expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L) + expect_identical(n_churn(root), 30L) +}) + +test_that("stable repo dirs and pkg downloads are never touched", { + root <- make_meta(patched = 30L, pkgs = 30L) + trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0) + meta <- file.path(root, "R", "pkgcache", "_metadata") + expect_true(dir.exists(file.path(meta, "CRAN-075c426938"))) + expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c"))) + expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz"))) + expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg"))) +}) diff --git a/local/trial-build-patch.R b/local/trial-build-patch.R index 2229074..4e8030a 100644 --- a/local/trial-build-patch.R +++ b/local/trial-build-patch.R @@ -40,26 +40,23 @@ cat(sprintf( patches_dir )) -# build_binary_package() catches build failures internally and RETURNS "error" -# for the failed tag rather than throwing (bincraft >= v4.4.7), so inspect the -# return value -- checking only for a thrown exception reports a broken build as -# passing (false green). -res <- tryCatch( - bincraft::build_binary_package( - package, - tag_limit = 1L, - patches = patches_dir, - archive = FALSE, - upload = FALSE, - store_build_metadata = FALSE - ), +ok <- tryCatch( + { + bincraft::build_binary_package( + package, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ) + TRUE + }, error = function(e) { - cat(sprintf("Trial build FAILED (threw): %s\n", conditionMessage(e))) - "error" + cat(sprintf("Trial build FAILED: %s\n", conditionMessage(e))) + FALSE } ) -flat <- as.character(unlist(res)) -ok <- length(flat) > 0L && !("error" %in% flat) if (ok) { cat(sprintf("Trial build OK: %s builds with the proposed patch.\n", package)) diff --git a/local/trial-build-registry.R b/local/trial-build-registry.R index 9708408..f649b62 100644 --- a/local/trial-build-registry.R +++ b/local/trial-build-registry.R @@ -52,16 +52,9 @@ current <- if (file.exists(registry_file)) { # silently treating the base as empty would trial-build the WHOLE registry # instead of just the entries the branch adds. registry_rel <- "local/patches/registry.json" -# system2() with captured output runs via /bin/sh, so shell-quote the git args -# (refs contain "^{}" and ":" that the shell would otherwise mangle). ref_ok <- suppressWarnings(system2( "git", - shQuote(c( - "rev-parse", - "--verify", - "--quiet", - sprintf("%s^{commit}", base_ref) - )), + c("rev-parse", "--verify", "--quiet", sprintf("%s^{commit}", base_ref)), stdout = TRUE, stderr = FALSE )) @@ -70,14 +63,14 @@ if (!is.null(attr(ref_ok, "status"))) { } in_base <- suppressWarnings(system2( "git", - shQuote(c("ls-tree", base_ref, "--", registry_rel)), + c("ls-tree", base_ref, "--", registry_rel), stdout = TRUE, stderr = FALSE )) file_in_base <- length(in_base) > 0L && any(nzchar(in_base)) base_json <- suppressWarnings(system2( "git", - shQuote(c("show", sprintf("%s:%s", base_ref, registry_rel))), + c("show", sprintf("%s:%s", base_ref, registry_rel)), stdout = TRUE, stderr = FALSE )) @@ -117,34 +110,23 @@ results <- vapply( pkgs, function(pkg) { cat(sprintf("\n=== trial build: %s ===\n", pkg)) - # bincraft::build_binary_package() catches build failures internally and - # RETURNS "error" for the failed tag rather than throwing, so a green gate - # must inspect the return value -- checking only for a thrown exception - # reports a broken build as passing. - res <- tryCatch( - bincraft::build_binary_package( - pkg, - tag_limit = 1L, - patches = patches_dir, - archive = FALSE, - upload = FALSE, - store_build_metadata = FALSE - ), + tryCatch( + { + bincraft::build_binary_package( + pkg, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ) + TRUE + }, error = function(e) { - cat(sprintf("FAILED %s (threw): %s\n", pkg, conditionMessage(e))) - "error" + cat(sprintf("FAILED %s: %s\n", pkg, conditionMessage(e))) + FALSE } ) - flat <- as.character(unlist(res)) - ok <- length(flat) > 0L && !("error" %in% flat) - if (!ok) { - cat(sprintf( - "FAILED %s: build did not succeed (result: %s)\n", - pkg, - if (length(flat) > 0L) toString(flat) else "" - )) - } - ok }, logical(1L) ) diff --git a/local/uvr-install.sh b/local/uvr-install.sh deleted file mode 100755 index 3966e85..0000000 --- a/local/uvr-install.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/sh -# Install R packages into the CI library with uvr (https://github.com/nbafrank/uvr). -# -# Usage: -# local/uvr-install.sh httr2 jsonlite -# local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft@v4.4.3 -# -# Replaces `pak::pak(...)`. uvr is project-scoped: `uvr add` refuses to run -# outside a project and always writes to `.uvr/library/`, and only -# `uvr sync --library` can target an existing library. The project is therefore -# minted in a scratch directory under TMPDIR and thrown away afterwards; that -# also keeps `uvr init`'s `.Rprofile` out of the repo checkout, where it would -# hijack `.libPaths()` for every other R call in the pipeline. -# -# Pruning is a no-op here: uvr disables it whenever `--library` is passed, -# precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds -# bincraft and its dependencies alongside whatever this script installs). -# -# System dependencies come from uvr's vendored r-system-requirements rules, so -# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed. -# -# Environment: -# UVR_R_BIN R interpreter to install for; set by install-bincraft.R so -# the per-R-minor passes target their own R, not the primary -# R_VERSION fallback interpreter selector (/opt/R//bin/R) -# UVR_TARGET_LIB target library; defaults to R_LIBS_USER, then to the -# active R's .libPaths()[1] (which is where pak wrote) -# UVR_INSTALL_DIR where the uvr binary lands (default /usr/local/bin) - -set -eu - -# renovate: datasource=github-releases depName=nbafrank/uvr -UVR_PIN="v0.4.6" - -if [ "$#" -eq 0 ]; then - echo "usage: $0 ..." >&2 - exit 2 -fi - -# The build images keep R under /opt/R/ and off PATH. uvr resolves the -# interpreter via PATH and never downloads one unless `uvr r install` is run, so -# put the requested R first. -r_bin="${UVR_R_BIN:-}" -if [ -z "$r_bin" ] && [ -n "${R_VERSION:-}" ] && [ -x "/opt/R/${R_VERSION}/bin/R" ]; then - r_bin="/opt/R/${R_VERSION}/bin/R" -fi -if [ -n "$r_bin" ]; then - PATH="$(dirname "$r_bin"):$PATH" - export PATH -else - r_bin="$(command -v R)" -fi - -target_lib="${UVR_TARGET_LIB:-${R_LIBS_USER:-}}" -if [ -z "$target_lib" ]; then - target_lib="$("$r_bin" --no-echo --no-save -e 'cat(.libPaths()[1])')" -fi -if [ -z "$target_lib" ]; then - echo "error: could not determine a target library; set UVR_TARGET_LIB" >&2 - exit 2 -fi -mkdir -p "$target_lib" - -# Pin the manifest to the active R so the lockfile's R stays in step with the -# library's R sentinel. Without that, uvr can decide the library is ABI-stale -# and wipe it -- and this target is shared with bincraft. uvr only discovers R -# via PATH/R_HOME (it does not scan /opt/R), so the R put on PATH above is the -# only candidate this constraint can resolve to. -# shellcheck disable=SC2016 # $major/$minor are R expressions, not shell vars -r_full="$("$r_bin" --no-echo --no-save -e 'cat(paste(R.version$major, R.version$minor, sep = "."))')" - -install_dir="${UVR_INSTALL_DIR:-/usr/local/bin}" -uvr_bin="${install_dir}/uvr" -if [ ! -x "$uvr_bin" ]; then - echo "Bootstrapping uvr ${UVR_PIN} into ${install_dir}" - UVR_INSTALL_DIR="$install_dir" UVR_VERSION="$UVR_PIN" \ - sh -c 'curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | sh' -fi - -project_dir="${TMPDIR:-/tmp}/uvr-ci-$$" -rm -rf "$project_dir" -mkdir -p "$project_dir" -trap 'rm -rf "$project_dir"' EXIT -cd "$project_dir" - -"$uvr_bin" init --here --r-version "$r_full" -# --no-install resolves and locks only; retry because concurrent shards can -# expose short-lived DNS or CRAN-index failures and uvr rolls the manifest back -# cleanly after an unsuccessful resolution. -add_attempt=1 -while ! "$uvr_bin" add --no-install "$@"; do - if [ "$add_attempt" -ge 4 ]; then - echo "error: uvr add failed after ${add_attempt} attempts" >&2 - exit 1 - fi - add_delay=$((add_attempt * 10)) - echo "warning: uvr add attempt ${add_attempt} failed; retrying in ${add_delay}s" >&2 - sleep "$add_delay" - add_attempt=$((add_attempt + 1)) -done - -# TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs -# `apt-get install` for every resolved system dependency without refreshing the -# index first, and the ubuntu build images end their apt layers with -# `rm -rf /var/lib/apt/lists/*`. With no index apt cannot resolve a package that -# exists and is enabled, so `igraph needs: libglpk-dev` fails the whole build -# with `E: Unable to locate package libglpk-dev` on ubuntu 26.04. -# -# Fixed upstream in `e491b2e`, tagged one day after v0.4.5 (nbafrank/uvr#250), -# and the images pick it up via build-env-images#23 — but only after an image -# rebuild is triggered, which is why this runs here too. -# -# apt only: `apk add` fetches its index implicitly and dnf refreshes expired -# metadata on its own. Non-fatal, since a refresh failure still leaves whatever -# index is already there, and the install's own error is the more actionable one. -if command -v apt-get >/dev/null 2>&1; then - apt-get update -qq || echo "warning: apt-get update failed; continuing" >&2 -fi - -"$uvr_bin" sync --library "$target_lib" --install-system-deps diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index e7ca776..6b32037 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -7,6 +7,7 @@ options(error = function() { suppressPackageStartupMessages(library(data.table)) library(DBI, quietly = TRUE) library(RPostgres, quietly = TRUE) +library(s3fs, quietly = TRUE) library(jsonlite, quietly = TRUE) library(httr2, quietly = TRUE) @@ -49,8 +50,8 @@ cat(sprintf( )) # --------------------------------------------------------------------------- -# 1. Query PostgreSQL for known build failures (before anything that uses curl, -# to avoid C++ pointer conflicts between curl and RPostgres/libpq) +# 1. Query PostgreSQL for known build failures (before s3fs init to avoid +# C++ pointer conflicts between s3fs/curl and RPostgres/libpq) # --------------------------------------------------------------------------- cat("Connecting to PostgreSQL...\n") con <- DBI::dbConnect( @@ -96,71 +97,62 @@ cran_dt <- data.table( ) # --------------------------------------------------------------------------- -# 3. Published binaries +# 3. S3 tarballs # --------------------------------------------------------------------------- -# Read the slot's own index rather than listing the bucket. An object being -# present does not mean a binary was built: when a build fails, bincraft -# publishes the CRAN source tarball in its place so the package stays -# installable, and a bucket listing cannot tell the two apart. That is how -# amd64/alpine324 came to hold 13,547 CRAN sources that this audit never -# reported. bincraft stamps `Built` only on records it actually built, so the -# index answers the question a listing cannot. -# -# A slot last indexed by a bincraft that predates the source-fallback fix -# stamps `Built` on every record, including the fallbacks, so this reports -# exactly what it used to until that slot is re-indexed. -index_url <- sprintf( - "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", +cat("Connecting to S3...\n") +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) + +s3_path <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, s3_codename ) -cat(sprintf("Reading package index: %s\n", index_url)) +cat(sprintf("Listing S3 path: %s\n", s3_path)) -index <- tryCatch( - { - con <- gzcon(url(index_url, open = "rb")) - on.exit(close(con), add = TRUE) - read.dcf(con, fields = c("Package", "Version", "Built")) - }, +s3_pkgs <- tryCatch( + s3fs::s3_dir_ls(s3_path, recurse = FALSE), error = function(e) { cat(sprintf( - "WARNING: Could not read %s: %s\n", - index_url, + "WARNING: Could not list S3 path %s: %s\n", + s3_path, conditionMessage(e) )) - NULL + character(0) } ) -if (is.null(index) || nrow(index) == 0) { - binary_dt <- data.table(Package = character(0), Version = character(0)) +file_names <- basename(s3_pkgs) +matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) +parts <- regmatches(file_names, matches) +parts <- parts[sapply(parts, length) == 3] +if (length(parts) == 0) { + s3_dt <- data.table(Package = character(0), Version = character(0)) } else { - built <- !is.na(index[, "Built"]) - binary_dt <- data.table( - Package = as.character(index[built, "Package"]), - Version = as.character(index[built, "Version"]) + s3_dt <- data.table( + Package = sapply(parts, `[`, 2), + Version = sapply(parts, `[`, 3) ) - cat(sprintf( - "Index holds %d records, %d of them built binaries (%d served as CRAN source)\n", - nrow(index), - sum(built), - sum(!built) - )) } cat(sprintf( - "S3 contains %d binaries for %s/%s\n", - nrow(binary_dt), + "S3 contains %d tarballs for %s/%s\n", + nrow(s3_dt), arch, s3_codename )) # --------------------------------------------------------------------------- -# 4. Find missing packages (CRAN release version without a binary in S3) +# 4. Find missing packages (CRAN release version not in S3) # --------------------------------------------------------------------------- setkey(cran_dt, Package, Version) -setkey(binary_dt, Package, Version) -missing_dt <- cran_dt[!binary_dt] +setkey(s3_dt, Package, Version) +missing_dt <- cran_dt[!s3_dt] cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) # --------------------------------------------------------------------------- @@ -336,6 +328,7 @@ if (nchar(forgejo_token) == 0) { } plat_header <- sprintf("## %s", platform) + arch_header <- sprintf("### %s", arch) plat_idx <- which(lines == plat_header) @@ -366,40 +359,31 @@ if (nchar(forgejo_token) == 0) { } plat_lines <- lines[seq(pi, plat_end)] + arch_local_idx <- which(plat_lines == arch_header) - # Arch subsection headers within the platform block (### arm64 / ### amd64). - # Match by prefix: headers carry a " (N missing, M to rebuild)" suffix, so - # exact-equality matching never found the existing block and silently - # appended a duplicate on every run. Remove *all* blocks for this arch - # (collapsing any previously accumulated duplicates), then write one fresh - # block, so the issue holds a single current subsection per arch. - sub_hdr <- which(grepl("^### ", plat_lines)) - arch_re <- sprintf("^### %s( |$)", arch) - - if (length(sub_hdr) == 0) { - # No arch subsections yet -- append after the platform header/preamble. - new_plat_lines <- c(plat_lines, "", arch_lines) + if (length(arch_local_idx) == 0) { + # Append arch subsection at end of platform block + lines <- c( + lines[seq_len(plat_end)], + "", + arch_lines, + lines[seq(plat_end + 1, length(lines))] + ) } else { - preamble <- plat_lines[seq_len(sub_hdr[1] - 1)] - # Each subsection runs from its ### header to the line before the next - # ### header (#### known-failures stays inside its own block). - sub_end <- c(sub_hdr[-1] - 1, length(plat_lines)) - kept <- character(0) - for (k in seq_along(sub_hdr)) { - block <- plat_lines[seq(sub_hdr[k], sub_end[k])] - if (!grepl(arch_re, block[1])) { - kept <- c(kept, block) - } - } - new_plat_lines <- c(preamble, kept, "", arch_lines) - } + ai <- pi + arch_local_idx[1] - 1 # absolute line index - tail_lines <- if (plat_end < length(lines)) { - lines[seq(plat_end + 1, length(lines))] - } else { - character(0) + # End of arch subsection + next_arch <- which( + grepl("^### |^## |^---", lines) & seq_along(lines) > ai + ) + arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end + + lines <- c( + lines[seq_len(ai - 1)], + arch_lines, + lines[seq(arch_end + 1, length(lines))] + ) } - lines <- c(lines[seq_len(pi - 1)], new_plat_lines, tail_lines) } # Rebuild excluded footer diff --git a/plans/2026-08-07-per-minor-edge-routing.md b/plans/2026-08-07-per-minor-edge-routing.md deleted file mode 100644 index 3c3e320..0000000 --- a/plans/2026-08-07-per-minor-edge-routing.md +++ /dev/null @@ -1,166 +0,0 @@ -# Per-R-minor edge routing implementation plan - -Spec: `specs/2026-08-07-per-minor-edge-routing-design.md` - -**Goal:** let a stock `install.packages()` see the per-minor packages by routing `PACKAGES*` requests to `…/src/contrib//`, where `bincraft` publishes a union index. - -**Architecture:** the union is built in `bincraft`; the edge script only redirects index requests, gated on a `UNION_SLOTS` script variable; the script lives in this repo and is applied by OpenTofu. - -**Tech stack:** Deno / TypeScript (Bunny Edge Scripting, SDK 0.12), OpenTofu with `BunnyWay/bunnynet` 0.17, R (bincraft). - -## Global constraints - -- Redirect only `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds`; never a tarball, because the union index already carries the correct tarball URL for both classes of package. -- Every redirect carries `Cache-Control: no-store`; redirect targets stay UA-independent. -- `UNION_SLOTS` is empty by default, so deploying the script is a no-op until a slot is backfilled. -- A slot is `/`, e.g. `amd64/alpine324`. -- Verified prerequisites: `PACKAGES*` is served `cdn-cache: BYPASS`, so the script sees every index request; `Deno.env.get()` reads script variables; the SDK local server listens on `127.0.0.1:8080`. - ---- - -## Task 1: Edge script and its test matrix - -**Files:** - -- Create: `edge/rpkgs-router.ts` -- Create: `edge/rpkgs-router.test.ts` -- Modify: `justfile` (add `edge-test`) - -**Produces:** a single-file script deployable as `bunnynet_compute_script.content`, reading `UNION_SLOTS` from the environment. - -- [ ] **Step 1: write the test matrix first** - -`edge/rpkgs-router.test.ts` spawns `deno run -A edge/rpkgs-router.ts` with `UNION_SLOTS=amd64/alpine324`, waits for `127.0.0.1:8080`, and issues requests with `redirect: "manual"`. - -Cases, asserted on the `location` header (or its absence): - -| # | path | User-Agent | expectation | -| --- | ------------------------------------------------------ | --------------------------------------------- | ------------------------------------------------------------ | -| 1 | `/amd64/alpine324/latest/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)` | 302 → `…/src/contrib/4.5/PACKAGES.gz` | -| 2 | same | `R (4.6.0 …)` | 302 → `…/src/contrib/4.6/PACKAGES.gz` | -| 3 | same, but slot `amd64/noble` | `R (4.5.3 …)` | no redirect (slot not in `UNION_SLOTS`) | -| 4 | `…/src/contrib/curl_7.1.0.tar.gz` | `R (4.5.3 …)` | no redirect | -| 5 | `…/src/contrib/4.5/PACKAGES.gz` | `R (4.5.3 …)` | no redirect (loop guard) | -| 6 | `…/src/contrib/PACKAGES.gz` | `curl/8.0` | no redirect (no R minor) | -| 7 | `/src/contrib/PACKAGES.gz` | alpine UA with `Alpine Linux … 3.24` | 302 → `/amd64/alpine324/latest/src/contrib/4.5/PACKAGES.gz` | -| 8 | `/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)`, no distro | 302 → `cran.r-project.org`, **not** a `linux-musl` slot | -| 9 | `/src/contrib/foo_1.0.tar.gz` | `R (4.5.1 aarch64-apple-darwin20 …)` | 302 → `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | -| 10 | `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | any | 302 → `cran.r-project.org` | -| 11 | any redirect above | — | `cache-control: no-store` | - -- [ ] **Step 2: run the tests and watch them fail** - -`just edge-test` → every case fails, because `edge/rpkgs-router.ts` does not exist. - -- [ ] **Step 3: write `edge/rpkgs-router.ts`** - -Order of evaluation in `onOriginRequest`: - -1. normalise `//` runs in the path -2. darwin `/src/contrib/*` → `/bin/macosx//contrib//` -3. `/bin/macosx/**` → CRAN -4. `/{arch}/{os}/latest/src/contrib/`: pass through if `rest` already starts with `/`, or is not an index file, or the slot is not in `UNION_SLOTS`, or the UA has no R minor; otherwise redirect into `/` -5. `/`, `/src/contrib`, `/src/contrib/**`: resolve arch+os from the UA, redirect to CRAN when the distro is unidentifiable, otherwise redirect to the qualified path, adding `/` under the same index-file rule -6. anything else: pass through - -The R minor comes from either `R/4.5.3` or `R (4.5.3 …)`, so a stock UA is enough. The `linux-gnu` / `linux-musl` fallback in `parseUserAgent` is deleted: those are not slot names. - -- [ ] **Step 4: run the tests until they pass** - -`just edge-test` - -- [ ] **Step 5: commit** - -```bash -git add edge/rpkgs-router.ts edge/rpkgs-router.test.ts justfile -git commit -m "feat(edge): route PACKAGES requests to the per-R-minor slot" -``` - ---- - -## Task 2: Manage the script from OpenTofu - -**Files:** - -- Modify: `cdn.tf` - -**Consumes:** `edge/rpkgs-router.ts` from Task 1. - -- [ ] **Step 1: add the resources** - -```terraform -resource "bunnynet_compute_script" "rpkgs_router" { - type = "middleware" - name = "rpkgs-router" - content = file("${path.module}/edge/rpkgs-router.ts") -} - -resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { - script = bunnynet_compute_script.rpkgs_router.id - name = "UNION_SLOTS" - default_value = "" - required = false -} -``` - -and replace `middleware_script = 29277` with `middleware_script = bunnynet_compute_script.rpkgs_router.id`. - -- [ ] **Step 2: validate** - -`tofu init -backend=false && tofu validate` - -- [ ] **Step 3: import the existing script (needs `BUNNYNET_API_KEY`)** - -```bash -tofu import bunnynet_compute_script.rpkgs_router 29277 -tofu plan -``` - -The plan must show an in-place `content` update and **no** replacement of the pull zone. A replacement means the import did not take. - -- [ ] **Step 4: commit** - -```bash -git add cdn.tf -git commit -m "feat(cdn): manage the edge middleware script from this repo" -``` - ---- - -## Task 3: Union index writer in bincraft - -**Files (repo `codefloe.com/rpkgs/bincraft`):** - -- Modify: `R/package_index.R` -- Test: `tests/testthat/test-package_index.R` - -**Produces:** `write_union_index(flat_records, minor_records)` returning the merged records, called from `upload_package_index()` when `r_minor` is set. - -- [ ] **Step 1: write the failing tests** - -- a package present in both slots keeps the per-minor record, with `Path = "4.5"` -- a package only in the flat slot survives with no `Path` -- a package only in the per-minor slot survives with `Path = "4.5"` -- a union smaller than the flat input raises an error rather than returning - -- [ ] **Step 2: run them and watch them fail** - -`Rscript -e 'testthat::test_file("tests/testthat/test-package_index.R")'` - -- [ ] **Step 3: implement `write_union_index()` and call it from `upload_package_index()`** - -After `update_PACKAGES()` has written the per-minor index, read the flat slot's `PACKAGES.rds`, set `Path = ` on the per-minor records, drop the flat records for packages the per-minor slot already has, and rewrite `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` in the per-minor slot. - -- [ ] **Step 4: run the tests until they pass** - -- [ ] **Step 5: commit and open the PR against bincraft** - ---- - -## Task 4: Roll out slot by slot - -- [ ] Re-index one slot (`amd64/alpine324`, R 4.5) and confirm the union index lists both `curl` (per-minor, `Path: 4.5`) and `jsonlite` (flat, no `Path`). -- [ ] Set `UNION_SLOTS = "amd64/alpine324"` and confirm in `reg.devxy.io/r/r-alpine:4.5-3.24` that `available.packages()` returns the union count and `"curl" %in% rownames(...)`. -- [ ] Add `arm64/alpine324`, then the remaining slots. - -`install.packages("curl")` will still fail to build on `alpine324` until that slot's source tarballs are replaced with real binaries. That is tracked separately. diff --git a/provider.tf b/provider.tf index d4f2564..b267fcc 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.18" + version = "~> 0.15" } } } diff --git a/renovate.json b/renovate.json index 7b2c3b3..53c7306 100644 --- a/renovate.json +++ b/renovate.json @@ -1,12 +1,7 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": ["local>devxy/renovate-config"], - "ignorePaths": [ - "docker/**", - ".crow/process-updates.yaml", - ".crow/build-all-versions.yaml", - ".crow/weekly-rebuild-missing.yaml" - ], + "ignorePaths": ["docker/**"], "customManagers": [ { "customType": "regex", @@ -19,13 +14,6 @@ "packageNameTemplate": "https://codefloe.com/rpkgs/bincraft.git", "datasourceTemplate": "git-tags", "extractVersionTemplate": "^v?(?.+)$" - }, - { - "customType": "regex", - "fileMatch": ["^local/uvr-install\\.sh$"], - "matchStrings": [ - "# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+UVR_PIN=\"(?[^\"]+)\"" - ] } ] } diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh deleted file mode 100755 index 648dfc3..0000000 --- a/scripts/purge_cdn_zone.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -# -# Purge the entire BunnyCDN pull zone. -# -# `purge_cdn_cache.sh` purges the five index files by URL, which is right after -# a normal update: new packages arrive at new URLs, so only the index is stale. -# -# A rebuild is different. It replaces an object *in place*: a package whose -# build failed was published as its CRAN source, and the rebuilt binary takes -# exactly the same URL. The zone caches tarballs for ~370 days -# (`cache_expiration_time` in cdn.tf), so without a purge every client keeps -# receiving the source tarball for up to a year, and nothing about it looks -# wrong from the outside. -# -# Purging per URL would mean one API call per replaced package -- ~13.5k per -# arch against a rate-limited endpoint, where a single missed call leaves a -# silently stale package. One zone purge is a single call regardless of how many -# objects were replaced. The cost is a cold cache for everything else, which is -# why this is not used by the daily update path. -# -# The public hostnames currently use separate pull zones, so callers must pass -# every zone that serves the repository. A zone can be identified by its -# numeric ID or by one of its hostnames; hostname lookup avoids persisting IDs -# that change when a zone is recreated. -# -# Usage: -# purge_cdn_zone.sh [...] -# -set -euo pipefail - -if (($# < 2)); then - echo "usage: $0 [...]" >&2 - exit 2 -fi - -api_key="$1" -shift - -resolve_zone_id() { - local zone="$1" - local response_file - local zone_id - - if [[ "${zone}" =~ ^[0-9]+$ ]]; then - echo "${zone}" - return - fi - - response_file=$(mktemp) - curl -sS -o "${response_file}" \ - -H "AccessKey: ${api_key}" \ - "https://api.bunny.net/pullzone" - zone_id=$( - jq -r --arg hostname "${zone}" \ - '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \ - "${response_file}" - ) - rm -f "${response_file}" - - if [[ -z "${zone_id}" ]]; then - echo "Could not find BunnyCDN pull zone for hostname ${zone}" >&2 - exit 1 - fi - - echo "${zone_id}" -} - -for zone in "$@"; do - zone_id=$(resolve_zone_id "${zone}") - echo "Purging BunnyCDN pull zone ${zone_id}" - - response_file="/tmp/purge_zone_response_${zone_id}.txt" - status=$( - curl -sS -o "${response_file}" -w '%{http_code}' -X POST \ - -H "AccessKey: ${api_key}" \ - -H "Content-Length: 0" \ - "https://api.bunny.net/pullzone/${zone_id}/purgeCache" - ) - - if [[ "${status}" != "200" && "${status}" != "204" ]]; then - echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 - cat "${response_file}" >&2 - exit 1 - fi - - echo "Purged pull zone ${zone_id} (HTTP ${status})" -done diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh deleted file mode 100755 index 28acd92..0000000 --- a/scripts/verify-r-minor-routing.sh +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env bash -# -# Verify per-R-minor index routing for cran.rpkgs.com across every published -# / slot. -# -# The edge router (edge/rpkgs-router.ts) rewrites PACKAGES* requests to -# `contrib//` when the slot is listed in UNION_SLOTS and the client's -# User-Agent carries an R minor. Two properties have to hold before a slot may -# be added to UNION_SLOTS: -# -# 1. the per-minor index is a UNION of the per-minor and flat slots, so -# routing to it hides nothing the flat index carries; and -# 2. every R minor a client might report resolves to an index that exists, -# because contribPath() does not check existence and has no fallback. -# -# Modes: -# (default) Resolve routing decisions without depending on UNION_SLOTS being -# set. Safe to run before enabling: it reads the per-minor indexes -# directly and reproduces the router's target path. -# --live Additionally drive the real CDN with R User-Agents and assert the -# bytes served match the expected index. Only meaningful once the -# slot is in UNION_SLOTS. -# -# Usage: -# scripts/verify-r-minor-routing.sh -# scripts/verify-r-minor-routing.sh --live -# MINORS="4.4 4.5" SAMPLE=10 scripts/verify-r-minor-routing.sh -# -# Exits non-zero if any check fails. - -set -uo pipefail - -BASE=${BASE:-https://cran.rpkgs.com} -ARCHES=${ARCHES:-"amd64 arm64"} -DISTROS=${DISTROS:-"resolute noble jammy rhel8 rhel9 rhel10 alpine323 alpine324"} -# The supported window: the current R minor plus the two previous, matching -# build-env-images' R_VERSION_LATEST/PREV1/PREV2 and cdn.tf's -# local.rpkgs_supported_minors. Each of these must have a published index. -MINORS=${MINORS:-"4.4 4.5 4.6"} -# Minors we deliberately do not serve. These must have NO published index and, -# once routing is live, must be sent to CRAN for sources rather than 404ing or -# being handed binaries built under another minor. -EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"} -# How many Path: targets to HEAD-check per slot/minor. 0 disables. -SAMPLE=${SAMPLE:-5} -# Largest package-count shortfall a non-primary minor may have against the best -# minor on the same slot before coverage counts as uneven. A slot built under -# one R minor carries fewer per-minor binaries for the others; until that gap -# closes, "full coverage for ABI-sensitive packages" is not a claim we can make. -PARITY_TOLERANCE=${PARITY_TOLERANCE:-25} -LIVE=0 - -for arg in "$@"; do - case "$arg" in - --live) LIVE=1 ;; - -h | --help) - sed -n '2,32p' "$0" - exit 0 - ;; - *) - echo "unknown argument: $arg" >&2 - exit 2 - ;; - esac -done - -WORK=$(mktemp -d) -trap 'rm -rf "$WORK"' EXIT - -PASS=0 -FAIL=0 -FAILURES="" - -ok() { - PASS=$((PASS + 1)) - printf ' ok %s\n' "$1" -} - -bad() { - FAIL=$((FAIL + 1)) - FAILURES="${FAILURES}\n - $1" - printf ' FAIL %s\n' "$1" -} - -# Fetch a URL into a file, echoing the HTTP status. Cached per URL. -fetch() { - local url=$1 dest=$2 ua=${3:-} - if [ -s "$dest" ]; then - cat "$dest.status" - return 0 - fi - local status - if [ -n "$ua" ]; then - status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) - else - status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) - fi - echo "$status" > "$dest.status" - echo "$status" -} - -head_status() { - curl -sS -o /dev/null -w '%{http_code}' -I --max-time 60 "$1" 2>/dev/null -} - -# Package names from a gzipped PACKAGES index, sorted. -pkg_names() { - gunzip -c "$1" 2>/dev/null | awk '/^Package:/ {print $2}' | sort -u -} - -# " " pairs for entries that carry a Path: field. -path_entries() { - gunzip -c "$1" 2>/dev/null | awk ' - /^Package:/ { pkg = $2; ver = ""; path = "" } - /^Version:/ { ver = $2 } - /^Path:/ { path = $2 } - /^$/ { if (pkg != "" && path != "") print pkg, ver, path; pkg = "" } - END { if (pkg != "" && path != "") print pkg, ver, path } - ' -} - -# An R User-Agent of the shape R actually sends. -r_user_agent() { - printf 'R/%s.0 (Ubuntu 24.04; codename=noble) (x86_64-pc-linux-gnu x86_64 linux-gnu)' "$1" -} - -echo "verify-r-minor-routing: $BASE" -echo " slots: $(echo "$ARCHES" | wc -w) arch x $(echo "$DISTROS" | wc -w) os" -echo " minors: $MINORS (excluded: $EXCLUDED_MINORS)" -echo " live: $LIVE" -echo - -for arch in $ARCHES; do - for distro in $DISTROS; do - slot="$arch/$distro" - echo "$slot" - - flat_url="$BASE/$slot/latest/src/contrib/PACKAGES.gz" - flat_file="$WORK/${arch}-${distro}-flat.gz" - flat_status=$(fetch "$flat_url" "$flat_file") - - if [ "$flat_status" != "200" ]; then - bad "$slot flat index unreachable (HTTP $flat_status)" - continue - fi - - pkg_names "$flat_file" > "$flat_file.names" - flat_count=$(wc -l < "$flat_file.names") - if [ "$flat_count" -lt 1000 ]; then - bad "$slot flat index has only $flat_count packages" - continue - fi - ok "$slot flat index: $flat_count packages" - - for minor in $MINORS; do - minor_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" - minor_file="$WORK/${arch}-${distro}-${minor}.gz" - minor_status=$(fetch "$minor_url" "$minor_file") - - # A minor the router would route to must exist, or clients on that R - # version get a 404 and see no packages at all. - if [ "$minor_status" != "200" ]; then - bad "$slot R $minor index missing (HTTP $minor_status) - routing would 404 for R $minor clients" - continue - fi - - pkg_names "$minor_file" > "$minor_file.names" - minor_count=$(wc -l < "$minor_file.names") - - # Union property: nothing the flat index carries may be missing here. - missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5) - missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l) - if [ "$missing_count" -ne 0 ]; then - bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))" - else - ok "$slot R $minor index: $minor_count packages, union holds" - fi - - # Path: entries steer to per-minor binaries; they must resolve. - if [ "$SAMPLE" -gt 0 ]; then - path_entries "$minor_file" > "$minor_file.paths" - total_paths=$(wc -l < "$minor_file.paths") - broken=0 - checked=0 - while read -r pkg ver path; do - [ -z "${pkg:-}" ] && continue - tarball="$BASE/$slot/latest/src/contrib/$path/${pkg}_${ver}.tar.gz" - status=$(head_status "$tarball") - checked=$((checked + 1)) - if [ "$status" != "200" ]; then - broken=$((broken + 1)) - [ "$broken" -le 2 ] && printf ' broken target: %s (HTTP %s)\n' "$tarball" "$status" - fi - done < <(shuf -n "$SAMPLE" "$minor_file.paths" 2>/dev/null || head -n "$SAMPLE" "$minor_file.paths") - - if [ "$broken" -ne 0 ]; then - bad "$slot R $minor: $broken/$checked sampled Path: targets do not resolve (of $total_paths total)" - elif [ "$checked" -gt 0 ]; then - ok "$slot R $minor: $checked/$checked sampled Path: targets resolve (of $total_paths total)" - fi - fi - - # Live routing: what a real R client on this minor actually receives. - if [ "$LIVE" -eq 1 ]; then - ua=$(r_user_agent "$minor") - live_file="$WORK/${arch}-${distro}-${minor}-live.gz" - live_status=$(fetch "$flat_url" "$live_file" "$ua") - if [ "$live_status" != "200" ]; then - bad "$slot R $minor live request failed (HTTP $live_status)" - elif cmp -s "$live_file" "$minor_file"; then - ok "$slot R $minor live request served the per-minor index" - elif cmp -s "$live_file" "$flat_file"; then - bad "$slot R $minor live request served the FLAT index - slot not in UNION_SLOTS?" - else - bad "$slot R $minor live request served neither the per-minor nor the flat index" - fi - fi - done - - # Coverage parity across minors. The union property only guarantees no - # client loses packages relative to the flat index; it says nothing about a - # 4.4 client seeing fewer packages than a 4.5 client on the same slot. - best=0 - for minor in $MINORS; do - f="$WORK/${arch}-${distro}-${minor}.gz.names" - [ -s "$f" ] || continue - c=$(wc -l < "$f") - [ "$c" -gt "$best" ] && best=$c - done - if [ "$best" -gt 0 ]; then - uneven="" - for minor in $MINORS; do - f="$WORK/${arch}-${distro}-${minor}.gz.names" - [ -s "$f" ] || continue - c=$(wc -l < "$f") - gap=$((best - c)) - [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap" - done - if [ -n "$uneven" ]; then - bad "$slot coverage uneven across minors (vs best $best):$uneven" - else - ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)" - fi - fi - - # Excluded minors: no published index, and under --live a redirect to CRAN. - for minor in $EXCLUDED_MINORS; do - ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" - ex_status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 60 "$ex_url" 2>/dev/null) - if [ "$ex_status" = "200" ]; then - bad "$slot R $minor is excluded but an index is published - the two lists disagree" - else - ok "$slot R $minor correctly has no published index" - fi - - if [ "$LIVE" -eq 1 ]; then - loc=$(curl -sS -o /dev/null -w '%{redirect_url}' -A "$(r_user_agent "$minor")" \ - --max-time 60 "$flat_url" 2>/dev/null) - case "$loc" in - https://cran.r-project.org/*) - ok "$slot R $minor is sent to CRAN ($loc)" - ;; - "") - bad "$slot R $minor was served directly instead of being sent to CRAN" - ;; - *) - bad "$slot R $minor redirected somewhere unexpected: $loc" - ;; - esac - fi - done - - # A client whose User-Agent carries no R version must keep getting the flat - # index, never a per-minor one. - if [ "$LIVE" -eq 1 ]; then - plain_file="$WORK/${arch}-${distro}-plain.gz" - plain_status=$(fetch "$flat_url" "$plain_file" "curl/8.0.0") - if [ "$plain_status" != "200" ]; then - bad "$slot non-R User-Agent request failed (HTTP $plain_status)" - elif cmp -s "$plain_file" "$flat_file"; then - ok "$slot non-R User-Agent still served the flat index" - else - bad "$slot non-R User-Agent was routed away from the flat index" - fi - - # Tarball requests must never be rewritten into a per-minor directory: - # flat-slot packages do not live there. - sample_pkg=$(gunzip -c "$flat_file" | awk '/^Package:/ {p=$2} /^Version:/ {print p, $2; exit}') - if [ -n "$sample_pkg" ]; then - # shellcheck disable=SC2086 # deliberate split into $1 (package) and $2 (version) - set -- $sample_pkg - tb="$BASE/$slot/latest/src/contrib/${1}_${2}.tar.gz" - tb_status=$(curl -sS -o /dev/null -w '%{http_code}' -A "$(r_user_agent 4.5)" --max-time 60 "$tb" 2>/dev/null) - if [ "$tb_status" = "200" ]; then - ok "$slot tarball request under an R User-Agent still resolves" - else - bad "$slot tarball ${1}_${2}.tar.gz broke under an R User-Agent (HTTP $tb_status)" - fi - fi - fi - done -done - -echo -echo "passed: $PASS failed: $FAIL" -if [ "$FAIL" -ne 0 ]; then - printf 'failures:%b\n' "$FAILURES" - exit 1 -fi diff --git a/specs/2026-08-07-per-minor-edge-routing-design.md b/specs/2026-08-07-per-minor-edge-routing-design.md deleted file mode 100644 index b55812c..0000000 --- a/specs/2026-08-07-per-minor-edge-routing-design.md +++ /dev/null @@ -1,174 +0,0 @@ -# Design: Routing clients to per-R-minor binary slots - -Date: 2026-08-07 -Status: Approved (pending spec review) - -## Problem - -`bincraft` routes ABI-"risky" packages to a per-minor slot `…/latest/src/contrib//` and indexes every directory independently (`upload_package_index()` calls `cranlike::update_PACKAGES()` on one prefix at a time). -Nothing unions those indices, and `contrib.url()` only ever yields `/src/contrib`, so no value of `options(repos)` can address a per-minor slot. -Only `uvr` resolves per-minor URLs, which means the per-minor slots are invisible to `install.packages()` by construction. - -Measured on 2026-08-07: - -| slot | flat `src/contrib` | `src/contrib/4.5` | unique packages only in the per-minor slot | -| ----------------- | ------------------ | ----------------- | ------------------------------------------ | -| `amd64/alpine324` | 21 640 | 3 310 | 2 886 | -| `amd64/noble` | 24 495 | 398 | 23 | - -This is what issue #63 records as "missing binaries" on `alpine324`. -The packages are not missing; they are in a directory base R cannot reach. -The user-visible symptom in `reg.devxy.io/r/r-alpine:4.5-3.24` is: - -``` -> install.packages("curl") -Warning message: -package 'curl' is not available for this version of R -``` - -A second, unrelated defect exists on the same slot and is **out of scope here**: many `alpine324` tarballs are byte-identical CRAN _source_ tarballs that the index nevertheless stamps `Built: R 4.5.3; …-linux-musl`. -Routing exposes `curl`; only a rebuild of that slot makes it install. - -## Goal - -Let a stock `install.packages()` see one complete package list for its own R minor, without duplicating tarballs and without an R-version-varying cache key anywhere in the CDN. - -## Key constraint that drives the design - -R resolves a package's download URL from the index, not from the request path, and it keeps the `contriburl` it _asked for_ rather than the one it was redirected to. -Measured with `options(repos = …/latest)` against a middleware that redirects the index into `4.5/`: - -``` -curl available: TRUE -curl repo: …/latest/src/contrib # the flat URL, not the 4.5 one it was served from -``` - -So the union index is always addressed relative to the **flat** directory, whatever path it was fetched from. -`available.packages()` honours a `Path:` field and folds it into the `Repository` column, which gives the whole routing for free: - -- a per-minor record carries `Path: `, so its tarball is fetched from `…/src/contrib//` -- a flat record carries no `Path`, so its tarball is fetched from `…/src/contrib/` - -Verified end to end against the live CDN with a locally built union index for `amd64/alpine324` (31 507 records): - -``` -curl: 7.1.0 -> …/latest/src/contrib/4.5 -> curl_7.1.0.tar.gz 717 725 B -jsonlite: 2.0.0 -> …/latest/src/contrib -> jsonlite_2.0.0.tar.gz 1 055 849 B -``` - -The corollary is that the edge script must **not** rewrite tarball requests: every tarball URL is already correct when it leaves the client, and redirecting one into `/` would break exactly the flat packages the union is meant to preserve. - -The complementary trick does not work: R's `gzcon()` reads only the first member of a concatenated gzip stream (10 291 of an expected 31 931 records), so an edge-side merge would have to fully decompress and recompress both indices and additionally 404 `PACKAGES.rds` to stop R preferring it. -That is why the union is produced in `bincraft`, not at the edge. - -## Approaches considered - -| Approach | Where the union lives | Verdict | -| ----------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- | -| **A. Union index written by `bincraft` (chosen)** | per-minor `PACKAGES*`, per-minor entries carry `Path: ` | Edge does one redirect; `PACKAGES.rds` stays correct; no duplication | -| B. Merge at the edge | middleware fetches both indices, recompresses | ~2 MB decompress/recompress per cache fill, cache key must include the R minor, breaks R's `.rds` fast path | -| C. Move the minor up the path (`latest//src/contrib/`) | addressable by `options(repos)` directly | No edge logic at all, but a full layout migration and breaks the published URL contract | - -Chosen: **A**. - -## Architecture - -### bincraft: union index (separate PR) - -After writing a per-minor index, republish it as a union of that slot and the flat slot: - -1. Read the flat slot's `PACKAGES.rds` and the per-minor slot's own records. -2. Set `Path: ` on every per-minor record, so its tarball resolves into the per-minor directory. -3. Drop every flat record whose package is already present in the per-minor slot, so the per-minor build always wins, and leave the survivors without a `Path`. -4. Write the merged `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib//`. - -Guard: refuse to publish a union with fewer records than the flat index it was built from. -A truncated union is worse than no union, because it silently removes packages from every client on that minor. - -### Edge script (this repo) - -The script routes `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` requests, and nothing else. - -``` -normalize path -parseClient(UA) -> { rMinor, arch, os } # rMinor from "R (4.5.3 …)" or "R/4.5.3" -darwin branches # unchanged -if path is /{arch}/{os}/latest/src/contrib/PACKAGES* - already under // ? pass through # loop guard - rMinor known && slot in UNION_SLOTS ? 302 -> …/src/contrib//PACKAGES* - else pass through # flat slot, today's behaviour -if path is /src/contrib/… # bare root - resolve arch + os; unknown -> 302 to CRAN - then apply the same PACKAGES* rule -else pass through -``` - -Redirects carry `Cache-Control: no-store`. -Every cacheable URL is therefore UA-independent, and no cache key has to vary by R version. - -### Repaired bare-root detection - -The bare `https://cran.rpkgs.com` form is currently broken for every Linux client that uses a stock R user agent. -`ALPINE_REGEX`, `UBUNTU_REGEX` and `RHEL_REGEX` only match a Posit-style user agent the user has to set by hand; stock R never carries the distro, so the script falls through to `extractOs()` and redirects to a slot that does not exist: - -``` -UA: R (4.5.3 x86_64-pc-linux-musl …) -> 302 /amd64/linux-musl/latest/… (404) -UA: R (4.5.3 x86_64-pc-linux-gnu …) -> 302 /amd64/linux-gnu/latest/… (404) -``` - -The fallback to a phantom `linux-musl` / `linux-gnu` slot is removed. -An unidentifiable distro redirects to CRAN, which is the existing behaviour for an unparseable user agent. -The R _minor_ is always present in a stock user agent, so per-minor routing itself does not depend on distro detection. - -### Rollout gate - -`UNION_SLOTS` is a `bunnynet_compute_script_variable` listing the slots whose per-minor index is already a union. -It is empty by default, so deploying the script changes nothing until `bincraft` has backfilled a slot, and a rollback is a variable edit rather than a code deploy. -All slots currently carry `4.4`, `4.5` and `4.6`; a client on any other minor falls through to the flat slot. - -### Deployment from this repo - -The script is a file in the repo, applied by the existing OpenTofu configuration: - -``` -edge/rpkgs-router.ts # the script -edge/rpkgs-router.test.ts # UA x path -> expected Location matrix -cdn.tf # bunnynet_compute_script + _variable -``` - -Provider `BunnyWay/bunnynet` v0.17.0 (already pinned) ships `bunnynet_compute_script` with `content` loadable via `file()`, plus `bunnynet_compute_script_variable`. -`middleware_script = bunnynet_compute_script.rpkgs_router.id` replaces the hard-coded `29277`, after a one-time `tofu import` of the existing script. - -## Error handling - -- Unknown R minor, or a slot not listed in `UNION_SLOTS`: pass through to the flat slot. - The client sees exactly today's behaviour. -- Unparseable distro on the bare-root form: redirect to CRAN. -- A request already under `…/src/contrib//`: pass through, so a redirect can never loop. -- A per-minor slot that does not exist for a listed minor: the client gets the origin's 404. - `UNION_SLOTS` is the operator's assertion that the slot is ready, so this is a configuration error, not a runtime condition to paper over. - -## Testing - -Local, before any apply: `deno run -A edge/rpkgs-router.ts` serves the middleware against the real origin, so `edge/rpkgs-router.test.ts` drives the whole matrix against that local server. - -- User agent matrix: R 4.4 / 4.5 / 4.6 on musl and gnu, both arches, Posit-style and stock forms, plus a darwin UA and a non-R UA. -- Path matrix: `PACKAGES`, `PACKAGES.gz`, `PACKAGES.rds`, a tarball, a path already under `4.5/`, and `/src/contrib/…` on the bare root. -- Assertion is the `Location` header (or its absence), not the body. - -After apply, a smoke test against `cran.rpkgs.com`: - -- `available.packages()` inside `reg.devxy.io/r/r-alpine:4.5-3.24` returns the union count, and `"curl" %in% rownames(...)` is `TRUE`. -- A flat-slot package still downloads from `…/src/contrib/`, and a per-minor package downloads from `…/src/contrib//`. - -## Out of scope - -- `Meta/archive.rds` stays flat-only, so `remotes::install_version()` does not see per-minor archives. -- The `alpine324` source-tarball defect: that slot serves CRAN sources stamped as binaries, and needs a rebuild independent of this work. -- Any change to how `uvr` resolves per-minor URLs; it already addresses the slots directly. - -## Split of work - -1. `bincraft`: union index writer plus its guard, and a re-index of one slot to validate. -2. This repo: `edge/rpkgs-router.ts`, its test matrix, and the `cdn.tf` resources with `UNION_SLOTS` empty. -3. Enable `UNION_SLOTS` slot by slot as `bincraft` backfills them. diff --git a/specs/2026-08-12-shard-weekly-rebuild-design.md b/specs/2026-08-12-shard-weekly-rebuild-design.md deleted file mode 100644 index f4ccbaa..0000000 --- a/specs/2026-08-12-shard-weekly-rebuild-design.md +++ /dev/null @@ -1,167 +0,0 @@ -# Design: Sharding and resuming the weekly rebuild - -Date: 2026-08-12 -Status: Approved (pending spec review) - -## Problem - -`weekly-rebuild-missing` runs one job per `-` and walks that slot's rebuild list serially in a single `R -q -e` invocation (`.crow/weekly-rebuild-missing.yaml:165`). -Until 2026-08-09 that was cheap, because every source fallback was skipped as "already built" and the list was effectively empty. -Since bincraft #105/#106/#107 and build-cran-binaries #159 the gate works, and the lists are now large. - -Share of records whose object is byte-identical to CRAN's source, measured against `cran.r-project.org` MD5s on 2026-08-12: - -| slot | records | source-served | share | -| ------------------ | ------: | ------------: | -----------: | -| `amd64/resolute` | 24 212 | 15 023 | 62.1% | -| `arm64/resolute` | 24 291 | 13 670 | 56.3% | -| `arm64/alpine324` | 24 328 | 9 514 | 39.2% | -| `amd64/alpine324` | 24 343 | 8 917 | 36.7% | -| `arm64/rhel10` | 24 695 | 5 384 | 21.9% | -| `amd64/rhel10` | 24 881 | 4 712 | 19.2% | -| 12 remaining slots | ~24 700 | 850 to 2 130 | 3.5% to 8.7% | - -A single serial job cannot absorb that. -Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) started on 2026-08-09, ran for roughly two days, reached `[8692/23885] cholera`, and was killed there. - -Two distinct failures follow from that shape. - -**No parallelism.** The work is embarrassingly parallel across packages, but one job does all of it. - -**No resumability, and no clean stopping point.** The loop has no terminating condition other than exhausting the list, so the only way to stop it is a kill. -A restarted run re-reads the same list and walks it from the first entry. -It skips completed packages via `check_s3_root_package()`, but that costs a CRAN version resolution and an S3 `HEAD` per package, thousands of times, before it reaches new work. -Worse, a kill is not a pipeline failure: the `Purge CDN cache` step is guarded by `when: status: [success, failure]` (`.crow/weekly-rebuild-missing.yaml:206-207`), and on 10910 it produced no output at all. -So the ~4 600 binaries that run did publish stayed hidden behind stale edge copies. - -## Goal - -Turn each slot's rebuild into bounded, parallel, restartable units, without introducing state that can disagree with the bucket. - -## Design - -### 1. Shard the matrix three ways - -Each of the 18 `OS`/`ARCH` rows in `.crow/weekly-rebuild-missing.yaml` gains `SPLIT_INTO: 3` and `SPLIT_INDEX: 1|2|3`, giving 54 rows. -This mirrors `.crow/build-all-versions.yaml:57-98`, which already shards its matrix four ways per arch. - -Routing needs no change. -The cron filter `cron: weekly-rebuild-missing-${OS}-${ARCH}` and the manual `evaluate: weekly_rebuild_missing == "${OS}-${ARCH}"` both match all three shards of a slot. -Placement stays on the `rpkgs-${ARCH}` group label, so shards queue against available capacity rather than oversubscribing it. - -### 2. Extract the loop into `local/rebuild-missing.R` - -The build is currently a single ~1 500-character `R -q -e` argument. -Shard arithmetic and resume logic do not belong in a YAML string, and none of it is testable there. -The loop moves to `local/rebuild-missing.R`, invoked as `Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX`, mirroring `local/build-all.R`. -Its body is unchanged in substance: read `/tmp/rebuild_pkgs.txt`, subtract `local/excluded-packages.json`, loop with `tryCatch` around `bincraft::build_binary_package()`. - -The slice is **interleaved**, not contiguous: - -```r -# the list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, -# rstan*), so contiguous thirds would be badly unbalanced -mine <- pkgs[seq(split_index, length(pkgs), by = split_into)] -``` - -`local/build-all.R:64` uses contiguous chunks via `cut()`. -That is fine there because its list is every CRAN package and version, so the chunks average out. -Here the list is a filtered backlog in which expensive families sit adjacent, so interleaving is the better default. -Interleaving also makes each shard's `[i/n]` progress representative of the slot as a whole. - -### 3. Resume by re-deriving state from the bucket - -Before the loop, the shard performs one `s3fs::s3_dir_info()` on `devxy-rpkgs-binaries///latest/src/contrib` and reads the `etag` column. -It fetches CRAN's `PACKAGES` once for the latest version and published `MD5sum` of every package. -A package is still outstanding if and only if the object at `_.tar.gz` has an ETag equal to CRAN's `MD5sum` for that version, which is the definition `check_s3_root_package()` already applies one package at a time. - -```r -# one paginated listing instead of ~2900 sequential HEAD requests per shard -info <- s3fs::s3_dir_info(slot_dir) -etag <- setNames(gsub('^"|"$', "", info$etag), basename(info$uri)) - -key <- sprintf("%s_%s.tar.gz", mine, cran_version[mine]) -# keep a package when no object exists yet, or when the object is still -# byte-identical to CRAN's source; drop it once a real binary is published -mine <- mine[is.na(etag[key]) | etag[key] == cran_md5[key]] -``` - -This is the whole resume mechanism. -There is no progress file, no volume, and no database cursor. -A restarted shard recomputes ground truth and continues where it stopped, and it is correct even when a sibling shard, a `process-updates` cron, or a manual `just rebuild` completed something in the meantime. - -Three properties make this the right source of truth: - -- **It is what the build itself checks.** Any other store can disagree with the bucket; this one cannot. -- **It is agent-independent.** `.crow/weekly-rebuild-missing.yaml` mounts no `volumes:`, unlike `.crow/build-all-versions.yaml:132-133`, so `/mnt/cache` is per-job and cannot carry progress anyway. -- **It costs one listing.** `cranlike`'s `s3` fork already does exactly this call against this bucket at ~24 000 objects, so the approach is proven at the required scale. - -It must read ETags rather than the slot index's `Built` field, which is how `local/packages-to-build.R:104-130` answers the same question. -Under this design the index is not rewritten until the dependent re-index pipeline runs (section 5), so mid-run it cannot reflect the current run's progress. - -Packages that genuinely fail to build re-publish their CRAN source, so they stay outstanding and would be retried on every restart. -That is already handled upstream: `bincraft::filter_packages_with_errors()` (`R/build_binaries.R:1018`, `:1143`) drops anything with `error_occurred = TRUE`, and `store_build_metadata = TRUE` is passed on every call. -No additional poison-pill filter is needed here. - -Only the flat `src/contrib` path is considered. -The rebuild call passes no `is_r_minor_sensitive`, so it defaults to `FALSE` and only ever targets the flat path; the resume filter matches that scope deliberately. - -### 4. Give each shard a wall-clock budget - -`local/rebuild-missing.R` takes a budget, defaulting to 20 hours, and breaks out of the loop once it is exceeded: - -```r -# exit cleanly rather than being killed, so the dependent re-index still runs -if (difftime(Sys.time(), started, units = "hours") > budget_hours) { - cat(sprintf("Budget of %sh reached after %d/%d packages; stopping cleanly\n", budget_hours, i, n)) - break -} -``` - -It exits 0 and reports how much of the slice it covered. -Every run then has a terminating condition, the re-index and purge always fire, and the remainder is picked up by the next run with no bookkeeping, because section 3 recomputes the outstanding set from scratch. - -### 5. Move the re-index and purge into `.crow/weekly-rebuild-reindex.yaml` - -Three shards per slot means three concurrent `upload_package_index()` calls on the same S3 prefix. -`cranlike::update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work. -The re-index steps (`.crow/weekly-rebuild-missing.yaml:171-176`) and the purge step (`:187-207`) therefore leave that file entirely. - -The new file carries: - -```yaml -depends_on: - - weekly-rebuild-missing -runs_on: [success, failure] -``` - -`runs_on: [success, failure]` validates as a workflow-level key under `crow lint`, so a failing shard no longer withholds the re-index. -The file uses the same 18-row matrix and the same `when:` gating as `weekly-rebuild-missing`, so it only re-indexes slots that actually ran. -Each row re-indexes the flat slot and every per-minor slot. -`scripts/purge_cdn_zone.sh` runs once on a single row, because all hostnames share pull zone `3857050` and 18 identical zone purges would be waste. - -## Failure behaviour - -| case | today | after | -| -------------------------- | ----------------------------------- | ----------------------------------------------------- | -| one package errors | `tryCatch` logs, loop continues | unchanged | -| a shard fails outright | purge runs, re-index does not | re-index and purge run via `runs_on` | -| a shard exceeds its budget | cannot happen, runs until killed | exits 0, re-index and purge run | -| a shard is killed | nothing runs | still nothing; trigger the re-index pipeline alone | -| a shard restarts | re-walks the list, HEAD per package | one listing, resumes at the first outstanding package | - -The known cost of `depends_on` being file-level rather than row-level: on the weekly cron no slot is re-indexed until the slowest of all 54 jobs finishes. -The 20-hour budget bounds that at roughly one day. - -## Out of scope - -- `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row for the platform and arch, which is precisely the source-fallback set. That is a separate change. -- Bunny Perma-Cache eviction. `scripts/purge_cdn_zone.sh` purges the regular edge cache only; see the note in `CLAUDE.md` and issue history. -- The audit that produces the rebuild list is unchanged. - -## Verification - -- `crow lint .crow/` passes for both pipeline files. -- `local/rebuild-missing.R` gets unit coverage in `local/tests/` for the two pure pieces: the interleaved slice (disjoint, covering, deterministic) and the outstanding-set filter (source-served ETag kept, binary ETag dropped, absent object kept). -- A single-slot manual run of `alpine-324-amd64` shard 1 confirms the listing shortcut against the live bucket, and that the reported outstanding count is close to the 8 917 measured above divided by three. -- Restarting that shard mid-run confirms it resumes rather than replaying, by comparing the outstanding count it reports on the second start.