diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 71e263e..bd127d6 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -28,6 +28,10 @@ matrix: ARCH: amd64 - CODENAME: redhat-10 ARCH: arm64 + - CODENAME: alpine322 + ARCH: amd64 + - CODENAME: alpine322 + ARCH: arm64 - CODENAME: alpine323 ARCH: amd64 - CODENAME: alpine323 diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index 5d6e6fe..7c33c5b 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -58,7 +58,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT backend_options: kubernetes: diff --git a/.crow/backfill-abi-classification.yaml b/.crow/backfill-abi-classification.yaml deleted file mode 100644 index aeab591..0000000 --- a/.crow/backfill-abi-classification.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# Populate the ABI classifier cache for packages that predate the per-minor scheme. -# -# bincraft decides whether a flat binary may be carried into a per-minor index -# from its risky-package set. That set is now the union of the per-minor tarball -# scan and the `abi_classification` cache, but the cache only holds verdicts for -# packages that passed through the build pipeline after the scheme existed. -# Everything older has no verdict, and no verdict means "carry it" - which is how -# a base64enc built under R 4.4.3 reached R 4.6 clients and died at dyn.load() -# on SETLENGTH. -# -# The verdict is computed from the package source, so it does not depend on OS, -# arch or R minor: one run populates the cache for every slot. The arch/codename -# below only choose which flat index to enumerate. -# -# Re-running is cheap. Cached verdicts are returned without touching the network, -# so a second run costs one query per package and downloads only what is still -# unclassified. -# -# crow pipeline create devxy/build-cran-binaries --var backfill_abi=run - -variables: - # A manual pipeline creation instantiates every file in .crow/, so the default - # must be a value the `when` clause rejects. - backfill_abi: - description: "Set to 'run' to classify the flat index into the ABI cache." - options: - - none - - run - default: none - BACKFILL_ARCH: - description: 'Arch of the flat index to enumerate.' - options: - - amd64 - - arm64 - default: amd64 - BACKFILL_CODENAME: - description: 'Codename of the flat index to enumerate.' - default: resolute - BACKFILL_LIMIT: - description: 'Classify at most this many packages (0 = no limit).' - default: '0' - -when: - - event: manual - evaluate: 'backfill_abi == "run"' - -skip_clone: true - -labels: - group: rpkgs-amd64 - -steps: - - name: 'Classify the flat index into the ABI cache' - image: reg.devxy.io/rpkgs/build-env-ubuntu:resolute - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - # Pipeline variables are substituted into the YAML, not exported, so they - # have to be bound here to be readable as env vars in the script below. - BACKFILL_ARCH: ${BACKFILL_ARCH} - BACKFILL_CODENAME: ${BACKFILL_CODENAME} - BACKFILL_LIMIT: ${BACKFILL_LIMIT} - R_LIBS_USER: /tmp/R-pkgs - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /tmp/R-pkgs - - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R - - | - set -eu - LIMIT="" - if [ "$$BACKFILL_LIMIT" != "0" ]; then - LIMIT="$$BACKFILL_LIMIT" - fi - /opt/R/4.5.3/bin/Rscript local/backfill-abi-classification.R \ - "$$BACKFILL_ARCH" "$$BACKFILL_CODENAME" $$LIMIT - backend_options: - docker: - resources: - requests: - memory: 4Gi - cpu: 1000m diff --git a/.crow/backfill-flat-safety.yaml b/.crow/backfill-flat-safety.yaml deleted file mode 100644 index 0db0735..0000000 --- a/.crow/backfill-flat-safety.yaml +++ /dev/null @@ -1,176 +0,0 @@ -# Record, per flat-slot object, whether it loads under every supported R minor. -# -# `union_index_records()` keeps a carried flat record when that object is -# recorded safe and drops it otherwise. Objects uploaded before that record -# existed have no verdict, so they are dropped: a full re-index removed roughly -# 2000 records per per-minor index (16398 across the 4.4 indexes, 20076 across -# 4.5, 15532 across 4.6), each one resolving to CRAN source for clients the -# routing serves correctly. -# -# This inspects what is already in the bucket and writes the verdict, which is -# what restores that coverage. Decided by symbols rather than heuristics: a -# binary fails at dyn.load() under minor M exactly when it references a symbol M -# does not export, and the volatile set is derived by diffing the R builds in -# the image. -# -# Re-running is cheap; recorded objects are skipped. -# -# crow pipeline create devxy/build-cran-binaries --var flat_safety=all - -variables: - # A manual pipeline creation instantiates every file in .crow/, so the default - # must match no matrix row. - flat_safety: - description: "Inspect target: a specific -, 'all' for every slot, or 'none'." - options: - - none - - all - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - FLAT_SAFETY_LIMIT: - description: 'Inspect at most this many objects per slot (0 = no limit).' - default: '0' - -when: - - event: manual - evaluate: 'flat_safety == "all" || flat_safety == "${OS}-${ARCH}"' - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: alpine-323 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-323 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: redhat-8 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-8 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-9 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-9 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-10 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: redhat-10 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: ubuntu-2204 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2204 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2404 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2404 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - -steps: - - name: 'Inspect flat objects and record safety' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - # uvr resolves bincraft's dependencies through the GitHub API, including - # the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the - # failure is not loud: uvr warns, falls back to the registry copy of - # cranlike, and the index write then dies on `unused argument (built = - # built)` because upstream has no such parameter. Intermittent by nature, - # since it depends on the rate-limit window. - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_LIBS_USER: /tmp/R-pkgs - FLAT_SAFETY_ARCH: ${ARCH} - FLAT_SAFETY_LIMIT: ${FLAT_SAFETY_LIMIT} - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /tmp/R-pkgs - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - | - set -eu - LIMIT="" - if [ "$$FLAT_SAFETY_LIMIT" != "0" ]; then - LIMIT="$$FLAT_SAFETY_LIMIT" - fi - # The codename comes from the image's /etc/os-release, exactly as the - # re-index step derives it, so it cannot drift from the matrix. - CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))') - echo "slot: $$FLAT_SAFETY_ARCH/$$CODENAME" - /opt/R/$R_VERSION/bin/Rscript local/backfill-flat-safety.R \ - "$$FLAT_SAFETY_ARCH" "$$CODENAME" $$LIMIT - backend_options: - docker: - resources: - requests: - memory: 4Gi - cpu: 1000m diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index ce3bc09..fb1229d 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -24,6 +24,7 @@ variables: OS_VERSION: description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' options: + - '3.22' - '3.23' - '3.24' - '8' @@ -34,14 +35,6 @@ variables: - 'resolute' default: '3.24' R_VERSION: - # The slot's *primary* R minor: what `local/build-all.R` builds into the - # generic slot. The loop below already runs `--sensitive-only` for every - # other installed minor, so filling a non-primary minor's gap needs this - # left alone, not changed. - # - # 4.6.0 was briefly offered here (#183) and removed: selecting it for a - # slot whose generic binaries are 4.5-built would publish 4.6 binaries - # into the generic slot and break every 4.5 client. description: 'Primary R version under /opt/R.' options: - 4.5.3 @@ -149,11 +142,7 @@ steps: # install). Pin bincraft here, exactly like the R-minor pass below. - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - # The dependency-prep job may run on another agent whose cache volume is - # not visible here. Install the packages needed to recompute the snapshot - # and query build metadata in every build job. - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) @@ -165,10 +154,7 @@ steps: LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true - R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true - # Also build current packages whose required dependency tree needs a newer R. - R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION" $SPLIT_INTO $SPLIT_INDEX done # archive missed packages; first arg is the codename (e.g. "alpine324"), # derived via bincraft like the upload step, not paste(OS, OS_VERSION). diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 531a47f..7600876 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -3,18 +3,20 @@ # Routing is preserved 1:1: # - cron: each existing `process-cran-updates--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `process_cran_updates` dropdown; -# "all" fans out every os/arch as parallel matrix workflows. +# - manual: pick a target from the `process_cran_updates` dropdown +# ("all" = every os/arch). # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. process_cran_updates: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." options: - none - all + - alpine-322-amd64 + - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -46,6 +48,18 @@ labels: matrix: include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + OS_ID: alpine322 + PROCESS_NEW: 'FALSE' + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + OS_ID: alpine322 + PROCESS_NEW: 'FALSE' - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 @@ -189,10 +203,9 @@ steps: - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - | @@ -205,10 +218,7 @@ steps: LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true - R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true - # Also build current packages whose required dependency tree needs a newer R. - R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION" done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/prune-stale-flat-binaries.yaml b/.crow/prune-stale-flat-binaries.yaml deleted file mode 100644 index 79ca003..0000000 --- a/.crow/prune-stale-flat-binaries.yaml +++ /dev/null @@ -1,242 +0,0 @@ -# Remove flat-slot binaries recorded unable to load under some R minor, then -# re-index. -# -# The list comes from `abi_flat_safety`, so it names the objects the binaries -# themselves report as broken, not the ones a source heuristic suspects. That -# distinction is the whole cost of the operation: the heuristic calls 93% of -# compiled packages risky and would delete roughly 3000 objects per slot to fix -# about 230. -# -# `upload_single_binary()` has written a non-portable package only to the -# per-minor slot since v5.3.0, so this is a one-time cleanup of what was -# uploaded before that. The flat slot is what every client that cannot be -# routed by R minor receives, and such a binary was built under whichever R was -# primary at the time, so it dies at load under any minor that dropped a symbol -# it uses: -# -# base64enc.so: undefined symbol: SETLENGTH -# -# Deleting the flat object does not create a hole. The source fallback -# republishes the CRAN source under the same key, so the package stays -# installable everywhere and compiles against the client's own R, while routed -# clients keep getting the per-minor binary. -# -# Dry run unless PRUNE_APPLY is 'yes'. Start with an explicit package list: -# -# crow pipeline create devxy/build-cran-binaries \ -# --var prune_flat=ubuntu-2604-amd64 \ -# --var PRUNE_PACKAGES="base64enc lobstr" \ -# --var PRUNE_APPLY=yes - -variables: - # A manual pipeline creation instantiates every file in .crow/, so the default - # must match no matrix row. - prune_flat: - description: "Prune target: a specific -, 'all' for every slot, or 'none'." - options: - - none - - all - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - PRUNE_PACKAGES: - description: 'Space-separated package names. Empty means every package the classifier cache calls risky.' - default: '' - PRUNE_APPLY: - description: "Set to 'yes' to actually delete. Anything else is a dry run." - options: - - 'no' - - 'yes' - default: 'no' - -when: - - event: manual - evaluate: 'prune_flat == "all" || prune_flat == "${OS}-${ARCH}"' - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: alpine-323 - ARCH: amd64 - CODENAME: alpine323 - IMG: alpine:3.23 - - OS: alpine-323 - ARCH: arm64 - CODENAME: alpine323 - IMG: alpine:3.23 - - OS: alpine-324 - ARCH: amd64 - CODENAME: alpine324 - IMG: alpine:3.24 - - OS: alpine-324 - ARCH: arm64 - CODENAME: alpine324 - IMG: alpine:3.24 - - OS: redhat-8 - ARCH: amd64 - CODENAME: rhel8 - IMG: redhat:8 - - OS: redhat-8 - ARCH: arm64 - CODENAME: rhel8 - IMG: redhat:8 - - OS: redhat-9 - ARCH: amd64 - CODENAME: rhel9 - IMG: redhat:9 - - OS: redhat-9 - ARCH: arm64 - CODENAME: rhel9 - IMG: redhat:9 - - OS: redhat-10 - ARCH: amd64 - CODENAME: rhel10 - IMG: redhat:10 - - OS: redhat-10 - ARCH: arm64 - CODENAME: rhel10 - IMG: redhat:10 - - OS: ubuntu-2204 - ARCH: amd64 - CODENAME: jammy - IMG: ubuntu:jammy - - OS: ubuntu-2204 - ARCH: arm64 - CODENAME: jammy - IMG: ubuntu:jammy - - OS: ubuntu-2404 - ARCH: amd64 - CODENAME: noble - IMG: ubuntu:noble - - OS: ubuntu-2404 - ARCH: arm64 - CODENAME: noble - IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - CODENAME: resolute - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - CODENAME: resolute - IMG: ubuntu:resolute - -steps: - - name: 'Prune stale flat binaries' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GIT_USER: pat-s - R_LIBS_USER: /tmp/R-pkgs - # Pipeline variables are substituted into the YAML, not exported, so they - # have to be bound here to be readable as env vars in the script below. - PRUNE_ARCH: ${ARCH} - PRUNE_CODENAME: ${CODENAME} - PRUNE_PACKAGES: ${PRUNE_PACKAGES} - PRUNE_APPLY: ${PRUNE_APPLY} - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /tmp/R-pkgs - - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R - - | - set -eu - APPLY="" - if [ "$$PRUNE_APPLY" = "yes" ]; then - APPLY="--apply" - fi - /opt/R/4.5.3/bin/Rscript local/prune-stale-flat-binaries.R \ - "$$PRUNE_ARCH" "$$PRUNE_CODENAME" $$APPLY $$PRUNE_PACKAGES - backend_options: - docker: - resources: - requests: - memory: 3Gi - cpu: 1000m - - # Deleting objects without re-indexing is worse than leaving them: the - # published index keeps naming files that no longer exist, so every client - # gets a 404 instead of a working source fallback. The re-index is what - # republishes them as CRAN source, so it belongs in the same run. - - name: 'Re-index the slot' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - R_LIBS_USER: /tmp/R-pkgs - PRUNE_APPLY: ${PRUNE_APPLY} - commands: - - | - set -eu - if [ "$$PRUNE_APPLY" != "yes" ]; then - echo "Dry run: nothing was deleted, so there is nothing to re-index." - exit 0 - fi - /opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - backend_options: - docker: - resources: - requests: - memory: 3Gi - cpu: 1000m - - # The edge caches indexes for ~370 days, so without this the deletion is - # invisible to clients and they keep being handed the removed binary. - - name: 'Purge CDN cache' - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - # Same zones as reindex.yaml: the production pull zone id, the alliance - # hostname and the staging zone, which shares the B2 origin. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' - PRUNE_APPLY: ${PRUNE_APPLY} - commands: - - | - set -eu - if [ "$$PRUNE_APPLY" != "yes" ]; then - echo "Dry run: nothing changed at the origin, so no purge is needed." - exit 0 - fi - apk add --no-cache -q bash curl jq - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES diff --git a/.crow/rebuild-abi-gap.yaml b/.crow/rebuild-abi-gap.yaml deleted file mode 100644 index 71ade22..0000000 --- a/.crow/rebuild-abi-gap.yaml +++ /dev/null @@ -1,214 +0,0 @@ -# Rebuild specific packages into a per-R-minor slot. -# -# The per-minor scheme only ever covered packages that passed through the build -# pipeline after it existed. A package built into the flat slot before that, and -# never updated on CRAN since, is never reclassified and never gets a per-minor -# copy - so the union index serves its flat binary to every minor. When that -# binary references a symbol the client's R no longer exports, the install fails -# at dyn.load(): -# -# base64enc.so: undefined symbol: SETLENGTH -# -# base64enc calls SETLENGTH, its flat binary was built under R 4.4.3, and R 4.6 -# does not export SETLENGTH. There was no way to rebuild one package into one -# minor from CI, only `just rebuild` from a workstation with the S3 and Postgres -# credentials to hand. This adds that path. -# -# `--sensitive-only` is deliberately NOT passed. That flag gates on the source -# classifier, which misses Rcpp: the repair therefore skipped Rcpp entirely, -# leaving its 4.5-built flat binary as the only copy and breaking every R 4.4 -# client on the slot. -# -# Since bincraft v5.3.0 the destination is decided by the built binary's symbols -# in `upload_single_binary()`, so building unconditionally is safe: a package -# that needs a per-minor slot lands there, and one that does not lands in the -# flat slot. Skipping before the build is what removed that decision. -# -# Trigger with the slot and an explicit package list: -# -# crow pipeline create devxy/build-cran-binaries \ -# --var rebuild_abi_gap=ubuntu-2604-amd64 \ -# --var ABI_PACKAGES="base64enc@0.1-6 lobstr@1.2.1" \ -# --var ABI_R_MINOR=4.6 - -variables: - # A manual pipeline creation instantiates every file in .crow/, so the - # default must match no matrix row. - rebuild_abi_gap: - description: "Rebuild target: a specific -, 'all' for every slot, or 'none'." - options: - - none - - all - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - ABI_PACKAGES: - description: 'Space-separated package@version list, e.g. "base64enc@0.1-6 lobstr@1.2.1".' - default: '' - ABI_R_MINOR: - description: 'The R minor to build into.' - options: - - '4.4' - - '4.5' - - '4.6' - default: '4.6' - -when: - - event: manual - evaluate: 'rebuild_abi_gap == "all" || rebuild_abi_gap == "${OS}-${ARCH}"' - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: alpine-323 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-323 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: redhat-8 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-8 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-9 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-9 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-10 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: redhat-10 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: ubuntu-2204 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2204 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2404 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2404 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - -steps: - - name: 'Rebuild packages into the per-minor slot' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - # Pipeline variables are substituted into the YAML, not exported, so they - # have to be bound here to be readable as env vars in the script below. - ABI_PACKAGES: ${ABI_PACKAGES} - ABI_R_MINOR: ${ABI_R_MINOR} - UVR_CACHE_DIR: /tmp/uvr/cache - UVR_PACKAGES_DIR: /tmp/uvr/packages - CCACHE_DIR: /tmp/ccache - NCPUS: 2 - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p $UVR_CACHE_DIR $UVR_PACKAGES_DIR $CCACHE_DIR - - | - set -eu - if [ -z "$$ABI_PACKAGES" ]; then - echo "ABI_PACKAGES is empty; nothing to rebuild." >&2 - exit 1 - fi - RBIN=$(ls -d /opt/R/$${ABI_R_MINOR}*/bin 2>/dev/null | head -1) - if [ -z "$$RBIN" ]; then - echo "R $$ABI_R_MINOR is not installed in this image." >&2 - exit 1 - fi - # Container-local library: this workflow mounts no shared cache volume, - # so it cannot disturb the build pipeline's per-agent caches. - LIB="/tmp/R-pkgs-$$ABI_R_MINOR" - mkdir -p "$$LIB" - export R_LIBS_USER="$$LIB" - rm -rf "$$LIB"/00LOCK-* - "$$RBIN/Rscript" local/install-bincraft.R - rc=0 - for spec in $$ABI_PACKAGES; do - pkg=$${spec%@*} - ver=$${spec#*@} - if [ "$$pkg" = "$$ver" ]; then - echo "malformed spec '$$spec'; expected package@version" >&2 - rc=1 - continue - fi - echo "=== rebuilding $$pkg $$ver into $$ABI_R_MINOR ===" - "$$RBIN/Rscript" local/build-one.R "$$pkg" "$$ver" || rc=1 - done - exit $$rc - backend_options: - docker: - resources: - requests: - memory: 5Gi - cpu: 1000m diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml deleted file mode 100644 index 803d9a5..0000000 --- a/.crow/reindex.yaml +++ /dev/null @@ -1,198 +0,0 @@ -# Re-index every slot without rebuilding anything. -# -# `weekly-rebuild-reindex` exists to run after `weekly-rebuild-missing`, so it -# depends on that workflow and shares its gate: triggering it manually also -# starts hours of package rebuilds. That is the wrong tool when only the index -# needs regenerating - after a bincraft release that changes how the index is -# written, for instance, where the objects in the bucket are already correct -# and only `PACKAGES*` is stale. -# -# This workflow does the index half on its own. It installs the latest bincraft -# release, republishes the generic and per-R-minor indexes for each slot, and -# purges the edge. No package is built. -# -# Trigger with the `reindex` variable set to `all` or to a single -# `-`, e.g. -# -# crow pipeline create devxy/build-cran-binaries --var reindex=all - -variables: - # A manual pipeline creation instantiates every file in .crow/, so the - # default must match no matrix row. - reindex: - description: "Re-index target: a specific -, 'all' for every slot, or 'none'." - options: - - none - - all - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - -when: - - event: manual - evaluate: 'reindex == "all" || reindex == "${OS}-${ARCH}"' - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: alpine-323 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-323 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: redhat-8 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-8 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-9 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-9 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-10 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: redhat-10 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: ubuntu-2204 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2204 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2404 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2404 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - -steps: - - name: 'Re-index the slot' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - # The index build reads the ABI classifier cache to decide which packages - # may be carried from the flat slot into a per-minor one. Without the - # password that query fails, and it fails soft, so the run still succeeds - # while quietly using an empty risky set: exactly the state the cache was - # added to fix. It logs one connection error per slot. - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - # uvr resolves bincraft's dependencies through the GitHub API, including - # the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the - # failure is not loud: uvr warns, falls back to the registry copy of - # cranlike, and the index write then dies on `unused argument (built = - # built)` because upstream has no such parameter. Intermittent by nature, - # since it depends on the rate-limit window. - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: ${R_VERSION} - PLATFORM: ${OS} - ARCH: ${ARCH} - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - # The codename is detected from the image's /etc/os-release. - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate - # Bunny pull zones, so both must be purged after the shared origin changes. - # The staging zone is listed too. It shares the B2 origin, so an index - # it still holds is a stale copy of the same object, and its - # cache_expiration_time is the same ~370 days: without a purge here it - # serves pre-reindex indexes indefinitely and any verification run - # against it measures the past. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' - commands: - - apk add --no-cache -q bash curl jq - # Crow carries the checkout from the re-index step into this step. - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES - # Runs on every row rather than on one designated slot: a cron fires only - # its own slot's row, so gating on a named slot would leave every other - # slot unpurged. A manual "all" run therefore purges the zone 18 times, - # which is a cheap API call and rare. - # - # Run it even when the re-index above failed: the objects were still - # replaced, and a stale edge is exactly what keeps them hidden. - when: - - status: [success, failure] diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml index cc09586..e56e1e9 100644 --- a/.crow/repair-built-stamp.yaml +++ b/.crow/repair-built-stamp.yaml @@ -16,7 +16,7 @@ # pipeline here gates on a variable named after itself for exactly that reason. # # crow pipeline create --branch main \ -# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.23 \ +# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \ # --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries variables: # Gates this pipeline. A manual pipeline creation instantiates every file in @@ -37,8 +37,9 @@ variables: - ubuntu default: alpine OS_VERSION: - description: 'OS image tag. Must match OS (alpine: 3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).' + description: 'OS image tag. Must match OS (alpine: 3.22/3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).' options: + - '3.22' - '3.23' - '3.24' - '8' @@ -47,7 +48,7 @@ variables: - 'jammy' - 'noble' - 'resolute' - default: '3.23' + default: '3.22' R_VERSION: description: 'R version whose stamp the slot should advertise.' options: diff --git a/.crow/repair-unsafe-flat-objects.yaml b/.crow/repair-unsafe-flat-objects.yaml deleted file mode 100644 index e0625c0..0000000 --- a/.crow/repair-unsafe-flat-objects.yaml +++ /dev/null @@ -1,173 +0,0 @@ -# Rebuild every flat object recorded as unable to load under some R minor. -# -# The list comes from `abi_flat_safety`, written by the flat-safety backfill -# from the binaries themselves, so it names exactly the objects that fail at -# dyn.load() somewhere rather than everything a source heuristic suspects. The -# heuristic calls 93% of compiled packages risky; the binaries say 4.3%. -# -# Each object is rebuilt under every R minor installed in the image, with no -# `--sensitive-only` gate. Since bincraft v5.3.0 the destination is decided by -# the built binary's symbols in `upload_single_binary()`, so a package needing a -# per-minor slot lands there and a portable one lands in the flat slot. Gating -# on the source classifier is what let Rcpp slip through and break every R 4.4 -# client on the slot. -# -# crow pipeline create devxy/build-cran-binaries --var repair_unsafe=all - -variables: - # A manual pipeline creation instantiates every file in .crow/, so the default - # must match no matrix row. - repair_unsafe: - description: "Repair target: a specific -, 'all' for every slot, or 'none'." - options: - - none - - all - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - -when: - - event: manual - evaluate: 'repair_unsafe == "all" || repair_unsafe == "${OS}-${ARCH}"' - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: alpine-323 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-323 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: redhat-8 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-8 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-9 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-9 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-10 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: redhat-10 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: ubuntu-2204 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2204 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2404 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2404 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - -steps: - - name: 'Rebuild objects recorded unsafe' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_LIBS_USER: /tmp/R-pkgs - NCPUS: 2 - REPAIR_ARCH: ${ARCH} - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /tmp/R-pkgs - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - | - set -eu - CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))') - /opt/R/$R_VERSION/bin/Rscript local/list-unsafe-flat-objects.R \ - "$$REPAIR_ARCH" "$$CODENAME" > /tmp/unsafe.txt - echo "objects to rebuild: $(wc -l < /tmp/unsafe.txt)" - rc=0 - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$$RBIN")")") - RMINOR=$(echo "$$RV" | cut -d. -f1-2) - echo "=== rebuilding under R $$RV ===" - LIB="/tmp/R-pkgs-$$RMINOR" - mkdir -p "$$LIB" - R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" local/install-bincraft.R || true - while read -r pkg ver; do - [ -z "$$pkg" ] && continue - R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" \ - local/build-one.R "$$pkg" "$$ver" || rc=1 - done < /tmp/unsafe.txt - done - exit $$rc - backend_options: - docker: - resources: - requests: - memory: 5Gi - cpu: 1000m diff --git a/.crow/retire-slot.yaml b/.crow/retire-slot.yaml deleted file mode 100644 index 1b62ba7..0000000 --- a/.crow/retire-slot.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# Delete a retired slot's objects from the bucket. -# -# The config half is a commit, so it runs on a workstation: -# just retire-slot --apply -# -# This half needs the B2 credentials, which only exist in CI. -# `local/retire-slot.R` refuses any slot still listed in `local.rpkgs_slots`, -# so this cannot delete a live slot. Both architectures go together. -# -# Dry run unless RETIRE_APPLY is 'yes': -# -# crow pipeline create devxy/build-cran-binaries --branch main \ -# --var retire_slot=alpine322 --var RETIRE_APPLY=yes - -variables: - # A manual pipeline creation instantiates every file in .crow/, so the - # default must be a value the script never accepts. - retire_slot: - description: "Codename of the slot to retire (e.g. alpine322), or 'none'." - default: none - RETIRE_APPLY: - description: "Set to 'yes' to delete. Anything else reports and changes nothing." - options: - - 'no' - - 'yes' - default: 'no' - -when: - - event: manual - evaluate: 'retire_slot != "none"' - -skip_clone: true - -labels: - group: rpkgs-amd64 - -steps: - - name: 'Delete the retired slot from the bucket' - image: reg.devxy.io/rpkgs/build-env-alpine:3.24 - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - R_LIBS_USER: /tmp/R-pkgs - # Pipeline variables are substituted into the YAML, not exported, so they - # have to be bound here to be readable as env vars in the script below. - RETIRE_SLOT: ${retire_slot} - RETIRE_APPLY: ${RETIRE_APPLY} - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /tmp/R-pkgs - - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R - - | - set -eu - APPLY="" - if [ "$$RETIRE_APPLY" = "yes" ]; then - APPLY="--apply" - fi - /opt/R/4.5.3/bin/Rscript local/retire-slot.R "$$RETIRE_SLOT" --assets-only $$APPLY - backend_options: - docker: - resources: - requests: - memory: 2Gi - cpu: 1000m diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml index 7826d3e..2195423 100644 --- a/.crow/trial-build-registry.yaml +++ b/.crow/trial-build-registry.yaml @@ -38,6 +38,14 @@ labels: matrix: include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 diff --git a/.crow/verify-repo-routing.yaml b/.crow/verify-repo-routing.yaml deleted file mode 100644 index b20c993..0000000 --- a/.crow/verify-repo-routing.yaml +++ /dev/null @@ -1,146 +0,0 @@ -# Install a fixed set of packages from a published slot under every R minor the -# image carries, through each URL form the website tells people to configure, -# and check that each one loads. -# -# This is the end-to-end counterpart to the ABI bookkeeping: `abi_flat_safety` -# records what the binaries say about themselves, this proves what a client -# actually receives once the CDN, the per-minor indexes and the `Path:` field -# have had their say. It runs on the build agents rather than a workstation so -# arm64 is covered on real hardware. -# -# crow pipeline create devxy/build-cran-binaries --var verify_routing=all - -variables: - # A manual pipeline creation instantiates every file in .crow/, so the default - # must match no matrix row. - verify_routing: - description: "Verify target: a specific -, 'all' for every slot, or 'none'." - options: - - none - - all - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - -when: - - event: manual - evaluate: 'verify_routing == "all" || verify_routing == "${OS}-${ARCH}"' - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: alpine-323 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-323 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: redhat-8 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-8 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-9 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-9 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-10 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: redhat-10 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: ubuntu-2204 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2204 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2404 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2404 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - -steps: - - name: 'Verify routing serves loadable binaries' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - VERIFY_ARCH: ${ARCH} - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - | - set -eu - CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))') - rc=0 - for RBIN in /opt/R/[0-9]*/bin/R; do - RV=$(basename "$(dirname "$(dirname "$$RBIN")")") - for FORM in arch generic flat; do - echo "=== R $$RV via $$FORM ===" - "$(dirname "$$RBIN")/Rscript" local/verify-repo-routing.R \ - "$$VERIFY_ARCH" "$$CODENAME" "$$FORM" || rc=1 - done - done - exit $$rc - backend_options: - docker: - resources: - requests: - memory: 4Gi - cpu: 1000m diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 2796a47..3a9d98f 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -3,18 +3,20 @@ # Routing is preserved 1:1: # - cron: each existing `weekly-audit-missing--` cron fires only # its matching matrix row (via the per-row `cron:` name filter). -# - manual: pick a target from the `weekly_audit_missing` dropdown; -# "all" fans out every os/arch as parallel matrix workflows. +# - manual: pick a target from the `weekly_audit_missing` dropdown +# ("all" = every os/arch). # Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64). variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. weekly_audit_missing: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." options: - none - all + - alpine-322-amd64 + - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -46,6 +48,14 @@ labels: matrix: include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 @@ -137,7 +147,7 @@ steps: - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' backend_options: docker: diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 7ea6c97..712e87d 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -53,7 +53,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index a93a7bd..a12676d 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -4,11 +4,17 @@ # - cron: each existing `weekly-rebuild-missing--` cron fires only # its matching matrix rows (via the per-row `cron:` name filter), # which is now all three shards of that slot. -# - manual: pick a target from the `weekly_rebuild_missing` dropdown; -# "all" fans out every os/arch and shard as parallel matrix -# workflows, while a single - runs its three shards. +# - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the +# previous bare manual trigger that ran every os/arch); pick a +# single - to run just one. # Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). # +# 9 OS versions x 2 arches x 3 shards = 54 rows. Crow counts the *declared* +# matrix against CROW_MAX_MATRIX_SIZE before any `when:` gate is applied, so a +# single-slot manual run expands all 54 too. The server default is 50 and was +# raised for this; `crow lint` does not check the limit, so adding an OS +# version here is only caught when a pipeline is triggered. +# # The shard picks up its own slice and re-derives what is still outstanding # from the bucket, so a restart resumes rather than replaying; see # local/rebuild-missing.R. @@ -21,10 +27,12 @@ variables: # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." options: - none - all + - alpine-322-amd64 + - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -56,6 +64,42 @@ labels: matrix: include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 @@ -388,7 +432,7 @@ steps: - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 698f503..224596b 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -16,10 +16,12 @@ variables: # exactly the slots it rebuilt. A manual pipeline creation instantiates every # file in .crow/, so the default must match no matrix row. weekly_rebuild_missing: - description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + description: "Manual run target: a specific -, 'all' for every os/arch, or 'none' to run nothing." options: - none - all + - alpine-322-amd64 + - alpine-322-arm64 - alpine-323-amd64 - alpine-323-arm64 - alpine-324-amd64 @@ -56,6 +58,14 @@ labels: matrix: include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 @@ -135,23 +145,8 @@ steps: from_secret: B2_S3_ACCESS_KEY B2_S3_SECRET_KEY: from_secret: B2_S3_SECRET_KEY - # The index build reads the ABI classifier cache to decide which packages - # may be carried from the flat slot into a per-minor one. Without the - # password that query fails, and it fails soft, so the run still succeeds - # while quietly using an empty risky set: exactly the state the cache was - # added to fix. It logs one connection error per slot. - PGPASS: - from_secret: PGPASS REPO_RO_TOKEN: from_secret: REPO_RO_TOKEN - # uvr resolves bincraft's dependencies through the GitHub API, including - # the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the - # failure is not loud: uvr warns, falls back to the registry copy of - # cranlike, and the index write then dies on `unused argument (built = - # built)` because upstream has no such parameter. Intermittent by nature, - # since it depends on the rate-limit window. - GITHUB_PAT: - from_secret: GITHUB_PAT GIT_USER: pat-s R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} @@ -178,18 +173,15 @@ steps: OTEL_R_METRICS_EXPORTER: none BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY - # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate - # Bunny pull zones, so both must be purged after the shared origin changes. - # The staging zone is listed too. It shares the B2 origin, so an index - # it still holds is a stale copy of the same object, and its - # cache_expiration_time is the same ~370 days: without a purge here it - # serves pre-reindex indexes indefinitely and any verification run - # against it measures the past. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + # All hostnames on the zone share this id, so one purge covers + # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. + BUNNY_PULLZONE: '3857050' commands: - - apk add --no-cache -q bash curl jq - # Crow carries the checkout from the re-index step into this step. - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES + - apk add --no-cache -q bash curl git + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" # Runs on every row rather than on one designated slot: a cron fires only # its own slot's row, so gating on a named slot would leave every other # slot unpurged. A manual "all" run therefore purges the zone 18 times, diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e0ba9a9..82461cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,19 +24,19 @@ repos: args: - --markdown-linebreak-ext=md - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.23.3 + rev: v0.23.2 hooks: - id: markdownlint-cli2 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.10 + rev: v3.9.6 hooks: - id: prettier - repo: https://github.com/posit-dev/air-pre-commit - rev: 0.12.0 + rev: 0.11.0 hooks: - id: air-format - repo: https://github.com/editorconfig-checker/editorconfig-checker - rev: v4.0.2 + rev: v3.11.1 hooks: - id: editorconfig-checker exclude: ^local/patches/.*\.patch$ diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 4b118a8..aca8ce1 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,79 +2,79 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/http" { - version = "3.6.2" + version = "3.6.0" hashes = [ - "h1:7+Ewr07oLewLfoe5K7Fhy4T5JUU4y7I7iRGTG2hrzy4=", - "h1:8AvJ6H30J1tgxwEoVKxe0B0Ac9XFn1gwOzidrIc+F5o=", - "h1:AllxtaK2m88aY+QitHui6kik/qJmBakrH5o5W7vOKI0=", - "h1:OimQGm1/9sIkdDCuUb8D5sxDorhektHeCQ337DHDfGk=", - "h1:T5zxkFXQ9To+ZysjCZu6gFqE8G1aTGh58Ol20bCk1Fc=", - "h1:TZlCCHsmKXSI3FyzmSgCU+TarKOHgiOft0wiYh85Y+E=", - "h1:VgyzPdGw1U/KU8gKhefKlIhy5kQYdpoZkpNcGiTgvVs=", - "h1:aF+Ztfy9m0Ll6nFOtC90NxN6WIEKAmfmVFhXAtFF8CI=", - "h1:bgViIoXF7kMXA7xgNikFYfv1D15rregO8I8Sq95sLGQ=", - "h1:d7tCAc1wr1seWpGMBx7tpRyGKOCicvozaHvn5X8ZFfM=", - "h1:dRCrpHH2UrkKs/lCqewyA00L0VfOCETVM14NBhyxHaM=", - "h1:eIqaNIixQTlhmr2GU1fAFH+eVYLQIfeGw7Ec+JT/bEo=", - "h1:hrK+Jzpgivi4Jo356K3LYDoeWBV1bMqKPBEqrEHwgpA=", - "h1:mrpIAC+TK3rQQz2jllAF8o2WLMKM14/WoclLgYKc4WA=", - "h1:zwE/uNBapBDpkKa5I24moawVPTanbG+eKumHRm/qINw=", - "zh:0cf6173a426f65a85e8baf4f9a7808dc94d0e02a82b34bc903555fda1eb45d7a", - "zh:2b5fa2b0487e692014fa86745a2a98d43f34f7aaea044acd3e2cee139627c9f1", - "zh:2c9c18893ce0f34e077559632c88237d172ac6646853bf00c84a5a4ec3b6c241", - "zh:316c848e892764aa10a890cf2455f52b592c6051a92ac8451955ca1d47ca1d2f", - "zh:4300e32ecaa967ca0ed56ca48f8dcd10796e70203bf6b8311e71dc43cd0d2aa1", - "zh:4a0a555d85a31bd257399253b502ad03816ccdba7b8fbbbd1e07a313d04d83fb", - "zh:6278773c31c1605c6d780fc982eacb83944e2807c8d8afe11534253b225c5aa3", - "zh:81b73da8dbcedf54604f131e0acd7ebd36cf673276174b4b4f10af0b0f10de11", - "zh:887beaa6ef6521d3f5e2b1193d458575fb91855f36bc66dd26a10988127a39fb", - "zh:91b2260424c5f8e5968d9e6d5bdbbdb60d61ccc4ff001aa0d637427629349c54", - "zh:a5d9d7032d00a6dd844e8d0d374b5ae89e25f9a6635a90e01c5f65e9dd14a93e", - "zh:bcf8b2aca448774e4701f221b74d762847be55dc312022b4aad66da3b389bf5d", - "zh:d62a3ad37ee1fb943ad0590978f4e7ee0ac064d19554c1c05ec98e9d2bb658f7", - "zh:e517b8b81fe88bf49868923f076515b9157771d1d3a80849cc8a73b9a19039a1", - "zh:fa9e0e39ef3e5ed2e6cf5886a0b569415ca3b89b9289fd6d01824ff11e470618", + "h1:0n4RBz9zNw6TTddh5+x7E8L2+qzPXNwKhK4uoZ/DUwE=", + "h1:22Ob7lpzMBSqdrCvoFN5EgmhGPHPBovV/9qo0c/Cd+A=", + "h1:2IRBvmWOYrq/ooaYYn2i86jZb7iIUvlg0KlmOMfDHoQ=", + "h1:5mucXikk4OcW3un3u94QnMx4AB4Wfih+sXeMd5QxSNk=", + "h1:5oU7Zm+2gAVGmxqtJ9E8uTudUkYy/DEn/y3IWphdv4k=", + "h1:5w0R4b1/VSzpqQF1tXXPr/qmaQLPVRXamOmPKWFcTk4=", + "h1:AEVeJr8xGmwad+JUUQ833C3x5d4W+W2szF5DfwxYppw=", + "h1:CPHJ+0zQbS/cX1m55Y90jIOgf1jV3ocUUnqsXAh+9Eg=", + "h1:JPewnGDOJudNer5+ghqwXoaJkfot3QRq9uiEYvo+JHU=", + "h1:QzbluV2vQLxsJYxjpziQCmPndIoJ/UGS4/UHH/GpwUM=", + "h1:TjUNbUdqweRBq/ycQ4ixpNkx5qaYwpXEOn9QCpqNZP8=", + "h1:XNbcODP60ajj21N/OO7af8bBg1ltIsYkq9egn7BYbiY=", + "h1:tgrbgmX7WYQz9G9ncgu7TkpVB+RlLjJA/Rvp9KPlZH8=", + "h1:vLxthX/ZWsOZ+aHKbAMqmNKqD0K5f4nJ8ppy0Ioyup0=", + "h1:wZOdGBAZkY8OKEPjKz82j1HloAKOmmvtjWyTxM+I110=", + "zh:0f719fa5426bc883e9fa6abf7f6498e48025edafbc29015e2f5c028f1cca3b9d", + "zh:1b4d7dafefd6c61764b2f9ed6943ceb9a200dee3590d18747e3a5f6b20ce85e0", + "zh:1d23a712984866d29f7b07028a4e99c783c71f1a5dddf08bc3d4e7da9d91a1fa", + "zh:257d23d58c3bb024b6bc8eb88736eaf912e934ad47c639d0c3c742bddda849a1", + "zh:479860e1a5468f5e04013b9364c9496d7ed0804bf9a1acd8e07558d57609993d", + "zh:4cb5e681bf599b411b27c4a2c4066a5fb2ed79aaa3a1a3cb5a30002fec062ce9", + "zh:4fb35c3f643dae9f3670d719397a415f815a0b95f8ed7bd8a72f27a94ba78092", + "zh:59ba40825ab38db5b4a0989a2db0df35cc15d8984f898176011ba352f27d77b7", + "zh:61fc1252eb88088638f4c69ea4e2171cde2e5089fa632ac1e943b13787348f73", + "zh:7c5d6dd5f7cbc460e95d368be35c29b4e0402069b8912dbd5d1cd7fa9acef216", + "zh:7f76d756240d4284642f359ad470226e5378670239aadc366ef54d9d914d4d2e", + "zh:8133ad0814098177e0d067c816ccf1bf48bbadacd18f6f2c808c90447505723b", + "zh:c93be06269bb728f1968f8c50506de56c887017ac1d6e4be1f925651d8437eb6", + "zh:ef47b78a10a82e6cf53344a6a85a94041c28286c10a70541c564d762f1cfede0", + "zh:f5796a53a74999135bd9087aff50fddda59129d09b2f9b1902ff8c0c1e047e48", ] } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.19.1" - constraints = "~> 0.19" + version = "0.17.0" + constraints = "~> 0.17" hashes = [ - "h1:02CmekB5A4NtAeptTR7waaECJl66Gf3ww7Wtyg1cy+g=", - "h1:0VWwMHkHQXusR2vdyYWN1pNvfJUPt9OWh6glgZshCck=", - "h1:2NGfMzSWPCcyO2P2T9/lDS5MExYnaOfLuDEMiYvFBg4=", - "h1:2dXsvcYEGXsIf1IHD2Rq9HzgwrNNEA4+qOzBKpbFEpw=", - "h1:8R0H9DumQe0DjXTHRJXNiZUyTBDHT2vjfu/vB+WdRIw=", - "h1:9Gi5M2iKY+Oz8wAYoJh7i0u3mySBk5pretuyzcwgJOw=", - "h1:BZwoP8Ubi1iDueWXhSWb3Tlt/II4bJI0/p6/tqw4i1M=", - "h1:F4yWXkyv55F+97IQi/jYBLhBLyhz4hkokVCyEEQzbM4=", - "h1:MGHXEapStuXemfw3SikP2aLQCT+b7Ee/RrCHevSAAWA=", - "h1:Q10nENYoX8jIsBMz1kP8OBrkMeSNmFI2CfB6/MsWKAc=", - "h1:R9z1oacW8gD9hPTGTO37ngDAhAvlP33/PRhzkgXDStY=", - "h1:UYLBa5/V+giaZ2YYjkbPDKMRO9eQRNcWEnpZqU2I/yU=", - "h1:Wz/QDxt1Kved07fCwqMh+6jtTtwQ/xKi9mXE2StV7vw=", - "h1:Yq0JSIW4Jir3EVO2w7C/iM76yzYKG5eYw51woX6CXJo=", - "h1:e/PEmAJszFbjdfsAMJosDbkxtHwHNnByd/3nKJmE3Sg=", - "h1:lMVwei5IGLQCrPMqAmcpbXs/BjaG7k6HbjhuWtr0kIA=", - "h1:m8dyuq1sXJ9sxjZkU3ZUsjC77BWY87GiraNvxHluSYs=", - "zh:15e84e2fca2bbaec9c9f4d14d1a037d02f4d8cc1d1bdff04709d51085f950774", - "zh:1b9d0cbb7f08e7a554afb8c1e733942a77aa0c8112e8b2b7092c040794f0b113", - "zh:28768988a8ec70e5392ca93375717b83a9aa2eeaa21be2f53c28550273b97065", - "zh:2f357c1d2c013627e4e785c350fc72ccc0bfbebaa6062e536ec7eb9cfecfc913", - "zh:4a41bab7fba5415b8c422f5c03d6fb054ae210e1eb180fab4081d4fa976080b5", - "zh:6c10fad0bd03bbf053b97ad51f0a24e9a0c43a7fc10727033a39eec90ed0a51e", - "zh:795980ac64b0f9bda521f829187eb7b296baa682b7e8d409126f85e6a6f9bcc0", - "zh:841a7e234852ecd516e2dc52c7d31025a704242aeafee15eb36bf392e62f918f", + "h1:+qDt35lVSK7acw6a1xHuPYrqmZEcHSmtd+6n1TxNuYw=", + "h1:1dCu2l4DhPBjizVAH/WwAjT1Xbo52K4PMvHoD5zUhuU=", + "h1:Dvn46Auwuel4jqrqZXs2D7kdujNhs17LEmqhuY0k4/4=", + "h1:M5eDL3m2uSEr1XATJW0foHzKl8pFhCtgKuOM24bJRwU=", + "h1:PddaC7nM/gY4x9i3xy6TxOs9MAu2/6g58Xs/gv4DRV8=", + "h1:QVIKiZluI+NQAKu8NpFBl3Nvyx+d81vW9btEUdIQREc=", + "h1:S6TnzXHsRoGYvC1vJBkDiVEc0spceksY4n6x5WN5iYw=", + "h1:VcxZDWqCWMSjcUsC1K4sB6uYEoeoou+BC0ePoJXmf3A=", + "h1:W0y/agBVqls1cJlFGFYMu2VnqoPXFzxVHPIYe3OqfYQ=", + "h1:XmNd5fP9a0O77ve5BMQP2vARExgIa7rYl6KvyUYXPSs=", + "h1:e0EFKrWSQwaa/kGhnha4DXk4T68Av8QxP84mRSdWC9M=", + "h1:eM+/lUiU0pNSgQKoqKPgE3xJrJ0MHIpKG+yhaGB/P0M=", + "h1:fPWWA4T0/y7GX+tCGN23l1jODhZ3uCdR/MKgZDXYpAE=", + "h1:g+r2GVi4gVC4DuQg3PL70gW9BDskgWUzCBIMXTUq63A=", + "h1:gaZ8eALDtVHqykVDHav8004gHiMGaYR/3KwET0FUgao=", + "h1:kbqW25eaiv4N/N/z+sxLdJZ15yh5cgnRD/q6RclPMLc=", + "h1:rGjxue3mXRyQQqpywTXC4zK//JAtf0Cz7RP+uPMMJjw=", + "zh:05943fef14c2028f4722bf078aa1889229e94302f7678cc6f63adb669d8ea612", + "zh:26a163930a92a7408f7bbd0130064b84df8a232b500d8c6c3989952986308539", + "zh:41305feaaade55391447521ec309f3c038b631ca542907ad95132fab71a7e116", + "zh:606919a930f0299948504adbdcd0f239a8af5c418f85741c48f8add370a3d038", + "zh:66963d5b445639511939fc508513fd31da3ee1d4ee1a565ee396c9532897a349", + "zh:6c981ec0c8545556395c43e2511861ab65ee9ecf2a960480e7889c3af0d23af3", + "zh:7334a1bdb726ce1f1bf0a3155f30f84f65206980c229c832ff5f0b0718c44e0b", + "zh:75f6c86bf74511e605423332d113711c76c8028361a32282fb3359d6c7ecae9e", + "zh:7aebb1a01cfe8be54903853202ae06eba14ad99c37d230ed93ce7d6633e05e9b", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:8c876a0a26b249151f0918ee836e24660513c2345c23a06e0f54a28c7c6966b7", - "zh:906a791496cf11dcfdfe43a4167414cb473e548f0bd86d4b8a6a95450b523133", - "zh:9431757bf5b7d055ecc1d3c8465216d5e7a648b75e0e35b136ac8fc25382e97d", - "zh:967fd248df86fc246536727e996a7334711f4c6fe9e3d01ec9f03abe621a9783", - "zh:a6710fbd30f8a4844eb025d616057e78bbb255a7d05cc73876dae55a5d6b0d06", - "zh:b5f260899555430c109da35d2c483999d61b127beda6c2a7b39eb67f27d754c8", - "zh:d24882eb63c6452ed27666e896224b1d866ad00645d6ecc53de84d1687d27fc4", - "zh:e573eb0c99ea7195ec1ed3da728caddacc68a126d82679fb490438c781382f4d", - "zh:fcf0a66c0c0b62af505c5f0f8fe59bca77d6085a0b07763be33420101183702e", + "zh:9041d0e20c9ceea532de6eebf5cb3a27dad0bb49d3f5b5154be2a08d68fbbf1f", + "zh:a6bbf65431a02be4df0ebb1cbe01185ad357ff6e33c01bd0558f59bed90c8f36", + "zh:c6d075a31096f080c388dfe46036f451c0cc114c3311a4f46ab8dbe1938a202f", + "zh:dd8703f7b55b8bc8e10f8718bea889781100b18e932b04898995b63178c3d36e", + "zh:dd92a5cd4e133a4000e7e5bc8cce876ae0ed803543cedd2f3d590661ba244d04", + "zh:e024fdf121bebc48c1e6debea344c6d4f174117f3ae605fca6e13b9705d92d22", + "zh:ee0e80c31b438e35fa1608f6a2f5824d2806db1e5e8b9f7a90986585c7bcb895", + "zh:fc2d4b705411b48f8c045981f9368a3ea2f74969dd6302008c31ff0bedd51f0a", ] } diff --git a/README.md b/README.md index 99d1143..989069e 100644 --- a/README.md +++ b/README.md @@ -84,23 +84,6 @@ This elastic server architecture offers a robust and performant backend while mi ## Technical Details -### Packages requiring a newer R - -The daily update and weekly full-build pipelines run `local/build-newer-r.R` under each non-primary installed R, alongside the ABI-sensitive pass. -It selects current CRAN versions whose `Depends: R` constraints cannot be met by the slot's primary R but can be met by the running R, including incompatibility inherited through `Depends`, `Imports`, and `LinkingTo`. -These builds retain bincraft's ABI classification and storage routing: sensitive binaries use the running minor, while safe binaries can be shared. -No shared S3 existence cache is passed, so bincraft checks the actual objects and distinguishes a source fallback from a binary. -The pass reads unfiltered CRAN metadata independently of the weekly snapshot and its primary-R failure exclusions; existing binaries and failures for the running minor retain bincraft's usual skip behavior. -Weekly workers split this extra work across their existing shards. -This does not select older archive versions for unsupported R minors or change index filtering. - -Preview the selection without building or accessing storage or build metadata: - -```sh -# Run with the target R and supply the slot's full primary R version. -/opt/R/4.6.0/bin/Rscript local/build-newer-r.R 4.4.3 --dry-run -``` - ### Creating/Updating the PACKAGES Index Files Currently, the {cranlike} and {desc} packages only work with files on a local file system. @@ -215,28 +198,6 @@ Example: Technically, no date-based snapshots are planned, so this component from the Posit PM structure is not included. -## Retiring a slot - -A slot is only reachable if it is listed in `local.rpkgs_slots` in `cdn.tf`, which feeds `UNION_SLOTS` in `edge/rpkgs-router.ts`. -An unrouted slot serves the flat index to every client, so its binaries fail at `dyn.load()` under every R minor but one. - -In order: - -1. Remove it from `local.rpkgs_slots` in `cdn.tf` and from `DistroId` in the website's `src/src/data/repos.ts`, then `tofu apply`. -2. `just retire-slot --apply`, then commit. -3. Delete its cron jobs on the Crow server; `local/retire-slot.R` prints them with the API call. -4. Delete its objects: - - ```sh - crow pipeline create devxy/build-cran-binaries --branch main \ - --var retire_slot= --var RETIRE_APPLY=yes - ``` - -Steps 2 and 4 refuse any slot still listed in `cdn.tf`. -Running step 4 before step 2 means the next nightly rebuilds everything just deleted. - -`just check-slots` fails when a pipeline still targets an unrouted slot. - ## Helpers Helper scripts are located in `local/`. diff --git a/cdn.tf b/cdn.tf index df30273..5e8899d 100644 --- a/cdn.tf +++ b/cdn.tf @@ -32,7 +32,7 @@ # cache_stale = ["offline", "updating"] # use_background_update = true -# block_ips = var.cdn_block_ips + # block_ips = var.cdn_block_ips # # 50 TB # limit_bandwidth = 50000000000000 @@ -52,26 +52,6 @@ ### cran.rpkgs.com -locals { - rpkgs_slots = [ - for pair in setproduct( - ["amd64", "arm64"], - ["resolute", "noble", "jammy", "rhel8", "rhel9", "rhel10", "alpine323", "alpine324"] - ) : "${pair[0]}/${pair[1]}" - ] - - # The supported R minors: the current one plus the two previous, which is - # exactly what build-env-images installs as R_VERSION_LATEST / PREV1 / PREV2. - # These must stay in step. A minor listed here without a published index - # sends those clients to a 404; a published minor missing from this list - # sends them to CRAN for sources instead of serving the binaries we built. - rpkgs_supported_minors = ["4.4", "4.5", "4.6"] - - # bunny.net serves every pull zone on .b-cdn.net, so staging needs no - # DNS record and is never advertised. - rpkgs_test_hostname = "cran-rpkgs-test.b-cdn.net" -} - # The edge middleware that resolves the bare cran.rpkgs.com form to an # / slot and routes PACKAGES* to the per-R-minor slot. The source of # truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release. @@ -89,26 +69,9 @@ resource "bunnynet_compute_script" "rpkgs_router" { # slot that is not listed here would hide every package the per-minor index does # not carry, so this stays empty until a slot has been backfilled. resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { - script = bunnynet_compute_script.rpkgs_router.id - name = "UNION_SLOTS" - # Enabled. Until this was set, every client resolved against the generic - # index and never reached a per-minor binary: an R 4.6.1 client on resolute - # downloaded the 4.5-built rlang (2079570 bytes) while the correct 4.6 build - # (2075106 bytes) sat unused one directory away, and died at load with - # `undefined symbol: SETLENGTH`. - # - # Verified before enabling, against the staging zone with the same script and - # the same origin: all 16 slots report zero regressions against the generic - # slot, an excluded R minor is sent to CRAN, a client without an R minor - # still gets the generic index, and tarball requests are never rewritten. - default_value = join(",", local.rpkgs_slots) - required = false -} - -resource "bunnynet_compute_script_variable" "rpkgs_router_known_minors" { script = bunnynet_compute_script.rpkgs_router.id - name = "KNOWN_MINORS" - default_value = join(",", local.rpkgs_supported_minors) + name = "UNION_SLOTS" + default_value = "" required = false } @@ -119,7 +82,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { cache_expiration_time = 31919000 websockets_enabled = false - errorpage_whitelabel = true + errorpage_whitelabel = true origin { type = "OriginUrl" @@ -184,151 +147,6 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } -### Staging zone for edge-router changes - -# Every published / slot. The staging zone enables per-minor routing -# for all of them at once; production adopts the same list only after -# `scripts/verify-r-minor-routing.sh --live` passes against staging. - -# A second copy of the same router, bound to the same B2 origin, so UNION_SLOTS -# can be exercised end to end before production is touched. -resource "bunnynet_compute_script" "rpkgs_router_test" { - type = "middleware" - name = "rpkgs-router-test" - content = file("${path.module}/edge/rpkgs-router.ts") -} - -resource "bunnynet_compute_script_variable" "rpkgs_router_test_union_slots" { - script = bunnynet_compute_script.rpkgs_router_test.id - name = "UNION_SLOTS" - default_value = join(",", local.rpkgs_slots) - required = false -} - -# Without this the staging zone rewrites to PUBLIC_CDN_ORIGIN, so its redirects -# land on production and the test silently measures the wrong system. -resource "bunnynet_compute_script_variable" "rpkgs_router_test_extra_hosts" { - script = bunnynet_compute_script.rpkgs_router_test.id - name = "EXTRA_PUBLIC_HOSTS" - default_value = local.rpkgs_test_hostname - required = false -} - -resource "bunnynet_compute_script_variable" "rpkgs_router_test_known_minors" { - script = bunnynet_compute_script.rpkgs_router_test.id - name = "KNOWN_MINORS" - default_value = join(",", local.rpkgs_supported_minors) - required = false -} - -resource "bunnynet_pullzone" "cran_rpkgs_test" { - name = "cran-rpkgs-test" - - cache_errors = false - - cache_expiration_time = 31919000 - websockets_enabled = false - errorpage_whitelabel = true - - origin { - type = "OriginUrl" - url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" - middleware_script = bunnynet_compute_script.rpkgs_router_test.id - } - - routing { - filters = [ - "scripting", - ] - } - - s3_auth_enabled = true - s3_auth_key = var.B2_S3_ACCESS_KEY - s3_auth_secret = var.B2_S3_SECRET_KEY - s3_auth_region = "eu-central-003" - - cache_enabled = true - request_coalescing_enabled = true - block_post_requests = true - - cache_vary_headers = ["User-Agent"] - - # Staging carries only synthetic verification traffic, so the production - # ceilings would be pure headroom. - limit_requests = 500 - limit_connections = 100 - - safehop_enabled = true - add_canonical_header = true - cache_stale = ["offline", "updating"] - block_ips = var.cdn_block_ips - - # 1 TB - limit_bandwidth = 1000000000000 - - block_root_path = true -} - - -# Alliance SwissPass historically used a separate, manually configured pull -# zone. Adopt it so both public repositories use the same B2 origin, middleware -# release and cache behavior. -import { - to = bunnynet_pullzone.cran_allianceswisspass - id = "3265648" -} - -resource "bunnynet_pullzone" "cran_allianceswisspass" { - name = "cran-allianceswisspass" - - cache_errors = false - cache_expiration_time = 31919000 - websockets_enabled = false - errorpage_whitelabel = true - - origin { - type = "OriginUrl" - url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" - middleware_script = bunnynet_compute_script.rpkgs_router.id - } - - routing { - filters = [ - "scripting", - ] - } - - s3_auth_enabled = true - s3_auth_key = var.B2_S3_ACCESS_KEY - s3_auth_secret = var.B2_S3_SECRET_KEY - s3_auth_region = "eu-central-003" - - cache_enabled = true - request_coalescing_enabled = true - block_post_requests = true - cache_vary_headers = ["User-Agent"] - - limit_requests = 5000 - limit_connections = 1000 - - safehop_enabled = true - add_canonical_header = true - cache_stale = ["offline", "updating"] - block_ips = var.cdn_block_ips - - # 50 TB - limit_bandwidth = 50000000000000 - - block_root_path = true -} - -resource "bunnynet_pullzone_hostname" "cran_allianceswisspass" { - pullzone = bunnynet_pullzone.cran_allianceswisspass.id - name = "cran.allianceswisspass.devxy.io" - force_ssl = true - tls_enabled = true -} - # resource "bunnynet_storage_zone" "devxy-r-binaries" { # name = "devxy-r-binaries-storage" # region = "DE" diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 3e2fe8b..553185d 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -17,12 +17,7 @@ const UNION_SLOTS = 'amd64/alpine324'; const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; -const UA_R43_MUSL = 'R (4.3.3 x86_64-pc-linux-musl x86_64 linux-musl)'; -const UA_R47_MUSL = 'R (4.7.0 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; -const UA_R45_RESOLUTE = 'R/4.5.3 (Ubuntu 26.04) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; -const UA_R45_FUTURE_UBUNTU = - 'R/4.5.3 (Ubuntu 28.04; codename=dynamic-dugong) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)'; const UA_CURL = 'curl/8.0.1'; @@ -98,33 +93,6 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`); }); - // We publish binaries only for the supported window. An excluded minor has - // no slot we can serve safely, so it goes to CRAN for sources rather than - // to a 404 or to binaries built under another minor. - await t.step('sends an excluded R minor to CRAN for the index', async () => { - const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R43_MUSL); - assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); - }); - - await t.step('sends a future R minor to CRAN too', async () => { - const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R47_MUSL); - assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); - }); - - // The index and the tarballs R resolves against it have to come from the - // same place. Serving one from CRAN and the other from here would hand R a - // binary where it expects a source tarball. - await t.step('sends an excluded minor to CRAN for tarballs as well', async () => { - const res = await probe(`${SLOT}/foo_1.0.tar.gz`, UA_R43_MUSL); - assertEquals(res.location, 'https://cran.r-project.org/src/contrib/foo_1.0.tar.gz'); - }); - - await t.step('leaves an excluded minor alone on a slot outside UNION_SLOTS', async () => { - const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R43_MUSL); - assertEquals(res.location, null); - assertEquals(res.status, 200); - }); - await t.step('routes PACKAGES and PACKAGES.rds too', async () => { for (const file of ['PACKAGES', 'PACKAGES.rds']) { const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL); @@ -149,13 +117,6 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.status, 200); }); - await t.step('serves an archived binary when it exists', async () => { - const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`; - const res = await probe(path, UA_R45_MUSL); - assertEquals(res.status, 200); - assertEquals(res.location, null); - }); - await t.step('does not redirect a path already under a minor', async () => { const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); assertEquals(res.location, null); @@ -173,16 +134,6 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); }); - await t.step('resolves Ubuntu 26.04 to the resolute slot', async () => { - const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_RESOLUTE); - assertEquals(res.location, 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz'); - }); - - await t.step('resolves a future Ubuntu release from its codename', async () => { - const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_FUTURE_UBUNTU); - assertEquals(res.location, 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz'); - }); - await t.step('sends an unidentifiable distro to CRAN', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL); assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index 05b6e18..9cd410b 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -27,17 +27,6 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; const CRAN_ORIGIN = 'https://cran.r-project.org'; -const PUBLIC_CDN_HOSTS = new Set([ - 'cran.rpkgs.com', - 'cran.allianceswisspass.devxy.io', - // Staging hostnames, so the identical script can run on a test pull zone and - // redirect within itself. Without this a test zone rewrites to - // PUBLIC_CDN_ORIGIN, quietly exercising production instead of itself. - ...(Deno.env.get('EXTRA_PUBLIC_HOSTS') ?? '') - .split(',') - .map((host) => host.trim()) - .filter((host) => host.length > 0), -]); /** Slots ("/", comma separated) whose per-minor index is a union. */ const UNION_SLOTS = new Set( @@ -47,21 +36,6 @@ const UNION_SLOTS = new Set( .filter((slot) => slot.length > 0), ); -/** - * R minors for which a per-minor index is actually published. - * - * contribPath() has no way to probe the origin, so a minor that is not - * published here must fall back to the flat index. Routing an unlisted minor - * would send that client to a 404 and it would see no packages at all - a - * silent, total failure rather than a degraded one. - */ -const KNOWN_MINORS = new Set( - (Deno.env.get('KNOWN_MINORS') ?? '4.4,4.5,4.6') - .split(',') - .map((minor) => minor.trim()) - .filter((minor) => minor.length > 0), -); - /** `///latest/src/contrib[/]` */ const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/; @@ -73,19 +47,13 @@ const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; -/** A binary archive URL whose upstream source counterpart CRAN can serve. */ -const ARCHIVE_TARBALL_REGEX = - /^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/; - const MACOS_BIN_REGEX = /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i; const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i; -const UBUNTU_CODENAME_REGEX = /Ubuntu [\d.]+;\s*codename=([a-z][a-z0-9-]*)/i; const UBUNTU_CODENAMES: Record = { - '26.04': 'resolute', '24.04': 'noble', '22.04': 'jammy', }; @@ -117,22 +85,6 @@ function redirectTo(location: string, status = 302): Response { }); } -function publicCdnOrigin(url: URL): string { - return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; -} - -/** - * True when the client reports an R minor that we deliberately do not serve. - * - * A client that reports no minor at all is not "unsupported": non-R fetchers - * (mirror scripts, image builds) must keep getting the flat slot. Only a - * known-and-excluded minor falls through to CRAN. - */ -function isExcludedMinor(userAgent: string): boolean { - const rMinor = extractRMinor(userAgent); - return rMinor !== null && !KNOWN_MINORS.has(rMinor); -} - function extractRMinor(userAgent: string): string | null { for (const regex of R_MINOR_REGEXES) { const match = userAgent.match(regex); @@ -175,11 +127,6 @@ function parseSlot(userAgent: string): string | null { const ubuntu = userAgent.match(UBUNTU_REGEX); if (ubuntu) { - const codenameMatch = userAgent.match(UBUNTU_CODENAME_REGEX); - if (codenameMatch) { - return `${arch}/${codenameMatch[1].toLowerCase()}`; - } - const codename = UBUNTU_CODENAMES[ubuntu[1]]; if (codename) { return `${arch}/${codename}`; @@ -214,8 +161,8 @@ function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: * The contrib path a request should be served from, relative to the slot. * * Returns the per-minor path for an index file when the slot is known to carry - * a union index and the client's R minor is one we publish; otherwise the flat - * path, which is what every client sees today. + * a union index and the client's R minor is known; otherwise the flat path, + * which is what every client sees today. */ function contribPath(slot: string, rest: string, userAgent: string): string { const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`; @@ -225,7 +172,7 @@ function contribPath(slot: string, rest: string, userAgent: string): string { } const rMinor = extractRMinor(userAgent); - return rMinor && KNOWN_MINORS.has(rMinor) ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; + return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; } BunnySDK.net.http @@ -234,14 +181,15 @@ BunnySDK.net.http const url = new URL(ctx.request.url); const path = normalizePathname(url.pathname); const userAgent = ctx.request.headers.get('User-Agent') || ''; - const publicOrigin = publicCdnOrigin(url); // macOS clients are served from CRAN's own binary tree. const srcContrib = path.match(SRC_CONTRIB_REGEX); if (srcContrib && /darwin/.test(userAgent)) { const mac = parseMacUserAgent(userAgent); if (mac) { - return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`)); + return Promise.resolve( + redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`), + ); } } @@ -261,21 +209,11 @@ BunnySDK.net.http return Promise.resolve(ctx.request); } - // An R minor outside the supported window has no binaries we can safely - // serve, so the whole interaction goes to CRAN: the index and the - // tarballs R will resolve against it. Serving the index from CRAN but - // tarballs from here would hand R a binary where it expects a source - // tarball, which fails in a far more confusing way than not being - // served at all. - if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) { - return Promise.resolve(redirectTo(`${CRAN_ORIGIN}/src/contrib/${rest}`)); - } - const target = contribPath(slot, rest, userAgent); if (target === path) { return Promise.resolve(ctx.request); } - return Promise.resolve(redirectTo(`${publicOrigin}${target}`)); + return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`)); } // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. @@ -285,32 +223,13 @@ BunnySDK.net.http return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); } - if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) { - return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); - } - const rest = srcContrib ? srcContrib[1] : ''; - return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); + return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`)); } return Promise.resolve(ctx.request); }) - .onOriginResponse(async (ctx) => { - const path = normalizePathname(new URL(ctx.request.url).pathname); - const archive = path.match(ARCHIVE_TARBALL_REGEX); - - // Binary archives can be incomplete when an older build never succeeded. - // Preserve renv/remotes version restores by falling back to CRAN's source - // package only for an absent archived tarball. A requested version can be - // either archived upstream or still current, so probe the archive first. - // Other 404s remain visible. - if (ctx.response.status === 404 && archive) { - const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`; - const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' }); - const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`; - return redirectTo(sourceUrl); - } - + .onOriginResponse((ctx) => { ctx.response.headers.append('X-Via', 'MyMiddleware'); return Promise.resolve(ctx.response); }); diff --git a/justfile b/justfile index 524ac9a..6c75b13 100644 --- a/justfile +++ b/justfile @@ -87,18 +87,3 @@ edge-test: denoland/deno:latest \ deno test -A edge/rpkgs-router.test.ts fi - -# fail if any pipeline still targets a slot the edge does not route -check-slots: - Rscript local/check-slots.R - -# strip a retired slot from .crow/*.yaml (dry run; pass --apply to write) -retire-slot codename *flags: - Rscript local/retire-slot.R {{ codename }} --config-only {{ flags }} - -# Format or lint selected R files, defaulting to local tooling. -format *files="local": - air format {{ files }} - -fix *files="local": - jarl check --fix {{ files }} diff --git a/local/backfill-abi-classification.R b/local/backfill-abi-classification.R deleted file mode 100644 index 0f48392..0000000 --- a/local/backfill-abi-classification.R +++ /dev/null @@ -1,129 +0,0 @@ -### Classify every compiled package in a slot's flat index, into the ABI cache. -# -# Usage: Rscript local/backfill-abi-classification.R [limit] -# -# The per-minor scheme only ever classified packages that passed through the -# build pipeline after it existed. Everything built before that, and not updated -# on CRAN since, has no verdict at all -- and bincraft's risky set is what -# decides whether a flat binary may be carried into a per-minor index. With no -# verdict the answer defaults to "carry it", which is how a base64enc built -# under R 4.4.3 reached R 4.6 clients and died at dyn.load() on SETLENGTH. -# -# The verdict is a property of the source, so it is independent of OS, arch and -# R minor: one slot's backfill populates the cache for every slot. The slot -# argument only selects which flat index to enumerate. -# -# Only packages with NeedsCompilation=yes are classified. A package with no -# compiled code has no symbols to bind and cannot fail this way, so downloading -# its source to prove that would be wasted work. -# -# Resumable: `classify_r_minor_sensitive()` returns cached verdicts without -# touching the network, so a re-run costs one query per package and only the -# unclassified remainder is downloaded. - -options(error = function() { - cat("ERROR:", geterrmessage(), "\n", file = stdout()) - traceback(2) - q(status = 1) -}) - -library(bincraft, quietly = TRUE) - -args <- commandArgs(trailingOnly = TRUE) -if (length(args) < 2L) { - stop( - "usage: backfill-abi-classification.R [limit]", - call. = FALSE - ) -} -arch <- args[[1L]] -codename <- args[[2L]] -limit <- if (length(args) >= 3L) as.integer(args[[3L]]) else NA_integer_ - -db <- list( - metadata_db_type = "postgres", - metadata_db_host = "r-binaries.devxy.io", - metadata_db_name = "build_metadata", - metadata_db_port = 15432L, - metadata_db_user = "rpkgs", - metadata_db_password = Sys.getenv("PGPASS"), - metadata_db_sslmode = "require" -) - -read_index <- function(index_url) { - con <- gzcon(url(index_url, open = "rb")) - on.exit(close(con), add = TRUE) - read.dcf(con, fields = c("Package", "Version", "Built")) -} - -flat <- read_index(sprintf( - "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", - arch, - codename -)) -# Source fallbacks have no binary to be unsafe, so skip them. -flat <- flat[!is.na(flat[, "Built"]), , drop = FALSE] - -cran <- read.dcf( - gzcon(url( - "https://cloud.r-project.org/src/contrib/PACKAGES.gz", - open = "rb" - )), - fields = c("Package", "NeedsCompilation") -) -compiled <- cran[ - !is.na(cran[, "NeedsCompilation"]) & cran[, "NeedsCompilation"] == "yes", - "Package" -] - -todo <- flat[flat[, "Package"] %in% compiled, , drop = FALSE] -cat(sprintf( - "%s/%s: %d binaries in the flat index, %d compiled and due classification\n", - arch, - codename, - nrow(flat), - nrow(todo) -)) -if (!is.na(limit) && limit > 0L && limit < nrow(todo)) { - todo <- todo[seq_len(limit), , drop = FALSE] - cat(sprintf("Limited to the first %d\n", limit)) -} - -sensitive <- 0L -failed <- 0L -for (i in seq_len(nrow(todo))) { - pkg <- todo[i, "Package"] - ver <- todo[i, "Version"] - verdict <- tryCatch( - do.call( - bincraft::classify_r_minor_sensitive, - c(list(package_name = pkg, tag = ver), db) - ), - error = function(e) { - cat(sprintf(" %s %s: %s\n", pkg, ver, conditionMessage(e))) - NA - } - ) - if (is.na(verdict)) { - failed <- failed + 1L - } else if (isTRUE(verdict)) { - sensitive <- sensitive + 1L - } - if (i %% 250L == 0L) { - cat(sprintf( - " classified %d/%d (%d r-minor-sensitive, %d failed)\n", - i, - nrow(todo), - sensitive, - failed - )) - utils::flush.console() - } -} - -cat(sprintf( - "Done: %d classified, %d r-minor-sensitive, %d failed\n", - nrow(todo), - sensitive, - failed -)) diff --git a/local/backfill-flat-safety.R b/local/backfill-flat-safety.R deleted file mode 100644 index 59faa3c..0000000 --- a/local/backfill-flat-safety.R +++ /dev/null @@ -1,199 +0,0 @@ -### Record, per flat-slot object, whether it loads under every supported R minor. -# -# Usage: Rscript local/backfill-flat-safety.R [limit] -# -# `union_index_records()` keeps a carried flat record when that object is -# recorded safe, and drops it otherwise. Objects uploaded before that record -# existed have no verdict, so they are dropped: a full re-index removed ~2000 -# records per per-minor index, all resolving to CRAN source for clients the -# routing serves correctly. -# -# This inspects the objects already in the bucket and writes the verdict, which -# is what restores that coverage. The question is decided by symbols, not -# heuristics: a binary fails at dyn.load() under minor M exactly when it -# references a symbol M does not export, and the volatile set is derived by -# diffing the R builds installed in this image. -# -# Only compiled packages are inspected. A package with no shared object cannot -# fail this way and is never dropped, so proving it would be wasted downloads. -# -# Resumable: objects already recorded are skipped, so a re-run costs one query. - -options(error = function() { - cat("ERROR:", geterrmessage(), "\n", file = stdout()) - traceback(2) - q(status = 1) -}) - -library(bincraft, quietly = TRUE) - -args <- commandArgs(trailingOnly = TRUE) -if (length(args) < 2L) { - stop("usage: backfill-flat-safety.R [limit]", call. = FALSE) -} -arch <- args[[1L]] -codename <- args[[2L]] -limit <- if (length(args) >= 3L) as.integer(args[[3L]]) else NA_integer_ - -sets <- bincraft:::installed_r_symbol_sets() -if (is.null(sets)) { - stop( - "Fewer than two R minors are installed here, so nothing can be compared. Refusing to record verdicts that would all read 'safe'.", # nolint - call. = FALSE - ) -} -cat(sprintf( - "Comparing R minors: %s (%d symbols vary across them)\n", - paste(names(sets), collapse = ", "), - length(bincraft:::cross_minor_volatile_symbols(sets)) -)) - -# The flat slot is listed from S3, not from PACKAGES.gz over HTTPS. Reading the -# index from inside R sends an R User-Agent, which the edge routes to a -# per-minor index; that enumerates union records carrying `Path: `, whose -# objects do not exist at the flat path. Every download then 404s: a first run -# skipped 292 of 300 that way. The bucket cannot be routed and is authoritative. -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE -) -contrib <- sprintf( - "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", - arch, - codename -) -objects <- s3fs::s3_dir_ls(contrib, recurse = TRUE) -relative <- sub(sprintf("^(s3://)?%s/", contrib), "", objects) -# Flat objects only: no `/` prefix and nothing under Archive/. -flat_files <- relative[ - grepl("\\.tar\\.gz$", relative) & !grepl("/", relative, fixed = TRUE) -] -flat_pkg <- sub("_.*$", "", flat_files) -flat_ver <- sub("\\.tar\\.gz$", "", sub("^[^_]*_", "", flat_files)) - -# Only compiled packages can fail at dyn.load(), so the rest are not worth -# downloading. Ask the slot's own index rather than CRAN's: CRAN lists only the -# versions it currently ships, so a package it has archived is absent, is never -# recognised as compiled, and is never inspected. No verdict is ever recorded -# for it, while `union_index_records()` reads the slot's own `NeedsCompilation` -# and happily drops it -- with nothing able to clear the flag, because the -# repair works from recorded verdicts too. `mixedMem` and `vcpen`, both removed -# from CRAN, were stuck out of every rhel8 per-minor index that way. -# -# Read it from the bucket, not over HTTPS: this runs inside R, so the edge would -# route the request to a per-minor index and the flat objects would not all be -# there. Same reason the object listing above comes from S3. -index_file <- file.path(tempdir(), "flat-PACKAGES.gz") -s3fs::s3_file_download( - sprintf("%s/PACKAGES.gz", contrib), - index_file, - overwrite = TRUE -) -index <- read.dcf( - gzfile(index_file), - fields = c("Package", "Version", "NeedsCompilation") -) -if (nrow(index) == 0L) { - stop( - sprintf( - "Read no records from %s/PACKAGES.gz; refusing to guess what is compiled.", - contrib - ), - call. = FALSE - ) -} -# Key on the object, not the package: a slot can carry several versions and -# only some of them compile. -compiled_objects <- paste( - index[, "Package"], - index[, "Version"], - sep = "_" -)[ - !is.na(index[, "NeedsCompilation"]) & - tolower(index[, "NeedsCompilation"]) == "yes" -] -keep <- paste(flat_pkg, flat_ver, sep = "_") %in% compiled_objects -compiled <- data.frame( - Package = flat_pkg[keep], - Version = flat_ver[keep], - stringsAsFactors = FALSE -) -cat(sprintf( - "%s/%s: %d flat objects, %d compiled and worth inspecting\n", - arch, - codename, - length(flat_files), - nrow(compiled) -)) -if (!is.na(limit) && limit > 0L && limit < nrow(compiled)) { - compiled <- compiled[seq_len(limit), , drop = FALSE] - cat(sprintf("Limited to the first %d\n", limit)) -} - -# Fetch from the bucket as well: the edge does not rewrite tarball requests, -# but it does cache them, and the object is what the verdict is about. -safe <- 0L -unsafe <- 0L -skipped <- 0L -for (i in seq_len(nrow(compiled))) { - pkg <- compiled[i, "Package"] - ver <- compiled[i, "Version"] - tarball <- file.path(tempdir(), sprintf("%s_%s.tar.gz", pkg, ver)) - ok <- tryCatch( - { - s3fs::s3_file_download( - sprintf("%s/%s_%s.tar.gz", contrib, pkg, ver), - tarball, - overwrite = TRUE - ) - file.exists(tarball) - }, - error = function(e) FALSE, - warning = function(w) FALSE - ) - if (!ok) { - skipped <- skipped + 1L - next - } - verdict <- bincraft:::tarball_abi_verdict(tarball, sets) - unlink(tarball) - if (!isTRUE(verdict$inspected)) { - skipped <- skipped + 1L - next - } - bincraft:::flat_safety_store( - pkg, - ver, - codename, - arch, - safe = !isTRUE(verdict$sensitive), - unsupported = verdict$unsupported - ) - if (isTRUE(verdict$sensitive)) { - unsafe <- unsafe + 1L - } else { - safe <- safe + 1L - } - if (i %% 250L == 0L) { - cat(sprintf( - " inspected %d/%d (%d safe, %d unsafe, %d skipped)\n", - i, - nrow(compiled), - safe, - unsafe, - skipped - )) - utils::flush.console() - } -} - -cat(sprintf( - "Done: %d inspected, %d safe for every minor, %d unsafe, %d skipped\n", - nrow(compiled), - safe, - unsafe, - skipped -)) diff --git a/local/build-all.R b/local/build-all.R index 349092f..37fc593 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -26,50 +26,9 @@ package_cache_files <- c( "/mnt/cache/packages/r_minor_sensitive_pkgs.rds", "/mnt/cache/packages/s3_cache.rds" ) -# Existence is not freshness. The snapshot describes S3 and CRAN state at the -# moment it was written, and the volume is per-agent, so an agent that ran an -# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324 -# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43 -# sensitive packages, while the install-deps step in the very same pipeline had -# just computed 6871 on another agent. -# -# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes -# once per agent and its shards then share the result. Off CI, or when no such -# variable is set, fall back to an age check. -snapshot_id_path <- "/mnt/cache/packages/snapshot.id" -snapshot_ttl_hours <- as.numeric( - Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2") -) -current_snapshot_id <- "" -for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) { - val <- Sys.getenv(v, unset = "") - if (nzchar(val)) { - current_snapshot_id <- paste(v, val, sep = "=") - break - } -} - -snapshot_is_stale <- function() { - if (!all(file.exists(package_cache_files))) { - return(TRUE) - } - if (nzchar(current_snapshot_id)) { - cached <- tryCatch( - readLines(snapshot_id_path, warn = FALSE)[1L], - error = function(e) NA_character_, - warning = function(w) NA_character_ - ) - return(!identical(cached, current_snapshot_id)) - } - age_hours <- as.numeric( - difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours") - ) - isTRUE(age_hours > snapshot_ttl_hours) -} - -if (snapshot_is_stale()) { +if (!all(file.exists(package_cache_files))) { message( - "Package snapshot missing or stale; recomputing via packages-to-build.R" + "Package snapshot missing from cache; recomputing via packages-to-build.R" ) dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { @@ -83,9 +42,6 @@ if (snapshot_is_stale()) { pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" ) - if (nzchar(current_snapshot_id)) { - writeLines(current_snapshot_id, snapshot_id_path) - } message("Package snapshot recomputed.") } @@ -154,25 +110,10 @@ con <- DBI::dbConnect( password = Sys.getenv("PGPASS"), sslmode = "require" ) -# Scope the skip to the R minor this pass is running under. `single_builds` -# records `r_version` per attempt, but querying without it made a non-primary -# pass skip everything the primary pass had already attempted under a different -# minor - so `--sensitive-only` under 4.6 skipped packages that had only ever -# been built for 4.5, and the per-minor slots never filled. That is why -# amd64/resolute served 4000 fewer packages to a 4.6 client than to a 4.5 one. -r_minor <- paste( - R.version$major, - strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], - sep = "." -) built <- DBI::dbGetQuery( con, - paste( - "SELECT name, tag FROM single_builds", - "WHERE platform = $1 AND arch = $2", - "AND substring(r_version from '^[0-9]+[.][0-9]+') = $3" - ), - params = list(platform, arch, r_minor) + "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2", + params = list(platform, arch) ) DBI::dbDisconnect(con) before <- nrow(chunk) @@ -180,56 +121,16 @@ chunk <- chunk[ !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] sprintf( - "Skipped %d package versions already attempted under R %s; %d remaining for this job", + "Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), - r_minor, nrow(chunk) ) # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, # saving significant memory for the dependency-installer subprocesses -s3_cache_paths <- readRDS("/mnt/cache/packages/s3_cache.rds") - -# The cache is stored as paths relative to the slot, so a pass can select the -# objects that belong to it. `build_binary_package()` compares basenames, which -# cannot distinguish `4.4/curl_1.0.tar.gz` from `curl_1.0.tar.gz`, so the choice -# has to be made here where the running R minor is known. -# -# Getting this wrong is expensive in both directions: hand it everything and a -# per-minor pass believes the flat slot's binaries are its own and builds -# nothing; hand it nothing and it rebuilds what it already has. amd64/resolute -# recompiled 8683 packages that way. -select_cache_for_pass <- function(paths, sensitive_only, r_minor) { - # A cache written before #191 holds bare basenames. Filtering those by path - # would select nothing and trigger a full rebuild, so use them as they are; - # #190's staleness check replaces it on the next pipeline anyway. - # - # The format is read from the marker `packages-to-build.R` sets, not guessed - # from the content: a slot-relative cache with no per-minor or Archive object - # holds bare names too, and treating that as legacy would hand a per-minor - # pass the flat slot's binaries and build nothing. - if (!isTRUE(attr(paths, "slot_relative"))) { - message("S3 cache is in the legacy basename format; using it unfiltered.") - return(paths) - } - in_minor <- grepl(sprintf("^%s/", r_minor), paths) - if (sensitive_only) { - basename(paths[in_minor]) - } else { - # The primary pass writes the flat slot. Archive/ counts as present there: - # those are versions built and later superseded. - basename(paths[!grepl("^[0-9]+\\.[0-9]+/", paths)]) - } -} - -s3_cache <- select_cache_for_pass(s3_cache_paths, sensitive_only, r_minor) -sprintf( - "S3 cache: %s files (%s of %s objects apply to this pass)", - length(s3_cache), - length(s3_cache), - length(s3_cache_paths) -) +s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") +sprintf("S3 cache: %s files", length(s3_cache)) n <- nrow(chunk) mapply( diff --git a/local/build-newer-r.R b/local/build-newer-r.R deleted file mode 100644 index 3d7794f..0000000 --- a/local/build-newer-r.R +++ /dev/null @@ -1,72 +0,0 @@ -source(file.path("local", "newer-r-helpers.R")) -source(file.path("local", "rebuild-missing-helpers.R")) - -args <- commandArgs(trailingOnly = TRUE) -pos <- args[!startsWith(args, "--")] -if (!length(pos) %in% c(1L, 3L)) { - stop("Usage: build-newer-r.R PRIMARY_R [SPLIT_INTO SPLIT_INDEX] [--dry-run]") -} -primary_r <- pos[1L] -target_r <- as.character(getRversion()) -pkgs <- newer_r_candidates(tools::CRAN_package_db(), primary_r, target_r) -exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] -pkgs <- pkgs[!pkgs$Package %in% exclude, , drop = FALSE] -if (length(pos) == 3L) { - pkgs <- pkgs[ - shard_slice(seq_len(nrow(pkgs)), pos[2L], pos[3L]), - , - drop = FALSE - ] -} -message(sprintf( - "R %s: %d package(s) unsupported by primary R %s", - target_r, - nrow(pkgs), - primary_r -)) -if ("--dry-run" %in% args) { - print(pkgs, row.names = FALSE) -} else { - options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL) - for (i in seq_len(nrow(pkgs))) { - message(sprintf( - "[%d/%d] %s %s", - i, - nrow(pkgs), - pkgs$Package[i], - pkgs$Version[i] - )) - bincraft::build_binary_package( - pkgs$Package[i], - tag = pkgs$Version[i], - # Keep bincraft's normal ABI routing: safe binaries can be shared; - # sensitive ones use this interpreter's minor for presence and output. - is_r_minor_sensitive = bincraft::classify_r_minor_sensitive( - pkgs$Package[i], - pkgs$Version[i], - metadata_db_host = "r-binaries.devxy.io", - metadata_db_name = "build_metadata", - metadata_db_user = "rpkgs", - metadata_db_password = Sys.getenv("PGPASS"), - metadata_db_sslmode = "require", - metadata_db_port = 15432 - ), - s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", - s3_region = "eu-central-003", - s3_bucket = "devxy-rpkgs-binaries", - s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - metadata_db_host = "r-binaries.devxy.io", - metadata_db_name = "build_metadata", - metadata_db_table = "single_builds", - metadata_db_user = "rpkgs", - metadata_db_password = Sys.getenv("PGPASS"), - metadata_db_sslmode = "require", - metadata_db_port = 15432, - archive = TRUE, - patches = "local/patches", - upload = TRUE, - store_build_metadata = TRUE - ) - } -} diff --git a/local/check-slots.R b/local/check-slots.R deleted file mode 100644 index 5345da9..0000000 --- a/local/check-slots.R +++ /dev/null @@ -1,151 +0,0 @@ -### Fail when a pipeline targets a slot that is not in `local.rpkgs_slots`. -# -# Rscript local/check-slots.R -# -# That list feeds `UNION_SLOTS`, which `edge/rpkgs-router.ts` gates per-R-minor -# routing on. An unrouted slot serves the flat index to every client, so its -# binaries fail at dyn.load() under every R minor but one. -# -# One-directional: a published slot missing from some pipeline is only -# reported, since several pipelines cover a subset on purpose. - -options(error = function() { - cat("ERROR:", geterrmessage(), "\n", file = stdout()) - q(status = 1) -}) - -# Parses the distro list out of the setproduct's second array. -published_codenames <- function(path = "cdn.tf") { - if (!file.exists(path)) { - stop( - sprintf("%s not found; run this from the repository root.", path), - call. = FALSE - ) - } - lines <- readLines(path, warn = FALSE) - start <- grep("rpkgs_slots\\s*=", lines) - if (length(start) != 1L) { - stop( - "Could not find exactly one `rpkgs_slots` local in cdn.tf.", - call. = FALSE - ) - } - block <- paste(lines[start:min(start + 10L, length(lines))], collapse = "\n") - arrays <- regmatches(block, gregexpr("\\[[^]]*\\]", block))[[1L]] - if (length(arrays) < 2L) { - stop( - "`rpkgs_slots` no longer has the two-array setproduct shape this parses.", - call. = FALSE - ) - } - codenames <- gsub( - '"', - "", - regmatches(arrays[[2L]], gregexpr('"[^"]+"', arrays[[2L]]))[[1L]] - ) - if (length(codenames) == 0L) { - stop("Parsed an empty distro list from cdn.tf.", call. = FALSE) - } - codenames -} - -UBUNTU_CODENAMES <- c( - resolute = "ubuntu-2604", - noble = "ubuntu-2404", - jammy = "ubuntu-2204" -) - -as_pipeline_os <- function(codename) { - ifelse( - codename %in% names(UBUNTU_CODENAMES), - UBUNTU_CODENAMES[codename], - ifelse( - grepl("^rhel[0-9]+$", codename), - sub("^rhel", "redhat-", codename), - ifelse( - grepl("^alpine[0-9]+$", codename), - sub("^alpine([0-9])([0-9]+)$", "alpine-\\1\\2", codename), - NA_character_ - ) - ) - ) -} - -# Structural lines only, so a codename in a comment is not a finding. -slots_in_pipeline <- function(path) { - lines <- readLines(path, warn = FALSE) - row <- regmatches( - lines, - regexec("^\\s*-?\\s*(OS|CODENAME|OS_ID):\\s*(\\S+)\\s*$", lines) - ) - opt <- regmatches( - lines, - regexec("^\\s*-\\s*([a-z0-9]+-[a-z0-9]+)-(amd64|arm64)\\s*$", lines) - ) - hits <- c( - vapply( - row, - function(m) if (length(m) == 3L) m[[3L]] else NA_character_, - "" - ), - vapply(opt, function(m) if (length(m) == 4L) m[[2L]] else NA_character_, "") - ) - hits <- hits[!is.na(hits)] - # `OS: alpine` (build-all-versions' bare image name) is not a slot. - unique(hits[grepl("^(alpine|redhat|ubuntu)-?[0-9]", hits)]) -} - -codenames <- published_codenames() -published_os <- unname(as_pipeline_os(codenames)) -if (anyNA(published_os)) { - stop( - sprintf( - "Unrecognised codename(s) in cdn.tf: %s", - paste(codenames[is.na(published_os)], collapse = ", ") - ), - call. = FALSE - ) -} -# Files spell a slot either way, so accept both. -published <- unique(c(published_os, codenames)) - -cat(sprintf( - "Published slots (cdn.tf): %s\n\n", - paste(sort(published_os), collapse = ", ") -)) - -files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE)) -findings <- list() -for (f in files) { - stale <- setdiff(slots_in_pipeline(f), published) - if (length(stale) > 0L) { - findings[[f]] <- stale - cat(sprintf( - "%-45s targets unpublished: %s\n", - f, - paste(sort(stale), collapse = ", ") - )) - } -} - -uncovered <- setdiff( - published_os, - unique(unlist(lapply(files, slots_in_pipeline))) -) -if (length(uncovered) > 0L) { - cat(sprintf( - "\nNote: %s appear(s) in no pipeline at all.\n", - paste(sort(uncovered), collapse = ", ") - )) -} - -if (length(findings) == 0L) { - cat("All pipeline slots are published and routed.\n") - quit(status = 0) -} - -cat(sprintf( - "\n%d file(s) target an unrouted slot. Retire it with:\n just retire-slot \n", - length(findings) -)) -quit(status = 1) diff --git a/local/list-unsafe-flat-objects.R b/local/list-unsafe-flat-objects.R deleted file mode 100644 index 60649a2..0000000 --- a/local/list-unsafe-flat-objects.R +++ /dev/null @@ -1,161 +0,0 @@ -### Print the flat-slot objects recorded as unable to load under some R minor. -# -# Usage: Rscript local/list-unsafe-flat-objects.R -# -# Prints one ` ` per line, for a caller that rebuilds each -# under every installed R minor. -# -# The verdicts come from `abi_flat_safety`, written by the flat-safety backfill -# from the binaries themselves: an object is unsafe when it references a symbol -# some supported minor does not export. That is the authoritative list of what -# needs a per-minor build, and it replaces the ad-hoc symbol scans that produced -# earlier repair lists. - -options(error = function() { - cat("ERROR:", geterrmessage(), "\n", file = stdout()) - q(status = 1) -}) - -args <- commandArgs(trailingOnly = TRUE) -if (length(args) < 2L) { - stop("usage: list-unsafe-flat-objects.R ", call. = FALSE) -} -arch <- args[[1L]] -codename <- args[[2L]] - -con <- DBI::dbConnect( - RPostgres::Postgres(), - dbname = "build_metadata", - host = "r-binaries.devxy.io", - port = 15432L, - user = "rpkgs", - password = Sys.getenv("PGPASS"), - sslmode = "require" -) -on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE) - -rows <- DBI::dbGetQuery( - con, - paste0( - "SELECT package, version, unsupported FROM abi_flat_safety ", - "WHERE NOT safe AND codename = $1 AND arch = $2 ", - "ORDER BY package, version" - ), - params = list(codename, arch) -) - -# Drop what is already repaired. The verdict describes the *flat* object, and a -# per-minor build does not change it, so a resumed run would otherwise rebuild -# everything it did last time: ~2500 objects of duplicated compute after the -# first fleet run was interrupted. -# -# An object is repaired once a per-minor build exists for every minor that -# cannot load the flat one. The bucket is the source of truth for that, and it -# cannot be routed. -if (nrow(rows) > 0L) { - s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE - ) - contrib <- sprintf( - "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", - arch, - codename - ) - objects <- tryCatch( - s3fs::s3_dir_ls(contrib, recurse = TRUE), - error = function(e) character(0) - ) - if (length(objects) == 0L) { - stop( - sprintf( - "Could not list %s; refusing to guess what is already repaired.", - contrib - ), - call. = FALSE - ) - } - present <- sub(sprintf("^(s3://)?%s/", contrib), "", objects) - - # No recorded minors means unsafe without a reason, so require a build under - # every minor the image carries rather than assume one is enough. - installed <- sub( - "^([0-9]+[.][0-9]+).*$", - "\\1", - basename(list.dirs("/opt/R", recursive = FALSE)) - ) - installed <- unique(installed[grepl("^[0-9]+[.][0-9]+$", installed)]) - - repaired <- vapply( - seq_len(nrow(rows)), - function(i) { - u <- rows$unsupported[[i]] - minors <- if (is.na(u) || !nzchar(u)) { - installed - } else { - trimws(strsplit(u, ",", fixed = TRUE)[[1L]]) - } - if (length(minors) == 0L) { - return(FALSE) - } - wanted <- sprintf( - "%s/%s_%s.tar.gz", - minors, - rows$package[[i]], - rows$version[[i]] - ) - all(wanted %in% present) - }, - logical(1L) - ) - if (any(repaired)) { - message(sprintf( - "%s/%s: %d of %d already have per-minor builds for every minor that needs one; skipping those.", - arch, - codename, - sum(repaired), - nrow(rows) - )) - } - rows <- rows[!repaired, , drop = FALSE] -} - -# An empty result is ambiguous: either nothing is unsafe, or the backfill never -# ran for this slot. Say which, rather than let the caller read silence as done. -if (nrow(rows) == 0L) { - total <- DBI::dbGetQuery( - con, - "SELECT count(*) AS n FROM abi_flat_safety WHERE codename = $1 AND arch = $2", - params = list(codename, arch) - )$n[[1L]] - if (total == 0L) { - stop( - sprintf( - "No verdicts recorded for %s/%s at all. Run the flat-safety backfill first; an empty repair list here would look like success.", # nolint - arch, - codename - ), - call. = FALSE - ) - } - # `count(*)` arrives as a double (or integer64), and `%d` rejects both, so the - # nothing-to-do path used to die on its own success message. - message(sprintf( - "%s/%s: %s objects recorded, none unsafe. Nothing to repair.", - arch, - codename, - format(as.numeric(total), scientific = FALSE) - )) - quit(status = 0) -} - -message(sprintf( - "%s/%s: %d objects need a per-minor build", - arch, - codename, - nrow(rows) -)) -cat(sprintf("%s %s", rows$package, rows$version), sep = "\n") diff --git a/local/newer-r-helpers.R b/local/newer-r-helpers.R deleted file mode 100644 index 4b83204..0000000 --- a/local/newer-r-helpers.R +++ /dev/null @@ -1,75 +0,0 @@ -# Evaluate every R constraint, including patch-level and upper bounds. -r_requirement_satisfied <- function(depends, r_version) { - if (is.na(depends) || !nzchar(trimws(depends))) { - return(TRUE) - } - entries <- trimws(strsplit(depends, ",", fixed = TRUE)[[1L]]) - entries <- entries[grepl("^R(\\s|\\(|$)", entries)] - all(vapply( - entries, - function(entry) { - if (entry == "R") { - return(TRUE) - } - match <- regmatches( - entry, - regexec( - "^R\\s*\\(\\s*(>=|<=|==|>|<)\\s*([0-9]+(?:[.-][0-9]+)*)\\s*\\)$", - entry, - perl = TRUE - ) - )[[1L]] - if (length(match) != 3L) { - stop("Cannot parse R dependency: ", entry) - } - comparison <- utils::compareVersion(r_version, match[3L]) - switch( - match[2L], - ">=" = comparison >= 0L, - "<=" = comparison <= 0L, - "==" = comparison == 0L, - ">" = comparison > 0L, - "<" = comparison < 0L - ) - }, - logical(1L) - )) -} - -# Use unfiltered current CRAN metadata: available.packages() would hide exactly -# the packages that cannot run under the primary interpreter. -# Propagate R incompatibility through required dependencies, not Suggests. -newer_r_candidates <- function(db, primary_r, target_r) { - db <- as.data.frame(db, stringsAsFactors = FALSE) - # CRAN also lists R-specific Recommended copies (e.g. Matrix for R-devel). - # Prefer the ordinary contrib record, or a future-R copy can incorrectly - # block every package depending on Matrix on all released interpreters. - if ("Path" %in% names(db)) { - db <- db[order(!is.na(db$Path) & nzchar(db$Path)), , drop = FALSE] - } - db <- db[!duplicated(db$Package), , drop = FALSE] - deps <- tools::package_dependencies( - packages = db$Package, - db = as.matrix(db), - which = c("Depends", "Imports", "LinkingTo"), - recursive = FALSE - ) - supported <- function(version) { - ok <- vapply(db$Depends, r_requirement_satisfied, logical(1L), version) - repeat { - blocked <- db$Package[!ok] - next_ok <- ok & - !vapply(deps, function(x) any(x %in% blocked), logical(1L)) - if (identical(ok, next_ok)) { - return(ok) - } - ok <- next_ok - } - } - selected <- !supported(primary_r) & supported(target_r) - if ("OS_type" %in% names(db)) { - selected <- selected & (is.na(db$OS_type) | db$OS_type != "windows") - } - result <- db[selected, c("Package", "Version"), drop = FALSE] - result[order(result$Package), , drop = FALSE] -} diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 31b667f..3b508fb 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -12,11 +12,11 @@ library(progressr, quietly = TRUE) suppressPackageStartupMessages(library(data.table)) # Sys.setenv("OS" = "alpine") -# Sys.setenv("OS_VERSION" = "3.23") +# Sys.setenv("OS_VERSION" = "3.22") # Sys.setenv("ARCH" = "arm64") arch <- Sys.getenv("ARCH") -# target: alpine-323, ubuntu-2404, redhat-9, etc. +# target: alpine-322, ubuntu-2404, redhat-9, etc. platform <- paste( Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), @@ -68,36 +68,9 @@ archive_versions <- archive_versions[ ] # Now get release versions (assuming cran_release has Package and Version columns) -# -# Packages published in the last few days are held back. `check_for_binary()` -# reads the published version from the `cran` GitHub mirror -# (`GET /repos/cran//commits`), and that mirror lags CRAN: a package that -# has just appeared has no repository there yet. The call then 404s, which is -# permanent, but it is wrapped in `purrr::insistently` and retried ten times -# with a backoff capped at 60s - so one unmirrored package burns about five -# minutes and then aborts the whole shard. -# -# Holding them back costs nothing: the daily update pipeline builds new and -# updated packages anyway, and they arrive here on the next run once the mirror -# has caught up. -mirror_lag_days <- as.numeric( - Sys.getenv("CRAN_MIRROR_LAG_DAYS", unset = "3") -) -published <- as.POSIXct(cran_release$Published, tz = "UTC") -too_recent <- !is.na(published) & - published > (Sys.time() - mirror_lag_days * 86400) -if (any(too_recent)) { - message(sprintf( - "Holding back %d package(s) published in the last %g day(s); the cran GitHub mirror will not have them yet: %s", - sum(too_recent), - mirror_lag_days, - paste(utils::head(cran_release$Package[too_recent], 10L), collapse = ", ") - )) -} - release_versions <- data.table( - Package = cran_release$Package[!too_recent], - Version = as.character(cran_release$Version[!too_recent]) + Package = cran_release$Package, + Version = as.character(cran_release$Version) ) pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE)) @@ -116,28 +89,7 @@ s3_pkgs <- s3fs::s3_dir_ls( recurse = TRUE ) -# `recurse = TRUE` walks the per-minor slots as well, and `basename()` throws -# the directory away - so `4.5/curl_1.0.tar.gz` and `curl_1.0.tar.gz` collapse -# to one name and a package present under *any* R minor counts as built for -# *all* of them. The candidate list then prunes exactly the packages a -# per-minor pass exists to build: arm64/alpine324 reported "0 remaining" for -# both 4.4 and 4.6 while its indexes were dropping 2400+ packages as missing. -# -# Per-minor objects are therefore excluded here. Presence in a specific minor -# is decided downstream, where the running R version is known: build-all.R -# filters on it, and `build_binary_package()` checks the per-minor path per -# package and skips what is already there. -# -# Archive/ is kept. Those are versions that were built and then superseded; -# dropping them would make every archived version look unbuilt. -per_minor_object <- grepl("/[0-9]+\\.[0-9]+/[^/]+$", s3_pkgs) -if (any(per_minor_object)) { - cat(sprintf( - "Excluding %d per-minor object(s) from the presence check; those are decided per pass\n", - sum(per_minor_object) - )) -} -file_names <- basename(s3_pkgs[!per_minor_object]) +file_names <- basename(s3_pkgs) # An object occupying a key is not proof a binary was built: a package whose # build failed has its CRAN source published under exactly that name. Left in @@ -154,140 +106,28 @@ index_url <- sprintf( arch, codename ) -read_source_served <- function(index) { - tryCatch( - { - con_idx <- gzcon(url(index, open = "rb")) - on.exit(close(con_idx), add = TRUE) - idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) - sprintf( - "%s_%s.tar.gz", - idx[is.na(idx[, "Built"]), "Package"], - idx[is.na(idx[, "Built"]), "Version"] - ) - }, - error = function(e) { - cat(sprintf( - "WARNING: could not read %s (%s); keeping that index's cache in full\n", - index, - conditionMessage(e) - )) - character(0) - } - ) -} - -source_served <- read_source_served(index_url) +source_served <- tryCatch( + { + con_idx <- gzcon(url(index_url, open = "rb")) + on.exit(close(con_idx), add = TRUE) + idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) + sprintf( + "%s_%s.tar.gz", + idx[is.na(idx[, "Built"]), "Package"], + idx[is.na(idx[, "Built"]), "Version"] + ) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read %s (%s); keeping the full S3 cache\n", + index_url, + conditionMessage(e) + )) + character(0) + } +) binary_cache <- setdiff(file_names, source_served) - -# The existence cache and the candidate list need different views of the same -# listing, and conflating them is what made this wrong in both directions. -# -# The candidate list must ignore per-minor objects, or a package present under -# one minor prunes itself from every other minor's work (#189). The existence -# cache must NOT ignore them, or `build_binary_package()` is told nothing is -# present under any minor and recompiles the lot: amd64/resolute recompiled -# 8683 packages it had already built, reporting "already exists in S3" three -# times. -# -# So the cache keeps the path relative to the slot, and `build-all.R` selects -# the part that matches the pass it is running: the flat slot for the primary, -# `/` for a per-minor pass. -# -# `s3_dir_ls()` returns keys with the `s3://` scheme attached, so stripping the -# prefix as a fixed substring takes it out of the middle and leaves -# `s3://4.4/curl_1.0.tar.gz`. That leading scheme defeats the `^/` test -# downstream, so every per-minor object is read as a flat-slot object: the -# sensitive passes see an empty cache and recompile everything they already -# have. Anchor the pattern and swallow the scheme with it. -contrib_prefix <- sprintf( - "^(s3://)?devxy-rpkgs-binaries/%s/%s/latest/src/contrib/", - arch, - codename -) -relative_paths <- sub(contrib_prefix, "", s3_pkgs) - -# The strip is load-bearing and fails silently, so assert it. Both counts are -# derived from the same listing and use the same shape of pattern, so they must -# agree exactly; a mismatch means the prefix no longer describes the keys. -stripped_per_minor <- grepl("^[0-9]+\\.[0-9]+/[^/]+$", relative_paths) -if (sum(stripped_per_minor) != sum(per_minor_object)) { - stop(sprintf( - paste0( - "S3 prefix strip failed: %d per-minor objects in the listing, %d after ", - "stripping /%s/. Example key: %s" - ), - sum(per_minor_object), - sum(stripped_per_minor), - contrib_prefix, - if (any(per_minor_object)) { - s3_pkgs[which(per_minor_object)[1L]] - } else { - "" - } - )) -} -source_basenames <- source_served - -# A source-fallback list describes ONE index, so it may only be applied to that -# index's objects. Applied to every path it deletes real per-minor binaries from -# the cache: a package the flat slot serves as CRAN source still has a genuine -# `4.4/.tar.gz`, and dropping that name makes every per-minor pass rebuild -# it, every run, forever. -# -# The overlap is total rather than partial, which is why this pinned the skip -# rate near 0%: the sensitive candidate list is exactly "sensitive packages with -# no flat binary", which is the same set this was removing. amd64/resolute -# listed 21212 per-minor objects but cached only 13572, then recompiled osmdata, -# osqp and outbreaker2 while their 4.4 binaries sat in the bucket. -# -# So judge each minor by its own index, and the flat slot by the flat index. -is_per_minor_path <- grepl("^[0-9]+\\.[0-9]+/", relative_paths) -path_minor <- ifelse( - is_per_minor_path, - sub("^([0-9]+\\.[0-9]+)/.*$", "\\1", relative_paths), - "" -) -minors_present <- sort(unique(path_minor[is_per_minor_path])) -per_minor_source <- lapply(minors_present, function(m) { - read_source_served(sprintf( - "https://cran.rpkgs.com/%s/%s/latest/src/contrib/%s/PACKAGES.gz", - arch, - codename, - m - )) -}) -names(per_minor_source) <- minors_present -cat(sprintf( - "Source fallbacks per index: flat=%d%s\n", - length(source_basenames), - if (length(minors_present)) { - paste0( - ", ", - paste( - sprintf("%s=%d", minors_present, lengths(per_minor_source)), - collapse = ", " - ) - ) - } else { - "" - } -)) - -served_as_source <- vapply( - seq_along(relative_paths), - function(i) { - scope <- if (nzchar(path_minor[i])) { - per_minor_source[[path_minor[i]]] - } else { - source_basenames - } - basename(relative_paths[i]) %in% scope - }, - logical(1L) -) -existence_cache <- relative_paths[!served_as_source] cat(sprintf( "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", length(file_names), @@ -298,13 +138,7 @@ cat(sprintf( # Save the S3 file listing for the build step to use as s3_package_cache. # This avoids loading s3fs/reticulate in the build container, saving memory for # the dependency-installer subprocesses -# Mark the format explicitly. `build-all.R` has to tell a slot-relative cache -# from a pre-#191 basename one, and sniffing for a "/" cannot: a new-format -# cache for a slot with no per-minor or Archive objects holds bare names too, -# and would be read as legacy and used unfiltered, which makes a per-minor pass -# believe the flat slot's binaries are its own and build nothing. -attr(existence_cache, "slot_relative") <- TRUE -saveRDS(existence_cache, "/mnt/cache/packages/s3_cache.rds") +saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") # Built from the filtered listing, not the raw one: `s3_dt` is subtracted from # the build list below, so a source fallback left in here would exclude the very # package that needs building. @@ -318,22 +152,11 @@ s3_dt <- data.table( ### Get all packages with build errors -# Scoped to the R minor this snapshot is computed under. A failure is a fact -# about one interpreter: without the scope a package that failed under the -# primary minor is dropped from the candidate list for every other minor too, -# which is the same omission fixed in local/build-all.R and in bincraft's -# check_package_error(). -snapshot_r_minor <- paste( - R.version$major, - strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], - sep = "." -) sql_query <- paste0( # nolint "SELECT error_occurred FROM ", "single_builds", - " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4", - " AND substring(r_version from '^[0-9]+[.][0-9]+') = $5" + " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4" ) # Function to query for a single package-version query_error <- function(pkg, ver) { @@ -341,7 +164,7 @@ query_error <- function(pkg, ver) { ~ DBI::dbGetQuery( con, sql_query, - params = list(pkg, ver, platform, arch, snapshot_r_minor) + params = list(pkg, ver, platform, arch) ), rate = purrr::rate_backoff( pause_base = 1L, @@ -355,30 +178,12 @@ query_error <- function(pkg, ver) { } # Fetch all relevant columns from the database -# Scoped to the snapshot's own R minor, like `check_package_error()` in -# bincraft (#119) and the per-minor error query (#189). Unscoped, a single -# failure under any one minor removed the package from the candidate list for -# every minor, permanently: the row is keyed by (name, tag, platform, arch) -# only, so an error under 4.6 also cancelled its 4.4 and 4.5 builds. -# -# That is what the residual per-minor gaps were. On amd64/alpine324, 505 of the -# 508 ABI-risky packages missing from 4.4 were also missing from 4.6 -- a 99% -# overlap, which per-minor build failures would not produce. None of them were -# even attempted by the run that had just succeeded. -# -# Errors specific to another minor are still respected downstream: -# `filter_packages_with_errors()` runs per pass with that pass's r_version. errored_pkgs <- DBI::dbGetQuery( con, sprintf( - paste0( - "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE ", - "and platform='%s' and arch='%s' ", - "and substring(r_version from '^[0-9]+[.][0-9]+') = '%s'" - ), + "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", platform, - arch, - snapshot_r_minor + arch ) ) errored_pkgs <- as.data.table(errored_pkgs) diff --git a/local/patches/RcppParallel/force-bundled-tbb-6.0.0.patch b/local/patches/RcppParallel/force-bundled-tbb-6.0.0.patch deleted file mode 100644 index a7f91dc..0000000 --- a/local/patches/RcppParallel/force-bundled-tbb-6.0.0.patch +++ /dev/null @@ -1,152 +0,0 @@ -diff --git a/R/aaa.R b/R/aaa.R -index 568a2aa..2744059 100644 ---- a/R/aaa.R -+++ b/R/aaa.R -@@ -5,4 +5,21 @@ TBB_LIB <- "" - TBB_INC <- "" - - TBB_NAME <- "tbb" --TBB_MALLOC_NAME <- "tbbmalloc" -\ No newline at end of file -+TBB_MALLOC_NAME <- "tbbmalloc" -+ -+# bincraft patch: build images (ours did until build-env-images c51ed09) and -+# plenty of user environments export TBB_ROOT / TBB_LIB / TBB_INC. A binary we -+# publish always carries its own oneTBB in RcppParallel/lib -- see the -+# companion changes in tools/config/configure.R and src/install.libs.R -- so -+# honouring those variables at run time is actively harmful: .onLoad() would -+# dyn.load() a second, unrelated TBB into the process next to the bundled one -+# (two copies of the same symbols in the global scope, which segfaults R on -+# load), and RcppParallelLibs() / CxxFlags() would hand that system TBB to -+# dependents such as rstan, putting NEEDED libtbb.so.12 back into their -+# binaries. Read the variables only when explicitly opted back in. -+bincraftGetenv <- function(name, unset = "") { -+ if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") -+ Sys.getenv(name, unset = unset) -+ else -+ unset -+} -\ No newline at end of file -diff --git a/R/tbb.R b/R/tbb.R -index 3d8fbb3..9560165 100644 ---- a/R/tbb.R -+++ b/R/tbb.R -@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { - sysname <- Sys.info()[["sysname"]] - - # find root for TBB install -- tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) -+ tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) - if (is.null(name)) - return(tbbRoot) - -@@ -60,7 +60,7 @@ tbbCxxFlags <- function() { - flags <- c("-DRCPP_PARALLEL_USE_TBB=1") - - # if TBB_INC is set, apply those library paths -- tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) -+ tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) - if (!file.exists(tbbInc)) { - tbbInc <- system.file("include", package = "RcppParallel") - } -@@ -100,7 +100,7 @@ tbbLdFlags <- function() { - } - - # shortcut if TBB_LIB defined -- tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) -+ tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) - if (nzchar(tbbLib)) { - if (R.version$os == "emscripten") { - fmt <- "-L%1$s -l%2$s" -diff --git a/src/install.libs.R b/src/install.libs.R -index 4dcbac2..57dbc74 100644 ---- a/src/install.libs.R -+++ b/src/install.libs.R -@@ -258,6 +258,18 @@ prependFlags <- function(prependFlags, toFlags) { - tbbLib <- Sys.getenv("TBB_LIB") - tbbInc <- Sys.getenv("TBB_INC") - -+# bincraft patch: the companion change in tools/config/configure.R stops an -+# ambient TBB_LIB / TBB_INC from selecting a system TBB, but this script is -+# also run directly by `R CMD INSTALL` (not only through the `tbb` rule in -+# src/Makevars, which passes the configured values), so at install time it -+# still sees the image's environment and would symlink the system libraries -+# into RcppParallel/lib. Drop them here for the same reason, under the same -+# opt-out. -+if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { -+ tbbLib <- "" -+ tbbInc <- "" -+} -+ - args <- commandArgs(trailingOnly = TRUE) - if (identical(args, "build")) { - if (nzchar(tbbLib) && nzchar(tbbInc)) { -diff --git a/tools/config/configure.R b/tools/config/configure.R -index a001e63..bfe8c8e 100644 ---- a/tools/config/configure.R -+++ b/tools/config/configure.R -@@ -144,6 +144,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) - tbbLib <- Sys.getenv("TBB_LIB", unset = NA) - tbbInc <- Sys.getenv("TBB_INC", unset = NA) - -+# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Our build -+# images exported these until build-env-images c51ed09 (a leftover from -+# RcppParallel 5.x, whose bundled Intel TBB would not build on musl or with -+# modern g++), and any of them switches the branches below to a system TBB. -+# The published binary then records NEEDED libtbb.so.12 (or libtbb.so.2 for -+# the classic Intel TBB) and gets a RcppParallel/lib full of absolute symlinks -+# into the image's library dir, so it cannot dyn.load on a consumer machine -+# without that exact TBB. 6.x bundles oneTBB 2022 and builds it with cmake on -+# every platform we ship, so the bundled copy is always the right choice here; -+# forcing it in the package rather than relying on the image environment keeps -+# the binary correct whichever image version CI happens to pull. Set -+# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. -+if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { -+ tbbRoot <- NA -+ tbbLib <- NA -+ tbbInc <- NA -+} -+ - tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") - tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") - -@@ -224,10 +242,21 @@ define( - ) - - # set PKG_LIBS -+# -+# bincraft patch: the library directories below are passed as plain '-L', not -+# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) -+# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', -+# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build -+# host with a distro TBB installed, '-ltbb' resolves to that library and -+# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the -+# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads -+# the system TBB rather than the copy shipped in RcppParallel/lib, and fails -+# outright on a machine that has no system TBB. gcc places a plain '-L' before -+# its built-in dirs, so the intended library is found first. - pkgLibs <- if (!is.na(tbbLib)) { - - c( -- "-Wl,-L\"$(TBB_LIB)\"", -+ "-L\"$(TBB_LIB)\"", - sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), - "-l$(TBB_NAME)", - "-l$(TBB_MALLOC_NAME)" -@@ -240,14 +269,14 @@ pkgLibs <- if (!is.na(tbbLib)) { - } else if (R.version$os == "emscripten") { - - c( -- "-Wl,-Ltbb/build/lib_release", -+ "-Ltbb/build/lib_release", - "-l$(TBB_NAME)" - ) - - } else { - - c( -- "-Wl,-Ltbb/build/lib_release", -+ "-Ltbb/build/lib_release", - "-l$(TBB_NAME)", - "-l$(TBB_MALLOC_NAME)" - ) diff --git a/local/patches/RcppParallel/force-bundled-tbb-6.1.x.patch b/local/patches/RcppParallel/force-bundled-tbb-6.1.x.patch deleted file mode 100644 index 948643c..0000000 --- a/local/patches/RcppParallel/force-bundled-tbb-6.1.x.patch +++ /dev/null @@ -1,157 +0,0 @@ -diff --git a/R/aaa.R b/R/aaa.R -index 568a2aa..2744059 100644 ---- a/R/aaa.R -+++ b/R/aaa.R -@@ -5,4 +5,21 @@ TBB_LIB <- "" - TBB_INC <- "" - - TBB_NAME <- "tbb" --TBB_MALLOC_NAME <- "tbbmalloc" -\ No newline at end of file -+TBB_MALLOC_NAME <- "tbbmalloc" -+ -+# bincraft patch: build images (ours did until build-env-images c51ed09) and -+# plenty of user environments export TBB_ROOT / TBB_LIB / TBB_INC. A binary we -+# publish always carries its own oneTBB in RcppParallel/lib -- see the -+# companion changes in tools/config/configure.R and src/install.libs.R -- so -+# honouring those variables at run time is actively harmful: .onLoad() would -+# dyn.load() a second, unrelated TBB into the process next to the bundled one -+# (two copies of the same symbols in the global scope, which segfaults R on -+# load), and RcppParallelLibs() / CxxFlags() would hand that system TBB to -+# dependents such as rstan, putting NEEDED libtbb.so.12 back into their -+# binaries. Read the variables only when explicitly opted back in. -+bincraftGetenv <- function(name, unset = "") { -+ if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") -+ Sys.getenv(name, unset = unset) -+ else -+ unset -+} -\ No newline at end of file -diff --git a/R/tbb.R b/R/tbb.R -index 152fd16..4b06aa5 100644 ---- a/R/tbb.R -+++ b/R/tbb.R -@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { - sysname <- Sys.info()[["sysname"]] - - # find root for TBB install -- tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) -+ tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) - if (is.null(name)) - return(tbbRoot) - -@@ -58,7 +58,7 @@ tbbCxxFlags <- function() { - flags <- c("-DRCPP_PARALLEL_USE_TBB=1") - - # if TBB_INC is set, apply those library paths -- tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) -+ tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) - if (!file.exists(tbbInc)) { - tbbInc <- system.file("include", package = "RcppParallel") - } -@@ -96,7 +96,7 @@ tbbLdFlags <- function() { - } - - # shortcut if TBB_LIB defined -- tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) -+ tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) - if (nzchar(tbbLib)) { - if (R.version$os == "emscripten") { - fmt <- "-L%1$s -l%2$s" -diff --git a/src/install.libs.R b/src/install.libs.R -index e770838..1dd2fc7 100644 ---- a/src/install.libs.R -+++ b/src/install.libs.R -@@ -492,6 +492,18 @@ prependFlags <- function(prependFlags, toFlags) { - tbbLib <- Sys.getenv("TBB_LIB") - tbbInc <- Sys.getenv("TBB_INC") - -+# bincraft patch: the companion change in tools/config/configure.R stops an -+# ambient TBB_LIB / TBB_INC from selecting a system TBB, but this script is -+# also run directly by `R CMD INSTALL` (not only through the `tbb` rule in -+# src/Makevars, which passes the configured values), so at install time it -+# still sees the image's environment and would symlink the system libraries -+# into RcppParallel/lib. Drop them here for the same reason, under the same -+# opt-out. -+if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { -+ tbbLib <- "" -+ tbbInc <- "" -+} -+ - args <- commandArgs(trailingOnly = TRUE) - if (identical(args, "build")) { - if (nzchar(tbbLib) && nzchar(tbbInc)) { -diff --git a/tools/config/configure.R b/tools/config/configure.R -index c3842f9..adf7876 100644 ---- a/tools/config/configure.R -+++ b/tools/config/configure.R -@@ -66,6 +66,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) - tbbLib <- Sys.getenv("TBB_LIB", unset = NA) - tbbInc <- Sys.getenv("TBB_INC", unset = NA) - -+# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Our build -+# images exported these until build-env-images c51ed09 (a leftover from -+# RcppParallel 5.x, whose bundled Intel TBB would not build on musl or with -+# modern g++), and any of them switches the branches below to a system TBB. -+# The published binary then records NEEDED libtbb.so.12 (or libtbb.so.2 for -+# the classic Intel TBB) and gets a RcppParallel/lib full of absolute symlinks -+# into the image's library dir, so it cannot dyn.load on a consumer machine -+# without that exact TBB. 6.x bundles oneTBB 2022 and builds it with cmake on -+# every platform we ship, so the bundled copy is always the right choice here; -+# forcing it in the package rather than relying on the image environment keeps -+# the binary correct whichever image version CI happens to pull. Set -+# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. -+if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { -+ tbbRoot <- NA -+ tbbLib <- NA -+ tbbInc <- NA -+} -+ - tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") - tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") - -@@ -146,6 +164,18 @@ define( - ) - - # set PKG_LIBS -+# -+# bincraft patch: the non-Windows library directories below are passed as -+# plain '-L', not '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, -+# /usr/lib/) into explicit '-L' options ahead of anything forwarded -+# verbatim with '-Wl,', so with '-Wl,-L' a system libtbb.so wins over the one -+# named here: on a build host with a distro TBB installed, '-ltbb' resolves to -+# that library and RcppParallel.so records its SONAME (libtbb.so.12, or -+# libtbb.so.2 for the classic Intel TBB) instead of the bundled 'libtbb.so'. -+# The binary then loads the system TBB rather than the copy shipped in -+# RcppParallel/lib, and fails outright on a machine that has no system TBB. -+# gcc places a plain '-L' before its built-in dirs, so the intended library is -+# found first. - pkgLibs <- if (.Platform$OS.type == "windows") { - - if (!is.na(tbbLib) && file.exists(file.path(tbbInc, "oneapi"))) { -@@ -183,7 +213,7 @@ pkgLibs <- if (.Platform$OS.type == "windows") { - } else if (!is.na(tbbLib)) { - - c( -- "-Wl,-L\"$(TBB_LIB)\"", -+ "-L\"$(TBB_LIB)\"", - sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), - "-l$(TBB_NAME)", - "-l$(TBB_MALLOC_NAME)" -@@ -192,14 +222,14 @@ pkgLibs <- if (.Platform$OS.type == "windows") { - } else if (R.version$os == "emscripten") { - - c( -- "-Wl,-Ltbb/build/lib_release", -+ "-Ltbb/build/lib_release", - "-l$(TBB_NAME)" - ) - - } else { - - c( -- "-Wl,-Ltbb/build/lib_release", -+ "-Ltbb/build/lib_release", - "-l$(TBB_NAME)", - "-l$(TBB_MALLOC_NAME)" - ) diff --git a/local/patches/RcppParallel/force-bundled-tbb-6.2.x.patch b/local/patches/RcppParallel/force-bundled-tbb.patch similarity index 66% rename from local/patches/RcppParallel/force-bundled-tbb-6.2.x.patch rename to local/patches/RcppParallel/force-bundled-tbb.patch index d88cfc5..c46a05e 100644 --- a/local/patches/RcppParallel/force-bundled-tbb-6.2.x.patch +++ b/local/patches/RcppParallel/force-bundled-tbb.patch @@ -1,5 +1,5 @@ diff --git a/R/aaa.R b/R/aaa.R -index 568a2aa..2744059 100644 +index 568a2aa..bfdab4c 100644 --- a/R/aaa.R +++ b/R/aaa.R @@ -5,4 +5,21 @@ TBB_LIB <- "" @@ -10,16 +10,16 @@ index 568a2aa..2744059 100644 \ No newline at end of file +TBB_MALLOC_NAME <- "tbbmalloc" + -+# bincraft patch: build images (ours did until build-env-images c51ed09) and -+# plenty of user environments export TBB_ROOT / TBB_LIB / TBB_INC. A binary we -+# publish always carries its own oneTBB in RcppParallel/lib -- see the -+# companion changes in tools/config/configure.R and src/install.libs.R -- so -+# honouring those variables at run time is actively harmful: .onLoad() would -+# dyn.load() a second, unrelated TBB into the process next to the bundled one -+# (two copies of the same symbols in the global scope, which segfaults R on -+# load), and RcppParallelLibs() / CxxFlags() would hand that system TBB to -+# dependents such as rstan, putting NEEDED libtbb.so.12 back into their -+# binaries. Read the variables only when explicitly opted back in. ++# bincraft patch: our build images (and plenty of user environments) export ++# TBB_ROOT / TBB_LIB / TBB_INC. A binary we publish always carries its own ++# oneTBB in RcppParallel/lib -- see the companion changes in ++# tools/config/configure.R and src/install.libs.R -- so honouring those ++# variables at run time is actively harmful: .onLoad() would dyn.load() a ++# second, unrelated TBB into the process next to the bundled one (two copies ++# of the same symbols in the global scope, which segfaults R on load), and ++# RcppParallelLibs() / CxxFlags() would hand that system TBB to dependents ++# such as rstan, putting NEEDED libtbb.so.12 back into their binaries. Read ++# the variables only when explicitly opted back in. +bincraftGetenv <- function(name, unset = "") { + if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") + Sys.getenv(name, unset = unset) @@ -28,24 +28,36 @@ index 568a2aa..2744059 100644 +} \ No newline at end of file diff --git a/R/tbb.R b/R/tbb.R -index b071833..e3bf7a5 100644 +index 6f6a745..e407986 100644 --- a/R/tbb.R +++ b/R/tbb.R -@@ -19,3 +19,3 @@ tbbLibraryPath <- function(name = NULL) { +@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { + sysname <- Sys.info()[["sysname"]] + # find root for TBB install - tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) + tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) if (is.null(name)) -@@ -63,3 +63,3 @@ tbbCxxFlags <- function() { + return(tbbRoot) + +@@ -58,7 +58,7 @@ tbbCxxFlags <- function() { + flags <- c("-DRCPP_PARALLEL_USE_TBB=1") + # if TBB_INC is set, apply those library paths - tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) + tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) if (!file.exists(tbbInc)) { -@@ -122,3 +122,3 @@ tbbLdFlags <- function() { + tbbInc <- system.file("include", package = "RcppParallel") + } +@@ -117,7 +117,7 @@ tbbLdFlags <- function() { + } + # shortcut if TBB_LIB defined - tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) + tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) if (nzchar(tbbLib)) { + if (R.version$os == "emscripten") { + fmt <- "-L%1$s -l%2$s" diff --git a/src/install.libs.R b/src/install.libs.R index 3b3cfda..c0e6f3e 100644 --- a/src/install.libs.R @@ -70,24 +82,24 @@ index 3b3cfda..c0e6f3e 100644 if (identical(args, "build")) { if (nzchar(tbbLib) && nzchar(tbbInc)) { diff --git a/tools/config/configure.R b/tools/config/configure.R -index c203a1f..0050b5c 100644 +index 6293fe1..eae4aaa 100644 --- a/tools/config/configure.R +++ b/tools/config/configure.R -@@ -79,6 +79,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) +@@ -40,6 +40,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) tbbLib <- Sys.getenv("TBB_LIB", unset = NA) tbbInc <- Sys.getenv("TBB_INC", unset = NA) -+# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Our build -+# images exported these until build-env-images c51ed09 (a leftover from -+# RcppParallel 5.x, whose bundled Intel TBB would not build on musl or with -+# modern g++), and any of them switches the branches below to a system TBB. -+# The published binary then records NEEDED libtbb.so.12 (or libtbb.so.2 for -+# the classic Intel TBB) and gets a RcppParallel/lib full of absolute symlinks -+# into the image's library dir, so it cannot dyn.load on a consumer machine -+# without that exact TBB. 6.x bundles oneTBB 2022 and builds it with cmake on -+# every platform we ship, so the bundled copy is always the right choice here; -+# forcing it in the package rather than relying on the image environment keeps -+# the binary correct whichever image version CI happens to pull. Set ++# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Several of ++# our build images export these (a leftover from RcppParallel 5.x, whose ++# bundled Intel TBB would not build on musl or with modern g++), and any of ++# them switches the branches below to a system TBB. The published binary then ++# records NEEDED libtbb.so.12 (or libtbb.so.2 for the classic Intel TBB) and ++# gets a RcppParallel/lib full of absolute symlinks into the image's library ++# dir, so it cannot dyn.load on a consumer machine without that exact TBB. ++# 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, ++# so the bundled copy is always the right choice here; forcing it in the ++# package rather than relying on the image environment keeps the binary ++# correct whichever image version CI happens to pull. Set +# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. +if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { + tbbRoot <- NA @@ -98,7 +110,7 @@ index c203a1f..0050b5c 100644 tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") -@@ -225,12 +243,23 @@ define( +@@ -186,12 +204,23 @@ define( ) # set PKG_LIBS @@ -123,7 +135,7 @@ index c203a1f..0050b5c 100644 if (.Platform$OS.type != "windows") sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), "-l$(TBB_NAME)", -@@ -240,14 +269,14 @@ pkgLibs <- if (!is.na(tbbLib)) { +@@ -201,14 +230,14 @@ pkgLibs <- if (!is.na(tbbLib)) { } else if (R.version$os == "emscripten") { c( diff --git a/local/patches/registry.json b/local/patches/registry.json index 1f341b7..a7e29dd 100644 --- a/local/patches/registry.json +++ b/local/patches/registry.json @@ -1,53 +1,13 @@ [ { "package": "RcppParallel", - "versions": "6.2.1", + "versions": ">=6.0.0", "platforms": ["*"], "env": {}, "configure_args": [], "makevars": {}, - "patch": "RcppParallel/force-bundled-tbb-6.2.x.patch", - "reason": "RcppParallel 6.x builds its bundled oneTBB 2022 with cmake, but tools/config/configure.R passes the library dir as '-Wl,-L...', and gcc places its own search dirs (/usr/lib64, /usr/lib/) ahead of anything forwarded with '-Wl,'. Every build image installs a distro libtbb, so '-ltbb' resolves to it: the binary records NEEDED libtbb.so.12 (libtbb.so.2 on el8/el9) instead of the bundled libtbb.so and cannot load on a machine without that exact system TBB. The patch passes a plain '-L' so the bundled build dir wins. As a safeguard it also makes configure.R, src/install.libs.R and R/tbb.R ignore an ambient TBB_ROOT/TBB_LIB/TBB_INC (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE): the build images stopped exporting them in build-env-images c51ed09, but a stale image or ~/.Renviron would otherwise switch the build back to the system TBB, and R reads ~/.Renviron after the process environment, so no pipeline env can override it. The read sites move between releases, so each entry pins a patch generated against that version. Upstream still uses '-Wl,-L' in 6.2.1; drop these entries once a release passes a plain '-L'." - }, - { - "package": "RcppParallel", - "versions": "6.2.0", - "platforms": ["*"], - "env": {}, - "configure_args": [], - "makevars": {}, - "patch": "RcppParallel/force-bundled-tbb-6.2.x.patch", - "reason": "RcppParallel 6.x builds its bundled oneTBB 2022 with cmake, but tools/config/configure.R passes the library dir as '-Wl,-L...', and gcc places its own search dirs (/usr/lib64, /usr/lib/) ahead of anything forwarded with '-Wl,'. Every build image installs a distro libtbb, so '-ltbb' resolves to it: the binary records NEEDED libtbb.so.12 (libtbb.so.2 on el8/el9) instead of the bundled libtbb.so and cannot load on a machine without that exact system TBB. The patch passes a plain '-L' so the bundled build dir wins. As a safeguard it also makes configure.R, src/install.libs.R and R/tbb.R ignore an ambient TBB_ROOT/TBB_LIB/TBB_INC (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE): the build images stopped exporting them in build-env-images c51ed09, but a stale image or ~/.Renviron would otherwise switch the build back to the system TBB, and R reads ~/.Renviron after the process environment, so no pipeline env can override it. The read sites move between releases, so each entry pins a patch generated against that version. Upstream still uses '-Wl,-L' in 6.2.1; drop these entries once a release passes a plain '-L'." - }, - { - "package": "RcppParallel", - "versions": "6.1.1", - "platforms": ["*"], - "env": {}, - "configure_args": [], - "makevars": {}, - "patch": "RcppParallel/force-bundled-tbb-6.1.x.patch", - "reason": "RcppParallel 6.x builds its bundled oneTBB 2022 with cmake, but tools/config/configure.R passes the library dir as '-Wl,-L...', and gcc places its own search dirs (/usr/lib64, /usr/lib/) ahead of anything forwarded with '-Wl,'. Every build image installs a distro libtbb, so '-ltbb' resolves to it: the binary records NEEDED libtbb.so.12 (libtbb.so.2 on el8/el9) instead of the bundled libtbb.so and cannot load on a machine without that exact system TBB. The patch passes a plain '-L' so the bundled build dir wins. As a safeguard it also makes configure.R, src/install.libs.R and R/tbb.R ignore an ambient TBB_ROOT/TBB_LIB/TBB_INC (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE): the build images stopped exporting them in build-env-images c51ed09, but a stale image or ~/.Renviron would otherwise switch the build back to the system TBB, and R reads ~/.Renviron after the process environment, so no pipeline env can override it. The read sites move between releases, so each entry pins a patch generated against that version. Upstream still uses '-Wl,-L' in 6.2.1; drop these entries once a release passes a plain '-L'." - }, - { - "package": "RcppParallel", - "versions": "6.1.0", - "platforms": ["*"], - "env": {}, - "configure_args": [], - "makevars": {}, - "patch": "RcppParallel/force-bundled-tbb-6.1.x.patch", - "reason": "RcppParallel 6.x builds its bundled oneTBB 2022 with cmake, but tools/config/configure.R passes the library dir as '-Wl,-L...', and gcc places its own search dirs (/usr/lib64, /usr/lib/) ahead of anything forwarded with '-Wl,'. Every build image installs a distro libtbb, so '-ltbb' resolves to it: the binary records NEEDED libtbb.so.12 (libtbb.so.2 on el8/el9) instead of the bundled libtbb.so and cannot load on a machine without that exact system TBB. The patch passes a plain '-L' so the bundled build dir wins. As a safeguard it also makes configure.R, src/install.libs.R and R/tbb.R ignore an ambient TBB_ROOT/TBB_LIB/TBB_INC (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE): the build images stopped exporting them in build-env-images c51ed09, but a stale image or ~/.Renviron would otherwise switch the build back to the system TBB, and R reads ~/.Renviron after the process environment, so no pipeline env can override it. The read sites move between releases, so each entry pins a patch generated against that version. Upstream still uses '-Wl,-L' in 6.2.1; drop these entries once a release passes a plain '-L'." - }, - { - "package": "RcppParallel", - "versions": "6.0.0", - "platforms": ["*"], - "env": {}, - "configure_args": [], - "makevars": {}, - "patch": "RcppParallel/force-bundled-tbb-6.0.0.patch", - "reason": "RcppParallel 6.x builds its bundled oneTBB 2022 with cmake, but tools/config/configure.R passes the library dir as '-Wl,-L...', and gcc places its own search dirs (/usr/lib64, /usr/lib/) ahead of anything forwarded with '-Wl,'. Every build image installs a distro libtbb, so '-ltbb' resolves to it: the binary records NEEDED libtbb.so.12 (libtbb.so.2 on el8/el9) instead of the bundled libtbb.so and cannot load on a machine without that exact system TBB. The patch passes a plain '-L' so the bundled build dir wins. As a safeguard it also makes configure.R, src/install.libs.R and R/tbb.R ignore an ambient TBB_ROOT/TBB_LIB/TBB_INC (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE): the build images stopped exporting them in build-env-images c51ed09, but a stale image or ~/.Renviron would otherwise switch the build back to the system TBB, and R reads ~/.Renviron after the process environment, so no pipeline env can override it. The read sites move between releases, so each entry pins a patch generated against that version. Upstream still uses '-Wl,-L' in 6.2.1; drop these entries once a release passes a plain '-L'." + "patch": "RcppParallel/force-bundled-tbb.patch", + "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, so the system-TBB workaround needed for 5.x is gone, but two things still steer the build back to a system TBB. (1) Ambient TBB_ROOT/TBB_LIB/TBB_INC: several build images still export these (build-env-images dropped them, but the images are rebuilt only by cron/manual runs, so a stale image keeps them), and R reads ~/.Renviron *after* the process environment, so no pipeline-side env override can undo it. configure.R then takes the system-TBB branch, install.libs.R symlinks the image's libraries into RcppParallel/lib as absolute paths, and the binary records NEEDED libtbb.so.12 (libtbb.so.2 for the classic Intel TBB on el8/el9) -- it cannot dyn.load on a consumer machine without that exact TBB. Both files are patched to ignore those variables (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE); install.libs.R needs it separately because R CMD INSTALL runs it outside the src/Makevars rule that passes the configured values. (2) Link order: the bundled branch passes its build dir as '-Wl,-Ltbb/build/lib_release', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,', so '-ltbb' would still resolve to a distro TBB when one is installed; a plain '-L' puts the bundled dir first. Verified on the current build-env-ubuntu:jammy image (which still exports TBB_INC/TBB_LIB): NEEDED libtbb.so, RUNPATH $ORIGIN/../lib, real libtbb.so.2 in RcppParallel/lib, and the package loads with every system libtbb moved away." }, { "package": "fs", diff --git a/local/prune-stale-flat-binaries.R b/local/prune-stale-flat-binaries.R deleted file mode 100644 index aea97b4..0000000 --- a/local/prune-stale-flat-binaries.R +++ /dev/null @@ -1,175 +0,0 @@ -### Remove flat-slot binaries recorded as unable to load under some R minor. -# -# Usage: -# Rscript local/prune-stale-flat-binaries.R [--apply] [pkg ...] -# -# The list comes from `abi_flat_safety`, written by the flat-safety backfill -# from the binaries' own symbols, so it names exactly the objects that fail at -# dyn.load() somewhere. It deliberately does *not* use -# `abi_cache_risky_packages()`: that source heuristic calls 93% of compiled -# packages risky where the binaries say 4.3%, and pruning on it would delete -# roughly 3000 working objects per slot to fix about 230 broken ones. Selecting -# per object rather than per package is the same correction #129 made to the -# index writer. -# -# `upload_single_binary()` has written a non-portable package only to the -# per-minor slot since v5.3.0, so this is a one-time cleanup of objects -# uploaded before that, not a recurring sweep. The flat slot is what every -# client that cannot be routed by R minor receives, and such a binary was built -# under whichever R was primary at the time, so it fails at load under any -# minor that dropped a symbol it uses: -# -# base64enc.so: undefined symbol: SETLENGTH -# -# Deleting the flat object does not create a hole: bincraft's source fallback -# publishes the CRAN source under the same key, so the package stays installable -# everywhere and compiles against the client's own R. Clients that are routed -# keep getting the per-minor binary. -# -# Dry run by default. Nothing is deleted without --apply. - -options(error = function() { - cat("ERROR:", geterrmessage(), "\n", file = stdout()) - traceback(2) - q(status = 1) -}) - -library(bincraft, quietly = TRUE) - -args <- commandArgs(trailingOnly = TRUE) -apply_changes <- "--apply" %in% args -args <- args[args != "--apply"] -if (length(args) < 2L) { - stop( - "usage: prune-stale-flat-binaries.R [--apply] [pkg ...]", - call. = FALSE - ) -} -arch <- args[[1L]] -codename <- args[[2L]] -explicit <- args[-c(1L, 2L)] - -bucket <- "devxy-rpkgs-binaries" -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE -) - -contrib <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename) -objects <- s3fs::s3_dir_ls(contrib, recurse = TRUE) -relative <- sub(sprintf("^(s3://)?%s/", contrib), "", objects) - -is_tarball <- grepl("\\.tar\\.gz$", relative) -# Flat objects only: no `/` prefix and not under Archive/. -is_flat <- is_tarball & !grepl("/", relative, fixed = TRUE) -flat_files <- relative[is_flat] -flat_pkg <- sub("_.*$", "", flat_files) - -if (length(explicit) > 0L) { - doomed <- flat_files[flat_pkg %in% explicit] - cat(sprintf( - "%s/%s: %d flat binaries, %d belong to the %d named package(s)\n", - arch, - codename, - length(flat_files), - length(doomed), - length(explicit) - )) -} else { - con <- DBI::dbConnect( - RPostgres::Postgres(), - dbname = "build_metadata", - host = "r-binaries.devxy.io", - port = 15432L, - user = "rpkgs", - password = Sys.getenv("PGPASS"), - sslmode = "require" - ) - on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE) - - # Every recorded-unsafe row, whichever minor it fails under: the flat object - # is served to clients on all of them, so one failing minor condemns it. - rows <- DBI::dbGetQuery( - con, - paste0( - "SELECT package, version FROM abi_flat_safety ", - "WHERE NOT safe AND codename = $1 AND arch = $2" - ), - params = list(codename, arch) - ) - if (nrow(rows) == 0L) { - stop( - "No unsafe objects are recorded for this slot. Refusing to proceed: that is indistinguishable from a slot whose flat-safety backfill never ran, and pruning nothing while reporting success would hide it.", # nolint - call. = FALSE - ) - } - - # Match the object, not the package. A package can have a safe flat binary at - # one version and an unsafe one at another, and deleting both would throw - # away a working binary. - unsafe_objects <- sprintf("%s_%s", rows$package, rows$version) - doomed <- flat_files[sub("\\.tar\\.gz$", "", flat_files) %in% unsafe_objects] - cat(sprintf( - "%s/%s: %d flat binaries, %d recorded unable to load under some R minor\n", - arch, - codename, - length(flat_files), - length(doomed) - )) -} - -# Never remove the last artifact of a package. The source fallback republishes -# the CRAN source under this key, but only for packages CRAN still ships, so -# confirm the source exists before deleting the binary. -cran <- tryCatch( - { - con <- gzcon(url( - "https://cloud.r-project.org/src/contrib/PACKAGES.gz", - open = "rb" - )) - on.exit(close(con), add = TRUE) - read.dcf(con, fields = "Package")[, "Package"] - }, - error = function(e) character(0) -) -if (length(cran) == 0L) { - stop("Could not read CRAN's index; refusing to prune blind.", call. = FALSE) -} - -orphaned <- doomed[!(sub("_.*$", "", doomed) %in% cran)] -if (length(orphaned) > 0L) { - cat(sprintf( - "Keeping %d object(s) whose package is no longer on CRAN, so there is no source to fall back to: %s\n", - length(orphaned), - paste(utils::head(orphaned, 5L), collapse = ", ") - )) - doomed <- setdiff(doomed, orphaned) -} - -if (length(doomed) == 0L) { - cat("Nothing to prune.\n") - quit(status = 0) -} - -for (f in doomed) { - cat(sprintf(" %s %s\n", if (apply_changes) "DELETE" else "would delete", f)) -} - -if (!apply_changes) { - cat(sprintf( - "\nDry run: %d object(s) left in place. Re-run with --apply to delete.\n", - length(doomed) - )) - quit(status = 0) -} - -for (f in doomed) { - s3fs::s3_file_delete(file.path(contrib, f)) -} -cat(sprintf( - "Deleted %d flat binary object(s). Re-index the slot so the source fallback republishes them.\n", - length(doomed) -)) diff --git a/local/retire-slot.R b/local/retire-slot.R deleted file mode 100644 index 383fd2c..0000000 --- a/local/retire-slot.R +++ /dev/null @@ -1,240 +0,0 @@ -### Retire a slot: strip it from the pipelines and delete its objects from S3. -# -# Rscript local/retire-slot.R [--apply] [--config-only|--assets-only] -# -# Retiring a distro retires both architectures, because `local.rpkgs_slots` in -# `cdn.tf` is a setproduct over them. -# -# Order: remove the slot from `local.rpkgs_slots` (and the website's -# `DistroId`) and apply that first, then run this. Deleting objects while a -# pipeline still targets the slot just means the next nightly rebuilds them. -# -# Cron jobs live on the Crow server, not here, and are printed at the end. -# -# Only structural YAML is edited. Prose mentioning the slot is left alone, so -# grep for the name afterwards. -# -# Dry run by default. - -options(error = function() { - cat("ERROR:", geterrmessage(), "\n", file = stdout()) - q(status = 1) -}) - -args <- commandArgs(trailingOnly = TRUE) -apply_changes <- "--apply" %in% args -config_only <- "--config-only" %in% args -assets_only <- "--assets-only" %in% args -args <- args[!args %in% c("--apply", "--config-only", "--assets-only")] -if (length(args) != 1L || (config_only && assets_only)) { - stop( - "usage: retire-slot.R [--apply] [--config-only|--assets-only]", - call. = FALSE - ) -} -codename <- args[[1L]] - -# The bucket and `CODENAME:` say `alpine322`; `OS:` and the options say -# `alpine-322`. Both have to go. -pipeline_os <- if (grepl("^rhel[0-9]+$", codename)) { - sub("^rhel", "redhat-", codename) -} else if (grepl("^alpine[0-9]+$", codename)) { - sub("^alpine", "alpine-", codename) -} else { - # Single bracket so an unknown codename yields NA, not an error. - c(resolute = "ubuntu-2604", noble = "ubuntu-2404", jammy = "ubuntu-2204")[ - codename - ] -} -if (is.na(pipeline_os) || length(pipeline_os) == 0L) { - stop(sprintf("Unrecognised codename '%s'.", codename), call. = FALSE) -} - -# The `OS_VERSION` dropdowns are keyed on the image tag, not the slot name. -image_tag <- if (grepl("^alpine-", pipeline_os)) { - sub("^alpine-([0-9])([0-9]+)$", "\\1.\\2", pipeline_os) -} else if (grepl("^redhat-", pipeline_os)) { - sub("^redhat-", "", pipeline_os) -} else { - codename -} - -# Refuse a slot that is still routed: deleting its objects would leave the edge -# serving indexes that name files which no longer exist. -tf <- readLines("cdn.tf", warn = FALSE) -start <- grep("rpkgs_slots\\s*=", tf) -if (length(start) != 1L) { - stop( - "Could not find exactly one `rpkgs_slots` local in cdn.tf.", - call. = FALSE - ) -} -block <- paste(tf[start:min(start + 10L, length(tf))], collapse = "\n") -if (grepl(sprintf('"%s"', codename), block, fixed = TRUE)) { - stop( - sprintf( - "'%s' is still listed in `local.rpkgs_slots` in cdn.tf, so the edge still routes it. Remove it there (and from the website's DistroId union) and apply the OpenTofu change first.", # nolint - codename - ), - call. = FALSE - ) -} - -cat(sprintf( - "Retiring %s (pipelines call it '%s', images tag it '%s')\n\n", - codename, - pipeline_os, - image_tag -)) - -## 1. Pipeline definitions ---------------------------------------------------- - -# Line-based on purpose: a YAML round trip would reflow all 18 files. -strip_slot <- function(lines) { - keep <- rep(TRUE, length(lines)) - i <- 1L - while (i <= length(lines)) { - line <- lines[[i]] - row <- regexec( - sprintf( - "^(\\s*)-\\s*(OS|CODENAME):\\s*(%s|%s)\\s*$", - pipeline_os, - codename - ), - line - ) - m <- regmatches(line, row)[[1L]] - if (length(m) == 4L) { - indent <- nchar(m[[2L]]) - keep[[i]] <- FALSE - j <- i + 1L - while (j <= length(lines)) { - nxt <- lines[[j]] - if ( - !nzchar(trimws(nxt)) || nchar(sub("^(\\s*).*$", "\\1", nxt)) <= indent - ) { - break - } - keep[[j]] <- FALSE - j <- j + 1L - } - i <- j - next - } - if (grepl(sprintf("^\\s*-\\s*%s-(amd64|arm64)\\s*$", pipeline_os), line)) { - keep[[i]] <- FALSE - } - if (grepl(sprintf("^\\s*-\\s*'%s'\\s*$", image_tag), line)) { - keep[[i]] <- FALSE - } - i <- i + 1L - } - out <- lines[keep] - # A dropdown cannot default to an option that no longer exists. - bad_default <- grepl(sprintf("^\\s*default:\\s*'%s'\\s*$", image_tag), out) - if (any(bad_default)) { - for (k in which(bad_default)) { - opts <- grep("^\\s*-\\s*'[^']+'\\s*$", out[seq_len(k - 1L)]) - if (length(opts) == 0L) { - next - } - # Walk back to this block's first option, not the nearest one. - first <- max(opts) - while (first > 1L && grepl("^\\s*-\\s*'[^']+'\\s*$", out[[first - 1L]])) { - first <- first - 1L - } - replacement <- sub("^\\s*-\\s*'([^']+)'\\s*$", "\\1", out[[first]]) - out[[k]] <- sub("'[^']+'", sprintf("'%s'", replacement), out[[k]]) - cat(sprintf(" default moved to '%s'\n", replacement)) - } - } - out -} - -if (!assets_only) { - files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE)) - touched <- 0L - for (f in files) { - lines <- readLines(f, warn = FALSE) - out <- strip_slot(lines) - if (length(out) == length(lines)) { - next - } - touched <- touched + 1L - cat(sprintf( - " %s %-45s -%d lines\n", - if (apply_changes) "EDIT " else "would edit", - f, - length(lines) - length(out) - )) - if (apply_changes) { - writeLines(out, f) - } - } - if (touched == 0L) { - cat(" No pipeline references the slot.\n") - } else if (apply_changes) { - cat(sprintf( - " %d file(s) edited. Verify with `crow lint .crow/` and `Rscript local/check-slots.R`.\n", - touched - )) - } - cat("\n") -} - -## 2. Bucket objects ---------------------------------------------------------- - -if (!config_only) { - library(s3fs, quietly = TRUE) - if (!nzchar(Sys.getenv("B2_S3_ACCESS_KEY"))) { - stop( - "B2_S3_ACCESS_KEY is unset. B2 lists nothing without credentials, which is indistinguishable from an already-clean slot.", # nolint - call. = FALSE - ) - } - bucket <- "devxy-rpkgs-binaries" - s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE - ) - - for (arch in c("amd64", "arm64")) { - prefix <- sprintf("%s/%s/%s", bucket, arch, codename) - if (!s3fs::s3_dir_exists(prefix)) { - cat(sprintf(" %s/%s: nothing in the bucket.\n", arch, codename)) - next - } - objects <- s3fs::s3_dir_ls(prefix, recurse = TRUE) - cat(sprintf( - " %s %s/%s: %d object(s)\n", - if (apply_changes) "DELETE" else "would delete", - arch, - codename, - length(objects) - )) - if (apply_changes && length(objects) > 0L) { - s3fs::s3_dir_delete(prefix) - cat(sprintf(" deleted s3://%s\n", prefix)) - } - } - cat("\n") -} - -## 3. Crons ------------------------------------------------------------------ - -if (!apply_changes) { - cat("Dry run: nothing was edited or deleted. Re-run with --apply.\n\n") -} - -cat("Crons still to remove on the Crow server (they are not in this repo):\n") -cat(sprintf( - " crow-at ci.devxy.io repo cron ls devxy/build-cran-binaries | grep %s\n", - pipeline_os -)) -cat(" # `crow repo cron rm` 404s here; the API base is /api/v1\n") -cat(" eval \"$(crow-at --export ci.devxy.io)\"\n") -cat(" curl -s -X DELETE -H \"Authorization: Bearer $CROW_TOKEN\" \\\n") -cat(" https://ci.devxy.io/api/v1/repos/5/cron/\n") diff --git a/local/tests/test-newer-r-helpers.R b/local/tests/test-newer-r-helpers.R deleted file mode 100644 index c202b7f..0000000 --- a/local/tests/test-newer-r-helpers.R +++ /dev/null @@ -1,81 +0,0 @@ -source(file.path("..", "newer-r-helpers.R")) - -test_that("R constraints respect patch versions, bounds, and missing fields", { - expect_true(r_requirement_satisfied(NA_character_, "4.4.3")) - expect_true(r_requirement_satisfied( - "Matrix (>= 1.4-0), R (>= 4.5.0)", - "4.5.3" - )) - expect_false(r_requirement_satisfied("R (>= 4.5.0)", "4.4.3")) - expect_false(r_requirement_satisfied("R (>= 4.5.4)", "4.5.3")) - expect_true(r_requirement_satisfied("R (>= 4.5), R (< 4.6)", "4.5.3")) - expect_false(r_requirement_satisfied("R (>= 4.5), R (< 4.6)", "4.6.0")) - expect_true(r_requirement_satisfied("R (== 4.5.3)", "4.5.3")) - expect_true(r_requirement_satisfied("R (> 4.5.2), R (<= 4.5.3)", "4.5.3")) - expect_true(r_requirement_satisfied("R, Rcpp (>= 1.0)", "4.4.3")) - expect_error(r_requirement_satisfied("R (~ 4.5)", "4.5.3"), "Cannot parse") -}) - -fixture <- function() { - data.frame( - Package = c( - "arules", - "arulesSequences", - "indirect", - "optional", - "ordinary", - "future", - "blocked" - ), - Version = rep("1.0", 7L), - Depends = c( - "R (>= 4.5.0)", - "arules", - NA, - NA, - "R (>= 4.0)", - "R (>= 4.7)", - NA - ), - Imports = c(NA, NA, "arulesSequences", NA, NA, NA, "arules, future"), - LinkingTo = NA_character_, - Suggests = c(NA, NA, NA, "arules", NA, NA, NA) - ) -} - -test_that("every compatible newer minor includes required dependents only", { - db <- fixture() - for (target in c("4.5.3", "4.6.0")) { - expect_identical( - newer_r_candidates(db, "4.4.3", target)$Package, - c("arules", "arulesSequences", "indirect") - ) - } - expect_equal(nrow(newer_r_candidates(db, "4.4.3", "4.4.3")), 0L) - expect_equal(nrow(newer_r_candidates(db, "4.5.3", "4.4.3")), 0L) - expect_equal(nrow(newer_r_candidates(db, "4.5.3", "4.6.0")), 0L) -}) - -test_that("LinkingTo, dependency cycles, Windows and empty metadata are handled", { - db <- fixture() - db$LinkingTo[5L] <- "arules" - db$Imports[1L] <- "ordinary" - db$OS_type <- c(NA, "windows", rep(NA_character_, 5L)) - expect_identical( - newer_r_candidates(db, "4.4.3", "4.6.0")$Package, - c("arules", "indirect", "ordinary") - ) - expect_equal(nrow(newer_r_candidates(db[FALSE, ], "4.4.3", "4.6.0")), 0L) -}) - -test_that("R-devel Recommended copies cannot block ordinary CRAN dependencies", { - db <- fixture() - db$Path <- NA_character_ - future_copy <- db[1L, ] - future_copy$Depends <- "R (>= 4.7)" - future_copy$Path <- "4.7.0/Recommended" - expect_identical( - newer_r_candidates(rbind(future_copy, db), "4.4.3", "4.6.1")$Package, - c("arules", "arulesSequences", "indirect") - ) -}) diff --git a/local/uvr-install.sh b/local/uvr-install.sh index e103e13..1e25e47 100755 --- a/local/uvr-install.sh +++ b/local/uvr-install.sh @@ -16,8 +16,8 @@ # precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds # bincraft and its dependencies alongside whatever this script installs). # -# System dependencies come from uvr's vendored r-system-requirements rules, -# with explicit prerequisites below for gaps in Alpine support. +# System dependencies come from uvr's vendored r-system-requirements rules, so +# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed. # # Environment: # UVR_R_BIN R interpreter to install for; set by install-bincraft.R so @@ -30,7 +30,7 @@ set -eu # renovate: datasource=github-releases depName=nbafrank/uvr -UVR_PIN="v0.4.7" +UVR_PIN="v0.4.6" if [ "$#" -eq 0 ]; then echo "usage: $0 ..." >&2 @@ -84,20 +84,9 @@ trap 'rm -rf "$project_dir"' EXIT cd "$project_dir" "$uvr_bin" init --here --r-version "$r_full" -# --no-install resolves and locks only; retry because concurrent shards can -# expose short-lived DNS or CRAN-index failures and uvr rolls the manifest back -# cleanly after an unsuccessful resolution. -add_attempt=1 -while ! "$uvr_bin" add --no-install "$@"; do - if [ "$add_attempt" -ge 4 ]; then - echo "error: uvr add failed after ${add_attempt} attempts" >&2 - exit 1 - fi - add_delay=$((add_attempt * 10)) - echo "warning: uvr add attempt ${add_attempt} failed; retrying in ${add_delay}s" >&2 - sleep "$add_delay" - add_attempt=$((add_attempt + 1)) -done +# --no-install: resolve and lock only. The install happens in the sync below, +# which is the only command that honours --library. +"$uvr_bin" add --no-install "$@" # TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs # `apt-get install` for every resolved system dependency without refreshing the @@ -117,11 +106,4 @@ if command -v apt-get >/dev/null 2>&1; then apt-get update -qq || echo "warning: apt-get update failed; continuing" >&2 fi -# Alpine's build images lack libintl.h, which data.table needs when R enables -# NLS. uvr does not resolve this prerequisite on Alpine, so install it before -# compiling bincraft's dependencies (including when the library cache is cold). -if command -v apk >/dev/null 2>&1; then - apk add --no-cache gettext-dev -fi - "$uvr_bin" sync --library "$target_lib" --install-system-deps diff --git a/local/verify-repo-routing.R b/local/verify-repo-routing.R deleted file mode 100644 index c19494d..0000000 --- a/local/verify-repo-routing.R +++ /dev/null @@ -1,193 +0,0 @@ -# Check that a published slot serves an installable, loadable binary for the R -# running this script, through one of the URL forms a user can configure. -# -# Rscript local/verify-repo-routing.R
-# -# One process per form on purpose: `available.packages()` caches its index in -# `tempdir()` keyed by repository URL, and the `arch` and `flat` forms share a -# URL. In a single process the second form would silently read the first form's -# index and the run would prove nothing. -# -# Forms: -# arch https://cran.rpkgs.com///latest with R's own agent -# generic https://cran.rpkgs.com/ with an agent naming the distribution -# flat the arch URL with a curl agent, so the edge cannot read an R minor -# from the request and the flat slot answers instead of a per-minor -# one -# -# `arch` and `generic` are contractual: they are what the website tells people -# to configure, so a failure there fails the run. `flat` measures what a client -# hiding its R minor is exposed to; it is reported and never fatal. - -args <- commandArgs(trailingOnly = TRUE) -if (length(args) != 3L) { - stop("usage: verify-repo-routing.R ", call. = FALSE) -} -arch <- args[[1L]] -codename <- args[[2L]] -form <- args[[3L]] - -# Compiled packages that exercise the R C API broadly, plus one pure-R package -# as a control: if `brew` fails too, the problem is the repository rather than -# the ABI. -PACKAGES <- c( - "Rcpp", - "data.table", - "stringi", - "xml2", - "curl", - "jsonlite", - "digest", - "base64enc", - "yaml", - "lobstr", - "brew" -) - -arch_url <- sprintf("https://cran.rpkgs.com/%s/%s/latest", arch, codename) - -# The agent the website's generic snippet builds, so this exercises the string -# real users send rather than an approximation of it. -generic_agent <- function() { - os_release <- readLines("/etc/os-release", warn = FALSE) - field <- function(key) { - line <- grep(sprintf("^%s=", key), os_release, value = TRUE) - if (length(line) == 0L) { - return("") - } - gsub('"', "", sub("^[^=]+=", "", line[[1L]])) - } - os <- paste(field("NAME"), field("VERSION_ID")) - codename_id <- field("VERSION_CODENAME") - if (nzchar(codename_id)) { - os <- sprintf("%s; codename=%s", os, codename_id) - } - sprintf( - "R/%s (%s) (%s)", - getRversion(), - os, - paste(R.version["platform"], R.version["arch"], R.version["os"]) - ) -} - -settings <- switch( - form, - arch = list(repo = arch_url, agent = getOption("HTTPUserAgent")), - generic = list(repo = "https://cran.rpkgs.com/", agent = generic_agent()), - flat = list(repo = arch_url, agent = "curl/8.5.0"), - stop(sprintf("unknown form '%s'", form), call. = FALSE) -) - -# No CRAN in `repos`: a package missing from our index must fail loudly rather -# than be quietly compiled from source, which would hide exactly the gap this -# script exists to find. -options(HTTPUserAgent = settings$agent, repos = c(rpkgs = settings$repo)) - -r_minor <- paste(getRversion()[1L, 1:2], collapse = ".") -rscript <- file.path(R.home("bin"), "Rscript") - -emit <- function(package, status, built = "", secs = NA_real_) { - cat( - sprintf( - "ROW\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - form, - arch, - codename, - r_minor, - package, - status, - # `%.1f` renders NA as the literal "NAs", which reads as a unit rather - # than as a missing measurement. A package that was never in the index - # has no timing at all. - if (is.na(secs)) built else sprintf("%s|%.1fs", built, secs) - ) - ) -} - -# `library()` in this process would abort the whole run on a bad `dyn.load()`, -# and the point of the exercise is to survive that and report it. -loads <- function(package, lib) { - status <- suppressWarnings(system2( - rscript, - c("-e", shQuote(sprintf('library("%s")', package))), - env = sprintf("R_LIBS=%s", lib), - stdout = FALSE, - stderr = FALSE - )) - identical(as.integer(status), 0L) -} - -built_field <- function(package, lib) { - desc <- file.path(lib, package, "DESCRIPTION") - if (!file.exists(desc)) { - return("") - } - value <- read.dcf(desc, fields = "Built")[[1L]] - if (is.na(value)) "" else sub(";.*$", "", value) -} - -available <- tryCatch( - rownames(available.packages()), - error = function(e) { - message(sprintf( - "index unreadable at %s: %s", - settings$repo, - conditionMessage(e) - )) - character() - } -) - -failures <- 0L -for (package in PACKAGES) { - if (!package %in% available) { - emit(package, "MISSING_FROM_INDEX") - failures <- failures + 1L - next - } - - lib <- file.path(tempdir(), sprintf("lib-%s-%s", form, package)) - dir.create(lib, showWarnings = FALSE, recursive = TRUE) - started <- Sys.time() - ok <- tryCatch( - { - # Hard dependencies come from the same repository and must be present - # for `R CMD INSTALL` to succeed, so they are installed too; Suggests are - # not, since a missing one does not stop a package loading. - install.packages( - package, - lib = lib, - dependencies = c("Depends", "Imports", "LinkingTo"), - quiet = TRUE - ) - package %in% rownames(installed.packages(lib.loc = lib)) - }, - error = function(e) FALSE - ) - secs <- as.numeric(difftime(Sys.time(), started, units = "secs")) - - if (!ok) { - emit(package, "INSTALL_FAILED", secs = secs) - failures <- failures + 1L - } else if (!loads(package, lib)) { - emit(package, "LOAD_FAILED", built_field(package, lib), secs) - failures <- failures + 1L - } else { - emit(package, "OK", built_field(package, lib), secs) - } - unlink(lib, recursive = TRUE, force = TRUE) -} - -message(sprintf( - "%s/%s R %s via %s: %d of %d packages usable", - arch, - codename, - r_minor, - form, - length(PACKAGES) - failures, - length(PACKAGES) -)) - -if (failures > 0L && !identical(form, "flat")) { - quit(status = 1L) -} diff --git a/provider.tf b/provider.tf index cb5a87d..badbbc1 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.19" + version = "~> 0.17" } } } diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 8e85cb6..391a814 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -18,95 +18,35 @@ # objects were replaced. The cost is a cold cache for everything else, which is # why this is not used by the daily update path. # -# The public hostnames currently use separate pull zones, so callers must pass -# every zone that serves the repository. A zone can be identified by its -# numeric ID or by one of its hostnames; hostname lookup avoids persisting IDs -# that change when a zone is recreated. +# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io, +# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them. # # Usage: -# purge_cdn_zone.sh [...] +# purge_cdn_zone.sh # set -euo pipefail if (($# < 2)); then - echo "usage: $0 [...]" >&2 + echo "usage: $0 " >&2 exit 2 fi api_key="$1" -shift +zone_id="$2" -resolve_zone_id() { - local zone="$1" - local response_file - local zone_id +echo "Purging BunnyCDN pull zone ${zone_id}" - if [[ "${zone}" =~ ^[0-9]+$ ]]; then - echo "${zone}" - return - fi +status=$( + curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \ + -H "AccessKey: ${api_key}" \ + -H "Content-Length: 0" \ + "https://api.bunny.net/pullzone/${zone_id}/purgeCache" +) - response_file=$(mktemp) - local status - status=$( - curl -sS -o "${response_file}" -w '%{http_code}' \ - -H "AccessKey: ${api_key}" \ - "https://api.bunny.net/pullzone?perPage=1000" - ) +if [[ "${status}" != "200" && "${status}" != "204" ]]; then + echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 + cat /tmp/purge_zone_response.txt >&2 + exit 1 +fi - if [[ "${status}" != "200" ]]; then - echo "Listing BunnyCDN pull zones failed with HTTP ${status}:" >&2 - head -c 500 "${response_file}" >&2 - echo >&2 - rm -f "${response_file}" - exit 1 - fi - - # The endpoint answers with a bare array on some accounts and a paginated - # object on others. `.Items // .` looks like it covers both but does not: - # indexing an array with a string is an *error*, and `//` only substitutes - # for null, so the array case aborted with - # "Cannot index array with string" and the zone was never purged. - zone_id=$( - jq -r --arg hostname "${zone}" \ - '(if type == "object" then (.Items // []) else . end)[] - | select(any(.Hostnames[]?; .Value == $hostname)) - | .Id' \ - "${response_file}" - ) - rm -f "${response_file}" - - if [[ -z "${zone_id}" ]]; then - echo "Could not find BunnyCDN pull zone for hostname ${zone}" >&2 - exit 1 - fi - - # Two zones sharing a hostname would purge only whichever jq emitted first. - if [[ $(wc -l <<<"${zone_id}") -gt 1 ]]; then - echo "Hostname ${zone} matched multiple pull zones: ${zone_id//$'\n'/ }" >&2 - exit 1 - fi - - echo "${zone_id}" -} - -for zone in "$@"; do - zone_id=$(resolve_zone_id "${zone}") - echo "Purging BunnyCDN pull zone ${zone_id}" - - response_file="/tmp/purge_zone_response_${zone_id}.txt" - status=$( - curl -sS -o "${response_file}" -w '%{http_code}' -X POST \ - -H "AccessKey: ${api_key}" \ - -H "Content-Length: 0" \ - "https://api.bunny.net/pullzone/${zone_id}/purgeCache" - ) - - if [[ "${status}" != "200" && "${status}" != "204" ]]; then - echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 - cat "${response_file}" >&2 - exit 1 - fi - - echo "Purged pull zone ${zone_id} (HTTP ${status})" -done +echo "Purged pull zone ${zone_id} (HTTP ${status})" diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh deleted file mode 100755 index 190245a..0000000 --- a/scripts/verify-r-minor-routing.sh +++ /dev/null @@ -1,434 +0,0 @@ -#!/usr/bin/env bash -# -# Verify per-R-minor index routing for cran.rpkgs.com across every published -# / slot. -# -# The edge router (edge/rpkgs-router.ts) rewrites PACKAGES* requests to -# `contrib//` when the slot is listed in UNION_SLOTS and the client's -# User-Agent carries an R minor. Two properties have to hold before a slot may -# be added to UNION_SLOTS: -# -# 1. the per-minor index is a UNION of the per-minor and flat slots, so -# routing to it hides nothing the flat index carries; and -# 2. every R minor a client might report resolves to an index that exists, -# because contribPath() does not check existence and has no fallback. -# -# Modes: -# (default) Resolve routing decisions without depending on UNION_SLOTS being -# set. Safe to run before enabling: it reads the per-minor indexes -# directly and reproduces the router's target path. -# --live Additionally drive the real CDN with R User-Agents and assert the -# bytes served match the expected index. Only meaningful once the -# slot is in UNION_SLOTS. -# -# Usage: -# scripts/verify-r-minor-routing.sh -# scripts/verify-r-minor-routing.sh --live -# MINORS="4.4 4.5" SAMPLE=10 scripts/verify-r-minor-routing.sh -# -# Exits non-zero if any check fails. - -set -uo pipefail - -BASE=${BASE:-https://cran.rpkgs.com} -ARCHES=${ARCHES:-"amd64 arm64"} -DISTROS=${DISTROS:-"resolute noble jammy rhel8 rhel9 rhel10 alpine323 alpine324"} -# The supported window: the current R minor plus the two previous, matching -# build-env-images' R_VERSION_LATEST/PREV1/PREV2 and cdn.tf's -# local.rpkgs_supported_minors. Each of these must have a published index. -MINORS=${MINORS:-"4.4 4.5 4.6"} -# Minors we deliberately do not serve. These must have NO published index and, -# once routing is live, must be sent to CRAN for sources rather than 404ing or -# being handed binaries built under another minor. -EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"} -# How many Path: targets to HEAD-check per slot/minor. 0 disables. -SAMPLE=${SAMPLE:-5} -# Package-count shortfall against the best minor on the same slot, above which -# coverage is reported as uneven. Reported, not failed on: the packages a -# non-primary minor lacks are ABI-risky ones built under the primary minor, -# which a client on another minor cannot safely load anyway, so their absence -# is correct. This gates the *claim* ("full coverage for ABI-sensitive -# packages"), not whether routing is safe to enable - MAX_REGRESSIONS does -# that. -PARITY_TOLERANCE=${PARITY_TOLERANCE:-25} -# Source fallbacks are reported, not failed on. Since bincraft learned to keep -# a matching-minor generic binary out of a fallback's shadow, a remaining -# fallback means the generic slot's binary was built under a *different* minor, -# which is unsafe for this client anyway: serving source there is correct, just -# slow. The gate below is what actually matters. -# -# A REGRESSION is a package this client would receive as source through -# per-minor routing but as a binary built under its own minor from the generic -# slot. That is strictly worse than not routing at all, and must be zero before -# a slot is added to UNION_SLOTS. -MAX_REGRESSIONS=${MAX_REGRESSIONS:-0} -LIVE=0 - -for arg in "$@"; do - case "$arg" in - --live) LIVE=1 ;; - -h | --help) - sed -n '2,32p' "$0" - exit 0 - ;; - *) - echo "unknown argument: $arg" >&2 - exit 2 - ;; - esac -done - -WORK=$(mktemp -d) -trap 'rm -rf "$WORK"' EXIT - -PASS=0 -FAIL=0 -FAILURES="" - -ok() { - PASS=$((PASS + 1)) - printf ' ok %s\n' "$1" -} - -bad() { - FAIL=$((FAIL + 1)) - FAILURES="${FAILURES}\n - $1" - printf ' FAIL %s\n' "$1" -} - -# Fetch a URL into a file, echoing the HTTP status. Cached per URL. -fetch() { - local url=$1 dest=$2 ua=${3:-} - if [ -s "$dest" ]; then - cat "$dest.status" - return 0 - fi - local status - # -L: the router answers an index request with a redirect, so the bytes a - # client ends up with are only visible by following it. - # - # no-cache: a purge is asynchronous, so a run started right after a reindex - # otherwise measures whatever the edge still holds. - if [ -n "$ua" ]; then - status=$(curl -sSL -A "$ua" -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) - else - status=$(curl -sSL -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) - fi - echo "$status" > "$dest.status" - echo "$status" -} - -head_status() { - curl -sS -o /dev/null -w '%{http_code}' -I --max-time 60 "$1" 2>/dev/null -} - -# Package names from a gzipped PACKAGES index, sorted. -pkg_names() { - gunzip -c "$1" 2>/dev/null | awk '/^Package:/ {print $2}' | sort -u -} - -# " " for a per-minor index: how many entries carry a -# Path: field, and how many of those lack a Built: field (i.e. are sources). -fallback_counts() { - gunzip -c "$1" 2>/dev/null | awk ' - /^Package:/ { pkg = $2; path = ""; built = "" } - /^Path:/ { path = $2 } - /^Built:/ { built = $2 } - /^$/ { if (pkg != "" && path != "") { n++; if (built == "") s++ } pkg = "" } - END { if (pkg != "" && path != "") { n++; if (built == "") s++ } - printf "%d %d\n", n, s } - ' -} - -# Packages this index serves from the generic slot with a binary built under a -# different R minor, while some other per-minor slot carries a build of them - -# which proves the ABI classifier called them risky. Serving those is the -# load-time crash the per-minor slots exist to prevent. bincraft drops them at -# index time, so a non-zero count means the slot has not been reindexed since -# that guard shipped. -abi_unsafe_count() { - local minor_file=$1 minor=$2 risky_file=$3 - gunzip -c "$minor_file" 2>/dev/null | awk -v m="$minor" ' - /^Package:/ { pkg = $2; path = ""; built = "" } - /^Path:/ { path = $2 } - /^Built:/ { built = $2 " " $3 } - /^$/ { if (pkg != "" && path == "" && built != "" && built !~ ("^R " m "\\.")) print pkg; pkg = "" } - ' | sort -u > "$minor_file.mismatched" - comm -12 "$minor_file.mismatched" "$risky_file" | wc -l -} - -# How many packages a client of would receive as source through -# per-minor routing while the generic slot holds a binary built under that very -# minor. Zero is the bar for enabling a slot. -regression_count() { - local minor_file=$1 flat_file=$2 minor=$3 - gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" ' - /^Package:/ { pkg = $2; built = "" } - /^Built:/ { built = $2 " " $3 } - /^$/ { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" } - ' | sort -u > "$minor_file.flatbin" - gunzip -c "$minor_file" 2>/dev/null | awk ' - /^Package:/ { pkg = $2; path = ""; built = "" } - /^Path:/ { path = $2 } - /^Built:/ { built = $2 } - /^$/ { if (pkg != "" && path != "" && built == "") print pkg; pkg = "" } - ' | sort -u > "$minor_file.src" - comm -12 "$minor_file.src" "$minor_file.flatbin" | wc -l -} - -# " " pairs for entries that carry a Path: field. -path_entries() { - gunzip -c "$1" 2>/dev/null | awk ' - /^Package:/ { pkg = $2; ver = ""; path = "" } - /^Version:/ { ver = $2 } - /^Path:/ { path = $2 } - /^$/ { if (pkg != "" && path != "") print pkg, ver, path; pkg = "" } - END { if (pkg != "" && path != "") print pkg, ver, path } - ' -} - -# An R User-Agent of the shape R actually sends. -r_user_agent() { - printf 'R/%s.0 (Ubuntu 24.04; codename=noble) (x86_64-pc-linux-gnu x86_64 linux-gnu)' "$1" -} - -echo "verify-r-minor-routing: $BASE" -echo " slots: $(echo "$ARCHES" | wc -w) arch x $(echo "$DISTROS" | wc -w) os" -echo " minors: $MINORS (excluded: $EXCLUDED_MINORS)" -echo " live: $LIVE" -echo - -for arch in $ARCHES; do - for distro in $DISTROS; do - slot="$arch/$distro" - echo "$slot" - - flat_url="$BASE/$slot/latest/src/contrib/PACKAGES.gz" - flat_file="$WORK/${arch}-${distro}-flat.gz" - flat_status=$(fetch "$flat_url" "$flat_file") - - if [ "$flat_status" != "200" ]; then - bad "$slot flat index unreachable (HTTP $flat_status)" - continue - fi - - pkg_names "$flat_file" > "$flat_file.names" - flat_count=$(wc -l < "$flat_file.names") - if [ "$flat_count" -lt 1000 ]; then - bad "$slot flat index has only $flat_count packages" - continue - fi - ok "$slot flat index: $flat_count packages" - - for minor in $MINORS; do - minor_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" - minor_file="$WORK/${arch}-${distro}-${minor}.gz" - minor_status=$(fetch "$minor_url" "$minor_file") - - # A minor the router would route to must exist, or clients on that R - # version get a 404 and see no packages at all. - if [ "$minor_status" != "200" ]; then - bad "$slot R $minor index missing (HTTP $minor_status) - routing would 404 for R $minor clients" - continue - fi - - pkg_names "$minor_file" > "$minor_file.names" - minor_count=$(wc -l < "$minor_file.names") - - # Union property: nothing the flat index carries may be missing here. - # bincraft deliberately drops an ABI-risky package whose only binary was - # built under another R minor: serving it is the load-time crash the - # per-minor slots exist to prevent. Those absences are correct. - # - # What must never go missing is a generic package built under *this* - # minor, which is safe to serve and has no reason to disappear. - comm -23 "$flat_file.names" "$minor_file.names" > "$minor_file.absent" - absent_count=$(wc -l < "$minor_file.absent") - gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" ' - /^Package:/ { pkg = $2; built = "" } - /^Built:/ { built = $2 " " $3 } - /^$/ { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" } - ' | sort -u > "$minor_file.flatsame" - lost=$(comm -12 "$minor_file.absent" "$minor_file.flatsame" | wc -l) - - if [ "${lost:-0}" -ne 0 ]; then - bad "$slot R $minor index dropped $lost generic package(s) built under R $minor, which were safe to serve" - else - ok "$slot R $minor index: $minor_count packages, union holds ($absent_count ABI-unsafe dropped)" - fi - - # A per-minor entry that is a source fallback resolves fine but makes the - # client compile. Routing to a slot that is mostly fallbacks does not - # deliver the binaries we advertise. - read -r steered fallbacks <<< "$(fallback_counts "$minor_file")" - if [ "${steered:-0}" -gt 0 ]; then - pct=$((fallbacks * 100 / steered)) - printf ' note: %s/%s per-minor entries are source fallbacks (%s%%)\n' \ - "$fallbacks" "$steered" "$pct" - fi - - # The gate: nothing may arrive as source here that the generic slot would - # have served as a binary built under this same minor. - regressions=$(regression_count "$minor_file" "$flat_file" "$minor") - if [ "${regressions:-0}" -gt "$MAX_REGRESSIONS" ]; then - bad "$slot R $minor: $regressions packages would be served as source but exist as an R $minor binary in the generic slot" - else - ok "$slot R $minor: no regression against the generic slot" - fi - - # Path: entries steer to per-minor binaries; they must resolve. - if [ "$SAMPLE" -gt 0 ]; then - path_entries "$minor_file" > "$minor_file.paths" - total_paths=$(wc -l < "$minor_file.paths") - broken=0 - checked=0 - while read -r pkg ver path; do - [ -z "${pkg:-}" ] && continue - tarball="$BASE/$slot/latest/src/contrib/$path/${pkg}_${ver}.tar.gz" - status=$(head_status "$tarball") - checked=$((checked + 1)) - if [ "$status" != "200" ]; then - broken=$((broken + 1)) - [ "$broken" -le 2 ] && printf ' broken target: %s (HTTP %s)\n' "$tarball" "$status" - fi - done < <(shuf -n "$SAMPLE" "$minor_file.paths" 2>/dev/null || head -n "$SAMPLE" "$minor_file.paths") - - if [ "$broken" -ne 0 ]; then - bad "$slot R $minor: $broken/$checked sampled Path: targets do not resolve (of $total_paths total)" - elif [ "$checked" -gt 0 ]; then - ok "$slot R $minor: $checked/$checked sampled Path: targets resolve (of $total_paths total)" - fi - fi - - # Live routing: what a real R client on this minor actually receives. - if [ "$LIVE" -eq 1 ]; then - ua=$(r_user_agent "$minor") - live_file="$WORK/${arch}-${distro}-${minor}-live.gz" - live_status=$(fetch "$flat_url" "$live_file" "$ua") - if [ "$live_status" != "200" ]; then - bad "$slot R $minor live request failed (HTTP $live_status)" - elif cmp -s "$live_file" "$minor_file"; then - ok "$slot R $minor live request served the per-minor index" - elif cmp -s "$live_file" "$flat_file"; then - bad "$slot R $minor live request served the FLAT index - slot not in UNION_SLOTS?" - else - bad "$slot R $minor live request served neither the per-minor nor the flat index" - fi - fi - done - - # Coverage parity across minors. The union property only guarantees no - # client loses packages relative to the flat index; it says nothing about a - # 4.4 client seeing fewer packages than a 4.5 client on the same slot. - best=0 - for minor in $MINORS; do - f="$WORK/${arch}-${distro}-${minor}.gz.names" - [ -s "$f" ] || continue - c=$(wc -l < "$f") - [ "$c" -gt "$best" ] && best=$c - done - if [ "$best" -gt 0 ]; then - uneven="" - for minor in $MINORS; do - f="$WORK/${arch}-${distro}-${minor}.gz.names" - [ -s "$f" ] || continue - c=$(wc -l < "$f") - gap=$((best - c)) - [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap" - done - if [ -n "$uneven" ]; then - printf ' note: %s coverage uneven across minors (vs best %s):%s\n' \ - "$slot" "$best" "$uneven" - else - ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)" - fi - fi - - # Packages carrying a Path in any per-minor index are risky by construction. - : > "$WORK/${arch}-${distro}.risky" - for minor in $MINORS; do - f="$WORK/${arch}-${distro}-${minor}.gz" - [ -s "$f" ] || continue - gunzip -c "$f" 2>/dev/null | awk ' - /^Package:/ { pkg = $2; path = "" } - /^Path:/ { path = $2 } - /^$/ { if (pkg != "" && path != "") print pkg; pkg = "" } - ' >> "$WORK/${arch}-${distro}.risky" - done - sort -u -o "$WORK/${arch}-${distro}.risky" "$WORK/${arch}-${distro}.risky" - - for minor in $MINORS; do - f="$WORK/${arch}-${distro}-${minor}.gz" - [ -s "$f" ] || continue - unsafe=$(abi_unsafe_count "$f" "$minor" "$WORK/${arch}-${distro}.risky") - if [ "${unsafe:-0}" -gt 0 ]; then - bad "$slot R $minor serves $unsafe ABI-risky package(s) built under another R minor - reindex this slot" - else - ok "$slot R $minor serves no ABI-risky package from another minor" - fi - done - - # Excluded minors: no published index, and under --live a redirect to CRAN. - for minor in $EXCLUDED_MINORS; do - ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" - ex_status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 60 "$ex_url" 2>/dev/null) - if [ "$ex_status" = "200" ]; then - bad "$slot R $minor is excluded but an index is published - the two lists disagree" - else - ok "$slot R $minor correctly has no published index" - fi - - if [ "$LIVE" -eq 1 ]; then - loc=$(curl -sS -o /dev/null -w '%{redirect_url}' -A "$(r_user_agent "$minor")" \ - --max-time 60 "$flat_url" 2>/dev/null) - case "$loc" in - https://cran.r-project.org/*) - ok "$slot R $minor is sent to CRAN ($loc)" - ;; - "") - bad "$slot R $minor was served directly instead of being sent to CRAN" - ;; - *) - bad "$slot R $minor redirected somewhere unexpected: $loc" - ;; - esac - fi - done - - # A client whose User-Agent carries no R version must keep getting the flat - # index, never a per-minor one. - if [ "$LIVE" -eq 1 ]; then - plain_file="$WORK/${arch}-${distro}-plain.gz" - plain_status=$(fetch "$flat_url" "$plain_file" "curl/8.0.0") - if [ "$plain_status" != "200" ]; then - bad "$slot non-R User-Agent request failed (HTTP $plain_status)" - elif cmp -s "$plain_file" "$flat_file"; then - ok "$slot non-R User-Agent still served the flat index" - else - bad "$slot non-R User-Agent was routed away from the flat index" - fi - - # Tarball requests must never be rewritten into a per-minor directory: - # flat-slot packages do not live there. - sample_pkg=$(gunzip -c "$flat_file" | awk '/^Package:/ {p=$2} /^Version:/ {print p, $2; exit}') - if [ -n "$sample_pkg" ]; then - # shellcheck disable=SC2086 # deliberate split into $1 (package) and $2 (version) - set -- $sample_pkg - tb="$BASE/$slot/latest/src/contrib/${1}_${2}.tar.gz" - tb_status=$(curl -sS -o /dev/null -w '%{http_code}' -A "$(r_user_agent 4.5)" --max-time 60 "$tb" 2>/dev/null) - if [ "$tb_status" = "200" ]; then - ok "$slot tarball request under an R User-Agent still resolves" - else - bad "$slot tarball ${1}_${2}.tar.gz broke under an R User-Agent (HTTP $tb_status)" - fi - fi - fi - done -done - -echo -echo "passed: $PASS failed: $FAIL" -if [ "$FAIL" -ne 0 ]; then - printf 'failures:%b\n' "$FAILURES" - exit 1 -fi