Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
| aba2063ea0 | |||
|
eeebef8edb |
|||
| b0d58f3f7c | |||
| a8820e6e90 | |||
| 77a2f1f04a | |||
|
e9782bb529 |
|||
|
|
d2333c6cbe | ||
|
|
f4ab6f9dc5 | ||
|
|
d1da0c6cb4 | ||
|
|
c7b4dca6e2 | ||
|
50495f5c3b |
|||
|
b75fd2f1c4 |
|||
|
132d1d2d3c |
|||
|
|
706fd10d79 | ||
| 9bded261ee | |||
| a1c1f5e78f | |||
| aa4c95457f | |||
| 4b7dc28cc8 |
21 changed files with 1778 additions and 156 deletions
|
|
@ -58,7 +58,7 @@ steps:
|
|||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /mnt/cache/R-pkgs
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT
|
||||
backend_options:
|
||||
kubernetes:
|
||||
|
|
|
|||
|
|
@ -3,15 +3,15 @@
|
|||
# Routing is preserved 1:1:
|
||||
# - cron: each existing `process-cran-updates-<os>-<arch>` cron fires only
|
||||
# its matching matrix row (via the per-row `cron:` name filter).
|
||||
# - manual: pick a target from the `process_cran_updates` dropdown
|
||||
# ("all" = every os/arch).
|
||||
# - manual: pick a target from the `process_cran_updates` dropdown;
|
||||
# "all" fans out every os/arch as parallel matrix workflows.
|
||||
# Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64).
|
||||
variables:
|
||||
# Gates this pipeline. A manual pipeline creation instantiates every file in
|
||||
# .crow/, and a declared default is applied even when the run never passed
|
||||
# this variable, so the default must be a value that matches no matrix row.
|
||||
process_cran_updates:
|
||||
description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
|
||||
description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
|
||||
options:
|
||||
- none
|
||||
- all
|
||||
|
|
@ -203,6 +203,7 @@ steps:
|
|||
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
|
||||
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||
# rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
|
||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||
|
|
@ -218,6 +219,7 @@ steps:
|
|||
LIB="/mnt/cache/R-pkgs-$RMINOR"
|
||||
mkdir -p "$LIB"
|
||||
R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
|
||||
R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
|
||||
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true
|
||||
done
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
||||
|
|
|
|||
|
|
@ -3,15 +3,15 @@
|
|||
# Routing is preserved 1:1:
|
||||
# - cron: each existing `weekly-audit-missing-<os>-<arch>` cron fires only
|
||||
# its matching matrix row (via the per-row `cron:` name filter).
|
||||
# - manual: pick a target from the `weekly_audit_missing` dropdown
|
||||
# ("all" = every os/arch).
|
||||
# - manual: pick a target from the `weekly_audit_missing` dropdown;
|
||||
# "all" fans out every os/arch as parallel matrix workflows.
|
||||
# Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64).
|
||||
variables:
|
||||
# Gates this pipeline. A manual pipeline creation instantiates every file in
|
||||
# .crow/, and a declared default is applied even when the run never passed
|
||||
# this variable, so the default must be a value that matches no matrix row.
|
||||
weekly_audit_missing:
|
||||
description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
|
||||
description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
|
||||
options:
|
||||
- none
|
||||
- all
|
||||
|
|
@ -147,7 +147,7 @@ steps:
|
|||
- mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")'
|
||||
backend_options:
|
||||
docker:
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ steps:
|
|||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /mnt/cache/R-pkgs
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue
|
||||
backend_options:
|
||||
|
|
|
|||
|
|
@ -1,18 +1,27 @@
|
|||
# Consolidated weekly-rebuild-missing pipeline (all platforms, both arches).
|
||||
# One matrix row per OS/arch replaces the former per-platform files.
|
||||
# Three matrix rows per OS/arch, one per shard of that slot's rebuild list.
|
||||
# Routing is preserved 1:1:
|
||||
# - cron: each existing `weekly-rebuild-missing-<os>-<arch>` cron fires only
|
||||
# its matching matrix row (via the per-row `cron:` name filter).
|
||||
# - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the
|
||||
# previous bare manual trigger that ran every os/arch); pick a
|
||||
# single <os>-<arch> to run just one.
|
||||
# its matching matrix rows (via the per-row `cron:` name filter),
|
||||
# which is now all three shards of that slot.
|
||||
# - manual: pick a target from the `weekly_rebuild_missing` dropdown;
|
||||
# "all" fans out every os/arch and shard as parallel matrix
|
||||
# workflows, while a single <os>-<arch> runs its three shards.
|
||||
# Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64).
|
||||
#
|
||||
# The shard picks up its own slice and re-derives what is still outstanding
|
||||
# from the bucket, so a restart resumes rather than replaying; see
|
||||
# local/rebuild-missing.R.
|
||||
#
|
||||
# Re-indexing and the CDN purge deliberately do NOT live here. Three shards
|
||||
# writing one slot's PACKAGES concurrently would race, so they moved to
|
||||
# .crow/weekly-rebuild-reindex.yaml, which depends on this pipeline.
|
||||
variables:
|
||||
# Gates this pipeline. A manual pipeline creation instantiates every file in
|
||||
# .crow/, and a declared default is applied even when the run never passed
|
||||
# this variable, so the default must be a value that matches no matrix row.
|
||||
weekly_rebuild_missing:
|
||||
description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
|
||||
description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
|
||||
options:
|
||||
- none
|
||||
- all
|
||||
|
|
@ -53,74 +62,326 @@ matrix:
|
|||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: alpine-322
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: alpine-322
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
|
||||
steps:
|
||||
- name: 'Rebuild missing binaries'
|
||||
|
|
@ -153,6 +414,12 @@ steps:
|
|||
PLATFORM: ${OS}
|
||||
ARCH: ${ARCH}
|
||||
NCPUS: 2
|
||||
SPLIT_INTO: ${SPLIT_INTO}
|
||||
SPLIT_INDEX: ${SPLIT_INDEX}
|
||||
# Wall clock after which the shard stops cleanly instead of having to be
|
||||
# killed. A kill matches neither `success` nor `failure`, so it would skip
|
||||
# the dependent re-index and leave rebuilt binaries behind a stale edge.
|
||||
REBUILD_BUDGET_HOURS: 20
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
|
|
@ -162,18 +429,7 @@ steps:
|
|||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
|
||||
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1
|
||||
# A rebuild replaces objects in place, so the slot's index still advertises
|
||||
# the old MD5 and, for anything that had been served from source, no Built
|
||||
# stamp. Re-index here rather than waiting for the next process-updates
|
||||
# run, or the rebuilt binaries stay invisible to clients until then.
|
||||
# The codename is detected from the image's /etc/os-release.
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
||||
- |
|
||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
||||
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
|
||||
done
|
||||
- $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1
|
||||
backend_options:
|
||||
docker:
|
||||
resources:
|
||||
|
|
@ -183,25 +439,3 @@ steps:
|
|||
limits:
|
||||
memory: 18Gi
|
||||
cpu: 3000m
|
||||
|
||||
- name: Purge CDN cache
|
||||
image: reg.devxy.io/docker.io/library/alpine:3.24
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
BUNNYNET_API_KEY:
|
||||
from_secret: BUNNYNET_API_KEY
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
# All hostnames on the zone share this id, so one purge covers
|
||||
# cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com.
|
||||
BUNNY_PULLZONE: '3857050'
|
||||
commands:
|
||||
- apk add --no-cache -q bash curl git
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE"
|
||||
# A rebuild that died part-way still replaced objects, and those are exactly
|
||||
# the ones a stale edge would keep hiding, so purge either way.
|
||||
when:
|
||||
- status: [success, failure]
|
||||
|
|
|
|||
191
.crow/weekly-rebuild-reindex.yaml
Normal file
191
.crow/weekly-rebuild-reindex.yaml
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
# Re-index and purge after weekly-rebuild-missing.
|
||||
#
|
||||
# weekly-rebuild-missing runs three shards per slot. Each of them replaces
|
||||
# objects in place, so the slot's index still advertises the old MD5 and, for
|
||||
# anything that had been served from source, no Built stamp. Re-indexing from
|
||||
# inside a shard would mean three concurrent `upload_package_index()` calls on
|
||||
# one prefix: `cranlike::update_PACKAGES()` lists the live bucket, so an early
|
||||
# lister that uploads last publishes an index missing its siblings' work.
|
||||
#
|
||||
# So it happens exactly once per slot, here, after every shard has finished.
|
||||
# `runs_on: [success, failure]` keeps that true when a shard fails; only an
|
||||
# explicit cancel skips it, and this pipeline can then be triggered on its own.
|
||||
|
||||
variables:
|
||||
# Mirrors the gate on weekly-rebuild-missing so a manual run re-indexes
|
||||
# exactly the slots it rebuilt. A manual pipeline creation instantiates every
|
||||
# file in .crow/, so the default must match no matrix row.
|
||||
weekly_rebuild_missing:
|
||||
description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
|
||||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-322-amd64
|
||||
- alpine-322-arm64
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
- alpine-324-arm64
|
||||
- redhat-8-amd64
|
||||
- redhat-8-arm64
|
||||
- redhat-9-amd64
|
||||
- redhat-9-arm64
|
||||
- redhat-10-amd64
|
||||
- redhat-10-arm64
|
||||
- ubuntu-2204-amd64
|
||||
- ubuntu-2204-arm64
|
||||
- ubuntu-2404-amd64
|
||||
- ubuntu-2404-arm64
|
||||
- ubuntu-2604-amd64
|
||||
- ubuntu-2604-arm64
|
||||
default: none
|
||||
|
||||
when:
|
||||
- event: cron
|
||||
cron: weekly-rebuild-missing-${OS}-${ARCH}
|
||||
- event: manual
|
||||
evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"'
|
||||
|
||||
depends_on:
|
||||
- weekly-rebuild-missing
|
||||
|
||||
runs_on: [success, failure]
|
||||
|
||||
skip_clone: true
|
||||
|
||||
labels:
|
||||
group: rpkgs-${ARCH}
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-322
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
|
||||
steps:
|
||||
- name: 'Re-index the slot'
|
||||
image: reg.devxy.io/rpkgs/build-env-${IMG}
|
||||
pull: true
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
RED_HAT_DEV_PW:
|
||||
from_secret: RED_HAT_DEV_PW
|
||||
B2_S3_ACCESS_KEY:
|
||||
from_secret: B2_S3_ACCESS_KEY
|
||||
B2_S3_SECRET_KEY:
|
||||
from_secret: B2_S3_SECRET_KEY
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
GIT_USER: pat-s
|
||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
R_VERSION: ${R_VERSION}
|
||||
PLATFORM: ${OS}
|
||||
ARCH: ${ARCH}
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /mnt/cache/R-pkgs
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
# The codename is detected from the image's /etc/os-release.
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
||||
- |
|
||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
||||
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
|
||||
done
|
||||
|
||||
- name: Purge CDN cache
|
||||
image: reg.devxy.io/docker.io/library/alpine:3.24
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
BUNNYNET_API_KEY:
|
||||
from_secret: BUNNYNET_API_KEY
|
||||
# cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
|
||||
# Bunny pull zones, so both must be purged after the shared origin changes.
|
||||
BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io'
|
||||
commands:
|
||||
- apk add --no-cache -q bash curl jq
|
||||
# Crow carries the checkout from the re-index step into this step.
|
||||
- bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
|
||||
# Runs on every row rather than on one designated slot: a cron fires only
|
||||
# its own slot's row, so gating on a named slot would leave every other
|
||||
# slot unpurged. A manual "all" run therefore purges the zone 18 times,
|
||||
# which is a cheap API call and rare.
|
||||
#
|
||||
# Run it even when the re-index above failed: the objects were still
|
||||
# replaced, and a stale edge is exactly what keeps them hidden.
|
||||
when:
|
||||
- status: [success, failure]
|
||||
|
|
@ -36,7 +36,7 @@ repos:
|
|||
hooks:
|
||||
- id: air-format
|
||||
- repo: https://github.com/editorconfig-checker/editorconfig-checker
|
||||
rev: v3.11.1
|
||||
rev: v3.11.2
|
||||
hooks:
|
||||
- id: editorconfig-checker
|
||||
exclude: ^local/patches/.*\.patch$
|
||||
|
|
|
|||
134
.terraform.lock.hcl
generated
134
.terraform.lock.hcl
generated
|
|
@ -2,79 +2,79 @@
|
|||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.opentofu.org/hashicorp/http" {
|
||||
version = "3.6.0"
|
||||
version = "3.6.1"
|
||||
hashes = [
|
||||
"h1:0n4RBz9zNw6TTddh5+x7E8L2+qzPXNwKhK4uoZ/DUwE=",
|
||||
"h1:22Ob7lpzMBSqdrCvoFN5EgmhGPHPBovV/9qo0c/Cd+A=",
|
||||
"h1:2IRBvmWOYrq/ooaYYn2i86jZb7iIUvlg0KlmOMfDHoQ=",
|
||||
"h1:5mucXikk4OcW3un3u94QnMx4AB4Wfih+sXeMd5QxSNk=",
|
||||
"h1:5oU7Zm+2gAVGmxqtJ9E8uTudUkYy/DEn/y3IWphdv4k=",
|
||||
"h1:5w0R4b1/VSzpqQF1tXXPr/qmaQLPVRXamOmPKWFcTk4=",
|
||||
"h1:AEVeJr8xGmwad+JUUQ833C3x5d4W+W2szF5DfwxYppw=",
|
||||
"h1:CPHJ+0zQbS/cX1m55Y90jIOgf1jV3ocUUnqsXAh+9Eg=",
|
||||
"h1:JPewnGDOJudNer5+ghqwXoaJkfot3QRq9uiEYvo+JHU=",
|
||||
"h1:QzbluV2vQLxsJYxjpziQCmPndIoJ/UGS4/UHH/GpwUM=",
|
||||
"h1:TjUNbUdqweRBq/ycQ4ixpNkx5qaYwpXEOn9QCpqNZP8=",
|
||||
"h1:XNbcODP60ajj21N/OO7af8bBg1ltIsYkq9egn7BYbiY=",
|
||||
"h1:tgrbgmX7WYQz9G9ncgu7TkpVB+RlLjJA/Rvp9KPlZH8=",
|
||||
"h1:vLxthX/ZWsOZ+aHKbAMqmNKqD0K5f4nJ8ppy0Ioyup0=",
|
||||
"h1:wZOdGBAZkY8OKEPjKz82j1HloAKOmmvtjWyTxM+I110=",
|
||||
"zh:0f719fa5426bc883e9fa6abf7f6498e48025edafbc29015e2f5c028f1cca3b9d",
|
||||
"zh:1b4d7dafefd6c61764b2f9ed6943ceb9a200dee3590d18747e3a5f6b20ce85e0",
|
||||
"zh:1d23a712984866d29f7b07028a4e99c783c71f1a5dddf08bc3d4e7da9d91a1fa",
|
||||
"zh:257d23d58c3bb024b6bc8eb88736eaf912e934ad47c639d0c3c742bddda849a1",
|
||||
"zh:479860e1a5468f5e04013b9364c9496d7ed0804bf9a1acd8e07558d57609993d",
|
||||
"zh:4cb5e681bf599b411b27c4a2c4066a5fb2ed79aaa3a1a3cb5a30002fec062ce9",
|
||||
"zh:4fb35c3f643dae9f3670d719397a415f815a0b95f8ed7bd8a72f27a94ba78092",
|
||||
"zh:59ba40825ab38db5b4a0989a2db0df35cc15d8984f898176011ba352f27d77b7",
|
||||
"zh:61fc1252eb88088638f4c69ea4e2171cde2e5089fa632ac1e943b13787348f73",
|
||||
"zh:7c5d6dd5f7cbc460e95d368be35c29b4e0402069b8912dbd5d1cd7fa9acef216",
|
||||
"zh:7f76d756240d4284642f359ad470226e5378670239aadc366ef54d9d914d4d2e",
|
||||
"zh:8133ad0814098177e0d067c816ccf1bf48bbadacd18f6f2c808c90447505723b",
|
||||
"zh:c93be06269bb728f1968f8c50506de56c887017ac1d6e4be1f925651d8437eb6",
|
||||
"zh:ef47b78a10a82e6cf53344a6a85a94041c28286c10a70541c564d762f1cfede0",
|
||||
"zh:f5796a53a74999135bd9087aff50fddda59129d09b2f9b1902ff8c0c1e047e48",
|
||||
"h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=",
|
||||
"h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=",
|
||||
"h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=",
|
||||
"h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=",
|
||||
"h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=",
|
||||
"h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=",
|
||||
"h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=",
|
||||
"h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=",
|
||||
"h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=",
|
||||
"h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=",
|
||||
"h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=",
|
||||
"h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=",
|
||||
"h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=",
|
||||
"h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=",
|
||||
"h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=",
|
||||
"zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9",
|
||||
"zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6",
|
||||
"zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab",
|
||||
"zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b",
|
||||
"zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c",
|
||||
"zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4",
|
||||
"zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502",
|
||||
"zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69",
|
||||
"zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6",
|
||||
"zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279",
|
||||
"zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6",
|
||||
"zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7",
|
||||
"zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df",
|
||||
"zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621",
|
||||
"zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424",
|
||||
]
|
||||
}
|
||||
|
||||
provider "registry.terraform.io/bunnyway/bunnynet" {
|
||||
version = "0.17.0"
|
||||
constraints = "~> 0.17"
|
||||
version = "0.18.2"
|
||||
constraints = "~> 0.18"
|
||||
hashes = [
|
||||
"h1:+qDt35lVSK7acw6a1xHuPYrqmZEcHSmtd+6n1TxNuYw=",
|
||||
"h1:1dCu2l4DhPBjizVAH/WwAjT1Xbo52K4PMvHoD5zUhuU=",
|
||||
"h1:Dvn46Auwuel4jqrqZXs2D7kdujNhs17LEmqhuY0k4/4=",
|
||||
"h1:M5eDL3m2uSEr1XATJW0foHzKl8pFhCtgKuOM24bJRwU=",
|
||||
"h1:PddaC7nM/gY4x9i3xy6TxOs9MAu2/6g58Xs/gv4DRV8=",
|
||||
"h1:QVIKiZluI+NQAKu8NpFBl3Nvyx+d81vW9btEUdIQREc=",
|
||||
"h1:S6TnzXHsRoGYvC1vJBkDiVEc0spceksY4n6x5WN5iYw=",
|
||||
"h1:VcxZDWqCWMSjcUsC1K4sB6uYEoeoou+BC0ePoJXmf3A=",
|
||||
"h1:W0y/agBVqls1cJlFGFYMu2VnqoPXFzxVHPIYe3OqfYQ=",
|
||||
"h1:XmNd5fP9a0O77ve5BMQP2vARExgIa7rYl6KvyUYXPSs=",
|
||||
"h1:e0EFKrWSQwaa/kGhnha4DXk4T68Av8QxP84mRSdWC9M=",
|
||||
"h1:eM+/lUiU0pNSgQKoqKPgE3xJrJ0MHIpKG+yhaGB/P0M=",
|
||||
"h1:fPWWA4T0/y7GX+tCGN23l1jODhZ3uCdR/MKgZDXYpAE=",
|
||||
"h1:g+r2GVi4gVC4DuQg3PL70gW9BDskgWUzCBIMXTUq63A=",
|
||||
"h1:gaZ8eALDtVHqykVDHav8004gHiMGaYR/3KwET0FUgao=",
|
||||
"h1:kbqW25eaiv4N/N/z+sxLdJZ15yh5cgnRD/q6RclPMLc=",
|
||||
"h1:rGjxue3mXRyQQqpywTXC4zK//JAtf0Cz7RP+uPMMJjw=",
|
||||
"zh:05943fef14c2028f4722bf078aa1889229e94302f7678cc6f63adb669d8ea612",
|
||||
"zh:26a163930a92a7408f7bbd0130064b84df8a232b500d8c6c3989952986308539",
|
||||
"zh:41305feaaade55391447521ec309f3c038b631ca542907ad95132fab71a7e116",
|
||||
"zh:606919a930f0299948504adbdcd0f239a8af5c418f85741c48f8add370a3d038",
|
||||
"zh:66963d5b445639511939fc508513fd31da3ee1d4ee1a565ee396c9532897a349",
|
||||
"zh:6c981ec0c8545556395c43e2511861ab65ee9ecf2a960480e7889c3af0d23af3",
|
||||
"zh:7334a1bdb726ce1f1bf0a3155f30f84f65206980c229c832ff5f0b0718c44e0b",
|
||||
"zh:75f6c86bf74511e605423332d113711c76c8028361a32282fb3359d6c7ecae9e",
|
||||
"zh:7aebb1a01cfe8be54903853202ae06eba14ad99c37d230ed93ce7d6633e05e9b",
|
||||
"h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=",
|
||||
"h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=",
|
||||
"h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=",
|
||||
"h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=",
|
||||
"h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=",
|
||||
"h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=",
|
||||
"h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=",
|
||||
"h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=",
|
||||
"h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=",
|
||||
"h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=",
|
||||
"h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=",
|
||||
"h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=",
|
||||
"h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=",
|
||||
"h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=",
|
||||
"h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=",
|
||||
"h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=",
|
||||
"h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=",
|
||||
"zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c",
|
||||
"zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178",
|
||||
"zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd",
|
||||
"zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef",
|
||||
"zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738",
|
||||
"zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc",
|
||||
"zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c",
|
||||
"zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f",
|
||||
"zh:9041d0e20c9ceea532de6eebf5cb3a27dad0bb49d3f5b5154be2a08d68fbbf1f",
|
||||
"zh:a6bbf65431a02be4df0ebb1cbe01185ad357ff6e33c01bd0558f59bed90c8f36",
|
||||
"zh:c6d075a31096f080c388dfe46036f451c0cc114c3311a4f46ab8dbe1938a202f",
|
||||
"zh:dd8703f7b55b8bc8e10f8718bea889781100b18e932b04898995b63178c3d36e",
|
||||
"zh:dd92a5cd4e133a4000e7e5bc8cce876ae0ed803543cedd2f3d590661ba244d04",
|
||||
"zh:e024fdf121bebc48c1e6debea344c6d4f174117f3ae605fca6e13b9705d92d22",
|
||||
"zh:ee0e80c31b438e35fa1608f6a2f5824d2806db1e5e8b9f7a90986585c7bcb895",
|
||||
"zh:fc2d4b705411b48f8c045981f9368a3ea2f74969dd6302008c31ff0bedd51f0a",
|
||||
"zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef",
|
||||
"zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3",
|
||||
"zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb",
|
||||
"zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e",
|
||||
"zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11",
|
||||
"zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7",
|
||||
"zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9",
|
||||
"zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1",
|
||||
"zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff",
|
||||
"zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339",
|
||||
]
|
||||
}
|
||||
|
|
|
|||
176
cdn.tf
176
cdn.tf
|
|
@ -32,7 +32,7 @@
|
|||
# cache_stale = ["offline", "updating"]
|
||||
# use_background_update = true
|
||||
|
||||
# block_ips = var.cdn_block_ips
|
||||
# block_ips = var.cdn_block_ips
|
||||
|
||||
# # 50 TB
|
||||
# limit_bandwidth = 50000000000000
|
||||
|
|
@ -52,6 +52,26 @@
|
|||
|
||||
### cran.rpkgs.com
|
||||
|
||||
locals {
|
||||
rpkgs_slots = [
|
||||
for pair in setproduct(
|
||||
["amd64", "arm64"],
|
||||
["resolute", "noble", "jammy", "rhel8", "rhel9", "rhel10", "alpine323", "alpine324"]
|
||||
) : "${pair[0]}/${pair[1]}"
|
||||
]
|
||||
|
||||
# The supported R minors: the current one plus the two previous, which is
|
||||
# exactly what build-env-images installs as R_VERSION_LATEST / PREV1 / PREV2.
|
||||
# These must stay in step. A minor listed here without a published index
|
||||
# sends those clients to a 404; a published minor missing from this list
|
||||
# sends them to CRAN for sources instead of serving the binaries we built.
|
||||
rpkgs_supported_minors = ["4.4", "4.5", "4.6"]
|
||||
|
||||
# bunny.net serves every pull zone on <name>.b-cdn.net, so staging needs no
|
||||
# DNS record and is never advertised.
|
||||
rpkgs_test_hostname = "cran-rpkgs-test.b-cdn.net"
|
||||
}
|
||||
|
||||
# The edge middleware that resolves the bare cran.rpkgs.com form to an
|
||||
# <arch>/<os> slot and routes PACKAGES* to the per-R-minor slot. The source of
|
||||
# truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release.
|
||||
|
|
@ -75,6 +95,13 @@ resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" {
|
|||
required = false
|
||||
}
|
||||
|
||||
resource "bunnynet_compute_script_variable" "rpkgs_router_known_minors" {
|
||||
script = bunnynet_compute_script.rpkgs_router.id
|
||||
name = "KNOWN_MINORS"
|
||||
default_value = join(",", local.rpkgs_supported_minors)
|
||||
required = false
|
||||
}
|
||||
|
||||
resource "bunnynet_pullzone" "cran_rpkgs_com" {
|
||||
name = "cran-rpkgs"
|
||||
|
||||
|
|
@ -82,7 +109,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" {
|
|||
|
||||
cache_expiration_time = 31919000
|
||||
websockets_enabled = false
|
||||
errorpage_whitelabel = true
|
||||
errorpage_whitelabel = true
|
||||
|
||||
origin {
|
||||
type = "OriginUrl"
|
||||
|
|
@ -147,6 +174,151 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" {
|
|||
tls_enabled = true
|
||||
}
|
||||
|
||||
### Staging zone for edge-router changes
|
||||
|
||||
# Every published <arch>/<os> slot. The staging zone enables per-minor routing
|
||||
# for all of them at once; production adopts the same list only after
|
||||
# `scripts/verify-r-minor-routing.sh --live` passes against staging.
|
||||
|
||||
# A second copy of the same router, bound to the same B2 origin, so UNION_SLOTS
|
||||
# can be exercised end to end before production is touched.
|
||||
resource "bunnynet_compute_script" "rpkgs_router_test" {
|
||||
type = "middleware"
|
||||
name = "rpkgs-router-test"
|
||||
content = file("${path.module}/edge/rpkgs-router.ts")
|
||||
}
|
||||
|
||||
resource "bunnynet_compute_script_variable" "rpkgs_router_test_union_slots" {
|
||||
script = bunnynet_compute_script.rpkgs_router_test.id
|
||||
name = "UNION_SLOTS"
|
||||
default_value = join(",", local.rpkgs_slots)
|
||||
required = false
|
||||
}
|
||||
|
||||
# Without this the staging zone rewrites to PUBLIC_CDN_ORIGIN, so its redirects
|
||||
# land on production and the test silently measures the wrong system.
|
||||
resource "bunnynet_compute_script_variable" "rpkgs_router_test_extra_hosts" {
|
||||
script = bunnynet_compute_script.rpkgs_router_test.id
|
||||
name = "EXTRA_PUBLIC_HOSTS"
|
||||
default_value = local.rpkgs_test_hostname
|
||||
required = false
|
||||
}
|
||||
|
||||
resource "bunnynet_compute_script_variable" "rpkgs_router_test_known_minors" {
|
||||
script = bunnynet_compute_script.rpkgs_router_test.id
|
||||
name = "KNOWN_MINORS"
|
||||
default_value = join(",", local.rpkgs_supported_minors)
|
||||
required = false
|
||||
}
|
||||
|
||||
resource "bunnynet_pullzone" "cran_rpkgs_test" {
|
||||
name = "cran-rpkgs-test"
|
||||
|
||||
cache_errors = false
|
||||
|
||||
cache_expiration_time = 31919000
|
||||
websockets_enabled = false
|
||||
errorpage_whitelabel = true
|
||||
|
||||
origin {
|
||||
type = "OriginUrl"
|
||||
url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com"
|
||||
middleware_script = bunnynet_compute_script.rpkgs_router_test.id
|
||||
}
|
||||
|
||||
routing {
|
||||
filters = [
|
||||
"scripting",
|
||||
]
|
||||
}
|
||||
|
||||
s3_auth_enabled = true
|
||||
s3_auth_key = var.B2_S3_ACCESS_KEY
|
||||
s3_auth_secret = var.B2_S3_SECRET_KEY
|
||||
s3_auth_region = "eu-central-003"
|
||||
|
||||
cache_enabled = true
|
||||
request_coalescing_enabled = true
|
||||
block_post_requests = true
|
||||
|
||||
cache_vary_headers = ["User-Agent"]
|
||||
|
||||
# Staging carries only synthetic verification traffic, so the production
|
||||
# ceilings would be pure headroom.
|
||||
limit_requests = 500
|
||||
limit_connections = 100
|
||||
|
||||
safehop_enabled = true
|
||||
add_canonical_header = true
|
||||
cache_stale = ["offline", "updating"]
|
||||
block_ips = var.cdn_block_ips
|
||||
|
||||
# 1 TB
|
||||
limit_bandwidth = 1000000000000
|
||||
|
||||
block_root_path = true
|
||||
}
|
||||
|
||||
|
||||
# Alliance SwissPass historically used a separate, manually configured pull
|
||||
# zone. Adopt it so both public repositories use the same B2 origin, middleware
|
||||
# release and cache behavior.
|
||||
import {
|
||||
to = bunnynet_pullzone.cran_allianceswisspass
|
||||
id = "3265648"
|
||||
}
|
||||
|
||||
resource "bunnynet_pullzone" "cran_allianceswisspass" {
|
||||
name = "cran-allianceswisspass"
|
||||
|
||||
cache_errors = false
|
||||
cache_expiration_time = 31919000
|
||||
websockets_enabled = false
|
||||
errorpage_whitelabel = true
|
||||
|
||||
origin {
|
||||
type = "OriginUrl"
|
||||
url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com"
|
||||
middleware_script = bunnynet_compute_script.rpkgs_router.id
|
||||
}
|
||||
|
||||
routing {
|
||||
filters = [
|
||||
"scripting",
|
||||
]
|
||||
}
|
||||
|
||||
s3_auth_enabled = true
|
||||
s3_auth_key = var.B2_S3_ACCESS_KEY
|
||||
s3_auth_secret = var.B2_S3_SECRET_KEY
|
||||
s3_auth_region = "eu-central-003"
|
||||
|
||||
cache_enabled = true
|
||||
request_coalescing_enabled = true
|
||||
block_post_requests = true
|
||||
cache_vary_headers = ["User-Agent"]
|
||||
|
||||
limit_requests = 5000
|
||||
limit_connections = 1000
|
||||
|
||||
safehop_enabled = true
|
||||
add_canonical_header = true
|
||||
cache_stale = ["offline", "updating"]
|
||||
block_ips = var.cdn_block_ips
|
||||
|
||||
# 50 TB
|
||||
limit_bandwidth = 50000000000000
|
||||
|
||||
block_root_path = true
|
||||
}
|
||||
|
||||
resource "bunnynet_pullzone_hostname" "cran_allianceswisspass" {
|
||||
pullzone = bunnynet_pullzone.cran_allianceswisspass.id
|
||||
name = "cran.allianceswisspass.devxy.io"
|
||||
force_ssl = true
|
||||
tls_enabled = true
|
||||
}
|
||||
|
||||
# resource "bunnynet_storage_zone" "devxy-r-binaries" {
|
||||
# name = "devxy-r-binaries-storage"
|
||||
# region = "DE"
|
||||
|
|
|
|||
|
|
@ -17,7 +17,12 @@ const UNION_SLOTS = 'amd64/alpine324';
|
|||
|
||||
const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)';
|
||||
const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)';
|
||||
const UA_R43_MUSL = 'R (4.3.3 x86_64-pc-linux-musl x86_64 linux-musl)';
|
||||
const UA_R47_MUSL = 'R (4.7.0 x86_64-pc-linux-musl x86_64 linux-musl)';
|
||||
const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24';
|
||||
const UA_R45_RESOLUTE = 'R/4.5.3 (Ubuntu 26.04) (aarch64-unknown-linux-gnu aarch64 linux-gnu)';
|
||||
const UA_R45_FUTURE_UBUNTU =
|
||||
'R/4.5.3 (Ubuntu 28.04; codename=dynamic-dugong) (aarch64-unknown-linux-gnu aarch64 linux-gnu)';
|
||||
const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)';
|
||||
const UA_CURL = 'curl/8.0.1';
|
||||
|
||||
|
|
@ -93,6 +98,33 @@ Deno.test('rpkgs-router', async (t) => {
|
|||
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`);
|
||||
});
|
||||
|
||||
// We publish binaries only for the supported window. An excluded minor has
|
||||
// no slot we can serve safely, so it goes to CRAN for sources rather than
|
||||
// to a 404 or to binaries built under another minor.
|
||||
await t.step('sends an excluded R minor to CRAN for the index', async () => {
|
||||
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R43_MUSL);
|
||||
assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz');
|
||||
});
|
||||
|
||||
await t.step('sends a future R minor to CRAN too', async () => {
|
||||
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R47_MUSL);
|
||||
assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz');
|
||||
});
|
||||
|
||||
// The index and the tarballs R resolves against it have to come from the
|
||||
// same place. Serving one from CRAN and the other from here would hand R a
|
||||
// binary where it expects a source tarball.
|
||||
await t.step('sends an excluded minor to CRAN for tarballs as well', async () => {
|
||||
const res = await probe(`${SLOT}/foo_1.0.tar.gz`, UA_R43_MUSL);
|
||||
assertEquals(res.location, 'https://cran.r-project.org/src/contrib/foo_1.0.tar.gz');
|
||||
});
|
||||
|
||||
await t.step('leaves an excluded minor alone on a slot outside UNION_SLOTS', async () => {
|
||||
const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R43_MUSL);
|
||||
assertEquals(res.location, null);
|
||||
assertEquals(res.status, 200);
|
||||
});
|
||||
|
||||
await t.step('routes PACKAGES and PACKAGES.rds too', async () => {
|
||||
for (const file of ['PACKAGES', 'PACKAGES.rds']) {
|
||||
const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL);
|
||||
|
|
@ -117,6 +149,13 @@ Deno.test('rpkgs-router', async (t) => {
|
|||
assertEquals(res.status, 200);
|
||||
});
|
||||
|
||||
await t.step('serves an archived binary when it exists', async () => {
|
||||
const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`;
|
||||
const res = await probe(path, UA_R45_MUSL);
|
||||
assertEquals(res.status, 200);
|
||||
assertEquals(res.location, null);
|
||||
});
|
||||
|
||||
await t.step('does not redirect a path already under a minor', async () => {
|
||||
const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL);
|
||||
assertEquals(res.location, null);
|
||||
|
|
@ -134,6 +173,16 @@ Deno.test('rpkgs-router', async (t) => {
|
|||
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`);
|
||||
});
|
||||
|
||||
await t.step('resolves Ubuntu 26.04 to the resolute slot', async () => {
|
||||
const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_RESOLUTE);
|
||||
assertEquals(res.location, 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz');
|
||||
});
|
||||
|
||||
await t.step('resolves a future Ubuntu release from its codename', async () => {
|
||||
const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_FUTURE_UBUNTU);
|
||||
assertEquals(res.location, 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz');
|
||||
});
|
||||
|
||||
await t.step('sends an unidentifiable distro to CRAN', async () => {
|
||||
const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL);
|
||||
assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz');
|
||||
|
|
|
|||
|
|
@ -27,6 +27,17 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12';
|
|||
|
||||
const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com';
|
||||
const CRAN_ORIGIN = 'https://cran.r-project.org';
|
||||
const PUBLIC_CDN_HOSTS = new Set([
|
||||
'cran.rpkgs.com',
|
||||
'cran.allianceswisspass.devxy.io',
|
||||
// Staging hostnames, so the identical script can run on a test pull zone and
|
||||
// redirect within itself. Without this a test zone rewrites to
|
||||
// PUBLIC_CDN_ORIGIN, quietly exercising production instead of itself.
|
||||
...(Deno.env.get('EXTRA_PUBLIC_HOSTS') ?? '')
|
||||
.split(',')
|
||||
.map((host) => host.trim())
|
||||
.filter((host) => host.length > 0),
|
||||
]);
|
||||
|
||||
/** Slots ("<arch>/<os>", comma separated) whose per-minor index is a union. */
|
||||
const UNION_SLOTS = new Set(
|
||||
|
|
@ -36,6 +47,21 @@ const UNION_SLOTS = new Set(
|
|||
.filter((slot) => slot.length > 0),
|
||||
);
|
||||
|
||||
/**
|
||||
* R minors for which a per-minor index is actually published.
|
||||
*
|
||||
* contribPath() has no way to probe the origin, so a minor that is not
|
||||
* published here must fall back to the flat index. Routing an unlisted minor
|
||||
* would send that client to a 404 and it would see no packages at all - a
|
||||
* silent, total failure rather than a degraded one.
|
||||
*/
|
||||
const KNOWN_MINORS = new Set(
|
||||
(Deno.env.get('KNOWN_MINORS') ?? '4.4,4.5,4.6')
|
||||
.split(',')
|
||||
.map((minor) => minor.trim())
|
||||
.filter((minor) => minor.length > 0),
|
||||
);
|
||||
|
||||
/** `/<arch>/<os>/latest/src/contrib[/<rest>]` */
|
||||
const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/;
|
||||
|
||||
|
|
@ -47,13 +73,19 @@ const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/;
|
|||
|
||||
const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/;
|
||||
|
||||
/** A binary archive URL whose upstream source counterpart CRAN can serve. */
|
||||
const ARCHIVE_TARBALL_REGEX =
|
||||
/^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/;
|
||||
|
||||
const MACOS_BIN_REGEX =
|
||||
/^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/;
|
||||
|
||||
const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i;
|
||||
|
||||
const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i;
|
||||
const UBUNTU_CODENAME_REGEX = /Ubuntu [\d.]+;\s*codename=([a-z][a-z0-9-]*)/i;
|
||||
const UBUNTU_CODENAMES: Record<string, string> = {
|
||||
'26.04': 'resolute',
|
||||
'24.04': 'noble',
|
||||
'22.04': 'jammy',
|
||||
};
|
||||
|
|
@ -85,6 +117,22 @@ function redirectTo(location: string, status = 302): Response {
|
|||
});
|
||||
}
|
||||
|
||||
function publicCdnOrigin(url: URL): string {
|
||||
return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the client reports an R minor that we deliberately do not serve.
|
||||
*
|
||||
* A client that reports no minor at all is not "unsupported": non-R fetchers
|
||||
* (mirror scripts, image builds) must keep getting the flat slot. Only a
|
||||
* known-and-excluded minor falls through to CRAN.
|
||||
*/
|
||||
function isExcludedMinor(userAgent: string): boolean {
|
||||
const rMinor = extractRMinor(userAgent);
|
||||
return rMinor !== null && !KNOWN_MINORS.has(rMinor);
|
||||
}
|
||||
|
||||
function extractRMinor(userAgent: string): string | null {
|
||||
for (const regex of R_MINOR_REGEXES) {
|
||||
const match = userAgent.match(regex);
|
||||
|
|
@ -127,6 +175,11 @@ function parseSlot(userAgent: string): string | null {
|
|||
|
||||
const ubuntu = userAgent.match(UBUNTU_REGEX);
|
||||
if (ubuntu) {
|
||||
const codenameMatch = userAgent.match(UBUNTU_CODENAME_REGEX);
|
||||
if (codenameMatch) {
|
||||
return `${arch}/${codenameMatch[1].toLowerCase()}`;
|
||||
}
|
||||
|
||||
const codename = UBUNTU_CODENAMES[ubuntu[1]];
|
||||
if (codename) {
|
||||
return `${arch}/${codename}`;
|
||||
|
|
@ -161,8 +214,8 @@ function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver:
|
|||
* The contrib path a request should be served from, relative to the slot.
|
||||
*
|
||||
* Returns the per-minor path for an index file when the slot is known to carry
|
||||
* a union index and the client's R minor is known; otherwise the flat path,
|
||||
* which is what every client sees today.
|
||||
* a union index and the client's R minor is one we publish; otherwise the flat
|
||||
* path, which is what every client sees today.
|
||||
*/
|
||||
function contribPath(slot: string, rest: string, userAgent: string): string {
|
||||
const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`;
|
||||
|
|
@ -172,7 +225,7 @@ function contribPath(slot: string, rest: string, userAgent: string): string {
|
|||
}
|
||||
|
||||
const rMinor = extractRMinor(userAgent);
|
||||
return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat;
|
||||
return rMinor && KNOWN_MINORS.has(rMinor) ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat;
|
||||
}
|
||||
|
||||
BunnySDK.net.http
|
||||
|
|
@ -181,15 +234,14 @@ BunnySDK.net.http
|
|||
const url = new URL(ctx.request.url);
|
||||
const path = normalizePathname(url.pathname);
|
||||
const userAgent = ctx.request.headers.get('User-Agent') || '';
|
||||
const publicOrigin = publicCdnOrigin(url);
|
||||
|
||||
// macOS clients are served from CRAN's own binary tree.
|
||||
const srcContrib = path.match(SRC_CONTRIB_REGEX);
|
||||
if (srcContrib && /darwin/.test(userAgent)) {
|
||||
const mac = parseMacUserAgent(userAgent);
|
||||
if (mac) {
|
||||
return Promise.resolve(
|
||||
redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`),
|
||||
);
|
||||
return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -209,11 +261,21 @@ BunnySDK.net.http
|
|||
return Promise.resolve(ctx.request);
|
||||
}
|
||||
|
||||
// An R minor outside the supported window has no binaries we can safely
|
||||
// serve, so the whole interaction goes to CRAN: the index and the
|
||||
// tarballs R will resolve against it. Serving the index from CRAN but
|
||||
// tarballs from here would hand R a binary where it expects a source
|
||||
// tarball, which fails in a far more confusing way than not being
|
||||
// served at all.
|
||||
if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) {
|
||||
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}/src/contrib/${rest}`));
|
||||
}
|
||||
|
||||
const target = contribPath(slot, rest, userAgent);
|
||||
if (target === path) {
|
||||
return Promise.resolve(ctx.request);
|
||||
}
|
||||
return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`));
|
||||
return Promise.resolve(redirectTo(`${publicOrigin}${target}`));
|
||||
}
|
||||
|
||||
// The bare `https://cran.rpkgs.com` form, resolved from the User-Agent.
|
||||
|
|
@ -223,13 +285,32 @@ BunnySDK.net.http
|
|||
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`));
|
||||
}
|
||||
|
||||
if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) {
|
||||
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`));
|
||||
}
|
||||
|
||||
const rest = srcContrib ? srcContrib[1] : '';
|
||||
return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`));
|
||||
return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`));
|
||||
}
|
||||
|
||||
return Promise.resolve(ctx.request);
|
||||
})
|
||||
.onOriginResponse((ctx) => {
|
||||
.onOriginResponse(async (ctx) => {
|
||||
const path = normalizePathname(new URL(ctx.request.url).pathname);
|
||||
const archive = path.match(ARCHIVE_TARBALL_REGEX);
|
||||
|
||||
// Binary archives can be incomplete when an older build never succeeded.
|
||||
// Preserve renv/remotes version restores by falling back to CRAN's source
|
||||
// package only for an absent archived tarball. A requested version can be
|
||||
// either archived upstream or still current, so probe the archive first.
|
||||
// Other 404s remain visible.
|
||||
if (ctx.response.status === 404 && archive) {
|
||||
const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`;
|
||||
const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' });
|
||||
const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`;
|
||||
return redirectTo(sourceUrl);
|
||||
}
|
||||
|
||||
ctx.response.headers.append('X-Via', 'MyMiddleware');
|
||||
return Promise.resolve(ctx.response);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -40,15 +40,11 @@ index 6f6a745..e407986 100644
|
|||
if (is.null(name))
|
||||
return(tbbRoot)
|
||||
|
||||
@@ -58,7 +58,7 @@ tbbCxxFlags <- function() {
|
||||
flags <- c("-DRCPP_PARALLEL_USE_TBB=1")
|
||||
|
||||
@@ -58,3 +58,3 @@ tbbCxxFlags <- function() {
|
||||
# if TBB_INC is set, apply those library paths
|
||||
- tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC)
|
||||
+ tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC)
|
||||
if (!file.exists(tbbInc)) {
|
||||
tbbInc <- system.file("include", package = "RcppParallel")
|
||||
}
|
||||
@@ -117,7 +117,7 @@ tbbLdFlags <- function() {
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[
|
||||
{
|
||||
"package": "RcppParallel",
|
||||
"versions": ">=6.0.0",
|
||||
"versions": "6.2.1",
|
||||
"platforms": ["*"],
|
||||
"env": {},
|
||||
"configure_args": [],
|
||||
|
|
|
|||
87
local/rebuild-missing-helpers.R
Normal file
87
local/rebuild-missing-helpers.R
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
# Pure helpers for local/rebuild-missing.R, kept separate so local/tests can
|
||||
# source them without executing a rebuild.
|
||||
|
||||
# Interleaved slice of the rebuild list.
|
||||
#
|
||||
# The list is alphabetical and build cost clusters by name (Rcpp*, Bioc*,
|
||||
# rstan*), so contiguous thirds would be badly unbalanced. Interleaving also
|
||||
# makes each shard's progress counter representative of the slot as a whole.
|
||||
shard_slice <- function(pkgs, split_into, split_index) {
|
||||
split_into <- as.integer(split_into)
|
||||
split_index <- as.integer(split_index)
|
||||
if (is.na(split_into) || is.na(split_index)) {
|
||||
stop("shard_slice(): split_into and split_index must be integers")
|
||||
}
|
||||
if (split_into < 1L || split_index < 1L || split_index > split_into) {
|
||||
stop(sprintf(
|
||||
"shard_slice(): need 1 <= split_index <= split_into, got %s of %s",
|
||||
split_index,
|
||||
split_into
|
||||
))
|
||||
}
|
||||
# seq() errors on a descending range, which is what an empty list or a shard
|
||||
# index past the end would produce.
|
||||
if (length(pkgs) < split_index) {
|
||||
return(pkgs[0L])
|
||||
}
|
||||
pkgs[seq.int(split_index, length(pkgs), by = split_into)]
|
||||
}
|
||||
|
||||
# Packages that still need building, decided from the bucket rather than from
|
||||
# remembered progress.
|
||||
#
|
||||
# This is bincraft's `check_s3_root_package()` evaluated in bulk: an object
|
||||
# whose ETag equals CRAN's published MD5sum is byte-identical to CRAN's source,
|
||||
# so the build that was supposed to replace it has not happened yet.
|
||||
#
|
||||
# `etag_by_file` named by `<pkg>_<ver>.tar.gz`, values are unquoted ETags
|
||||
# `cran_version` named by package
|
||||
# `cran_md5` named by `<pkg>_<ver>`
|
||||
#
|
||||
# Unknown always means "already a binary", never "rebuild it", so an unreadable
|
||||
# CRAN index or a multipart ETag can never mass-schedule work.
|
||||
outstanding_packages <- function(pkgs, etag_by_file, cran_version, cran_md5) {
|
||||
if (length(pkgs) == 0L) {
|
||||
return(pkgs)
|
||||
}
|
||||
|
||||
# An empty table indexes to zero length rather than to NA, which would
|
||||
# recycle the whole result away and silently report "nothing to build".
|
||||
lookup <- function(table, key) {
|
||||
if (length(table) == 0L) {
|
||||
return(rep(NA_character_, length(key)))
|
||||
}
|
||||
unname(as.character(table[key]))
|
||||
}
|
||||
|
||||
version <- lookup(cran_version, pkgs)
|
||||
file <- sprintf("%s_%s.tar.gz", pkgs, version)
|
||||
etag <- lookup(etag_by_file, file)
|
||||
md5 <- lookup(cran_md5, paste(pkgs, version, sep = "_"))
|
||||
|
||||
# No CRAN version means the package cannot be resolved to a tarball at all;
|
||||
# leave it in and let bincraft report why.
|
||||
unresolved <- is.na(version)
|
||||
# No object at the key: never built, so it is outstanding by definition.
|
||||
absent <- !unresolved & is.na(etag)
|
||||
# A multipart upload carries a compound ETag rather than an MD5.
|
||||
unknown <- !is.na(etag) & grepl("-", etag, fixed = TRUE)
|
||||
|
||||
is_source <- !unresolved &
|
||||
!is.na(etag) &
|
||||
!unknown &
|
||||
!is.na(md5) &
|
||||
etag == md5
|
||||
|
||||
pkgs[unresolved | absent | is_source]
|
||||
}
|
||||
|
||||
parse_rebuild_args <- function(args) {
|
||||
pos <- args[!startsWith(args, "--")]
|
||||
budget <- as.numeric(pos[3L])
|
||||
list(
|
||||
split_into = as.integer(pos[1L]),
|
||||
split_index = as.integer(pos[2L]),
|
||||
budget_hours = if (is.na(budget)) 20 else budget
|
||||
)
|
||||
}
|
||||
194
local/rebuild-missing.R
Normal file
194
local/rebuild-missing.R
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
### Rebuild one shard of a slot's missing-binary list.
|
||||
#
|
||||
# Usage: Rscript local/rebuild-missing.R <split_into> <split_index> [budget_hours]
|
||||
#
|
||||
# The list itself comes from local/fetch-rebuild-packages-from-issue.R, which
|
||||
# writes $REBUILD_PKG_LIST (default /tmp/rebuild_pkgs.txt).
|
||||
#
|
||||
# Two properties matter here and are the reason this is a script rather than an
|
||||
# `R -q -e` argument in the pipeline:
|
||||
#
|
||||
# * it is restartable. The outstanding set is re-derived from the bucket on
|
||||
# every start, so a shard that died resumes where it stopped without any
|
||||
# progress file, and without replaying thousands of per-package HEADs.
|
||||
# * it terminates. A wall-clock budget stops the loop cleanly instead of the
|
||||
# run having to be killed, which is what previously skipped the re-index and
|
||||
# CDN purge and left rebuilt binaries hidden behind stale edge copies.
|
||||
|
||||
options(error = function() {
|
||||
cat("ERROR:", geterrmessage(), "\n", file = stdout())
|
||||
traceback(2)
|
||||
q(status = 1)
|
||||
})
|
||||
|
||||
library(bincraft, quietly = TRUE)
|
||||
|
||||
source(file.path("local", "rebuild-missing-helpers.R"))
|
||||
|
||||
args <- parse_rebuild_args(commandArgs(trailingOnly = TRUE))
|
||||
if (is.na(args$split_into) || is.na(args$split_index)) {
|
||||
stop("usage: rebuild-missing.R <split_into> <split_index> [budget_hours]")
|
||||
}
|
||||
|
||||
list_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt")
|
||||
pkgs <- if (file.exists(list_file)) readLines(list_file) else character(0)
|
||||
pkgs <- pkgs[nzchar(pkgs)]
|
||||
if (length(pkgs) == 0L) {
|
||||
cat("Nothing to rebuild\n")
|
||||
q("no")
|
||||
}
|
||||
|
||||
excluded <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]]
|
||||
pkgs <- setdiff(pkgs, excluded)
|
||||
|
||||
mine <- shard_slice(pkgs, args$split_into, args$split_index)
|
||||
cat(sprintf(
|
||||
"Shard %s/%s: %s of %s listed packages\n",
|
||||
args$split_index,
|
||||
args$split_into,
|
||||
length(mine),
|
||||
length(pkgs)
|
||||
))
|
||||
|
||||
### Resume: ask the bucket what is still outstanding
|
||||
|
||||
codename <- bincraft::set_codename(NULL)
|
||||
local_machine <- Sys.info()[["machine"]]
|
||||
arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64"
|
||||
slot_dir <- sprintf(
|
||||
"devxy-rpkgs-binaries/%s/%s/latest/src/contrib",
|
||||
arch,
|
||||
codename
|
||||
)
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region_name = "eu-central-003",
|
||||
refresh = TRUE
|
||||
)
|
||||
|
||||
# One paginated listing instead of a HEAD per package. Not recursed: the
|
||||
# rebuild passes no `is_r_minor_sensitive`, so it only ever targets the flat
|
||||
# path, and the resume filter matches that scope deliberately.
|
||||
info <- tryCatch(s3fs::s3_dir_info(slot_dir), error = function(e) NULL)
|
||||
etag_by_file <- if (is.null(info) || nrow(info) == 0L) {
|
||||
cat(sprintf(
|
||||
"WARNING: could not list %s; building the whole shard\n",
|
||||
slot_dir
|
||||
))
|
||||
stats::setNames(character(), character())
|
||||
} else {
|
||||
stats::setNames(
|
||||
gsub('^"|"$', "", as.character(info$etag)),
|
||||
basename(as.character(info$uri))
|
||||
)
|
||||
}
|
||||
|
||||
cran <- tryCatch(
|
||||
{
|
||||
con <- gzcon(url(
|
||||
"https://cloud.r-project.org/src/contrib/PACKAGES.gz",
|
||||
open = "rb"
|
||||
))
|
||||
on.exit(close(con), add = TRUE)
|
||||
read.dcf(con, fields = c("Package", "Version", "MD5sum"))
|
||||
},
|
||||
error = function(e) {
|
||||
cat(sprintf(
|
||||
"WARNING: could not read CRAN's index (%s)\n",
|
||||
conditionMessage(e)
|
||||
))
|
||||
NULL
|
||||
}
|
||||
)
|
||||
cran_version <- stats::setNames(character(), character())
|
||||
cran_md5 <- stats::setNames(character(), character())
|
||||
if (!is.null(cran)) {
|
||||
cran_version <- stats::setNames(
|
||||
as.character(cran[, "Version"]),
|
||||
as.character(cran[, "Package"])
|
||||
)
|
||||
keep <- !is.na(cran[, "MD5sum"])
|
||||
cran_md5 <- stats::setNames(
|
||||
as.character(cran[keep, "MD5sum"]),
|
||||
paste(cran[keep, "Package"], cran[keep, "Version"], sep = "_")
|
||||
)
|
||||
}
|
||||
|
||||
before <- length(mine)
|
||||
mine <- outstanding_packages(mine, etag_by_file, cran_version, cran_md5)
|
||||
cat(sprintf(
|
||||
"Resume: %s of %s already carry a binary; %s outstanding\n",
|
||||
before - length(mine),
|
||||
before,
|
||||
length(mine)
|
||||
))
|
||||
|
||||
if (length(mine) == 0L) {
|
||||
cat("Nothing outstanding for this shard\n")
|
||||
q("no")
|
||||
}
|
||||
|
||||
### Build
|
||||
|
||||
options(
|
||||
crayon.enabled = TRUE,
|
||||
Ncpus = as.integer(Sys.getenv("NCPUS", "2")),
|
||||
future.globals.onReference = NULL
|
||||
)
|
||||
|
||||
started <- Sys.time()
|
||||
n <- length(mine)
|
||||
completed <- 0L
|
||||
for (i in seq_along(mine)) {
|
||||
elapsed <- as.numeric(difftime(Sys.time(), started, units = "hours"))
|
||||
if (elapsed > args$budget_hours) {
|
||||
cat(sprintf(
|
||||
"Budget of %sh reached after %d/%d packages; stopping cleanly. The next run resumes from the bucket.\n",
|
||||
args$budget_hours,
|
||||
completed,
|
||||
n
|
||||
))
|
||||
break
|
||||
}
|
||||
|
||||
x <- mine[i]
|
||||
cat(sprintf("[%d/%d] %s\n", i, n, x))
|
||||
tryCatch(
|
||||
bincraft::build_binary_package(
|
||||
x,
|
||||
tag_limit = 1L,
|
||||
patches = "local/patches",
|
||||
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
s3_region = "eu-central-003",
|
||||
s3_bucket = "devxy-rpkgs-binaries",
|
||||
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
metadata_db_host = "r-binaries.devxy.io",
|
||||
metadata_db_name = "build_metadata",
|
||||
metadata_db_table = "single_builds",
|
||||
metadata_db_user = "rpkgs",
|
||||
metadata_db_password = Sys.getenv("PGPASS"),
|
||||
metadata_db_sslmode = "require",
|
||||
metadata_db_port = 15432,
|
||||
archive = TRUE,
|
||||
upload = TRUE,
|
||||
store_build_metadata = TRUE
|
||||
),
|
||||
error = function(e) {
|
||||
cat(sprintf("ERROR building %s - %s\n", x, conditionMessage(e)))
|
||||
}
|
||||
)
|
||||
completed <- completed + 1L
|
||||
}
|
||||
|
||||
cat(sprintf(
|
||||
"Shard %s/%s finished: %d/%d packages processed in %.1fh\n",
|
||||
args$split_index,
|
||||
args$split_into,
|
||||
completed,
|
||||
n,
|
||||
as.numeric(difftime(Sys.time(), started, units = "hours"))
|
||||
))
|
||||
94
local/tests/test-rebuild-missing.R
Normal file
94
local/tests/test-rebuild-missing.R
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
source(file.path("..", "rebuild-missing-helpers.R"))
|
||||
|
||||
test_that("shard_slice partitions the list without gaps or overlap", {
|
||||
pkgs <- letters[1:10]
|
||||
parts <- lapply(1:3, function(i) shard_slice(pkgs, 3, i))
|
||||
|
||||
expect_identical(parts[[1]], c("a", "d", "g", "j"))
|
||||
expect_identical(parts[[2]], c("b", "e", "h"))
|
||||
expect_identical(parts[[3]], c("c", "f", "i"))
|
||||
|
||||
expect_identical(sort(unlist(parts)), sort(pkgs))
|
||||
expect_identical(anyDuplicated(unlist(parts)), 0L)
|
||||
})
|
||||
|
||||
test_that("shard_slice is deterministic and survives short lists", {
|
||||
expect_identical(
|
||||
shard_slice(letters[1:10], 3, 2),
|
||||
shard_slice(letters[1:10], 3, 2)
|
||||
)
|
||||
expect_identical(shard_slice(character(0), 3, 1), character(0))
|
||||
# more shards than packages: the tail shards get nothing rather than erroring
|
||||
expect_identical(shard_slice(c("a"), 3, 1), "a")
|
||||
expect_identical(shard_slice(c("a"), 3, 2), character(0))
|
||||
})
|
||||
|
||||
test_that("shard_slice rejects an out-of-range index", {
|
||||
expect_error(shard_slice(letters, 3, 4), "split_index")
|
||||
expect_error(shard_slice(letters, 3, 0), "split_index")
|
||||
})
|
||||
|
||||
test_that("outstanding_packages keeps source fallbacks and drops real binaries", {
|
||||
cran_version <- c(httr = "1.4.8", R6 = "2.6.1", curl = "7.1.0")
|
||||
cran_md5 <- c(
|
||||
httr_1.4.8 = "8756015b94a9cff6f410ca4de8557f12",
|
||||
R6_2.6.1 = "f01b1787f12797c29194d63c9afd5d70",
|
||||
curl_7.1.0 = "8af2ccbf5d85dc18866f45f1f26f348d"
|
||||
)
|
||||
etag <- c(
|
||||
# byte-identical to CRAN: the build never happened
|
||||
"httr_1.4.8.tar.gz" = "8756015b94a9cff6f410ca4de8557f12",
|
||||
# a real binary was published
|
||||
"R6_2.6.1.tar.gz" = "9d6087ee9adda3f0a3b8067cfc652c05"
|
||||
# curl has no object at all
|
||||
)
|
||||
|
||||
out <- outstanding_packages(
|
||||
c("httr", "R6", "curl"),
|
||||
etag,
|
||||
cran_version,
|
||||
cran_md5
|
||||
)
|
||||
expect_identical(out, c("httr", "curl"))
|
||||
})
|
||||
|
||||
test_that("outstanding_packages treats unknowns as already built", {
|
||||
cran_version <- c(a = "1.0", b = "1.0")
|
||||
cran_md5 <- c(a_1.0 = "aaaa")
|
||||
|
||||
# a multipart ETag carries no MD5, and `b` is missing from CRAN's index:
|
||||
# neither may schedule a rebuild
|
||||
etag <- c("a_1.0.tar.gz" = "abc-3", "b_1.0.tar.gz" = "bbbb")
|
||||
|
||||
expect_identical(
|
||||
outstanding_packages(c("a", "b"), etag, cran_version, cran_md5),
|
||||
character(0)
|
||||
)
|
||||
})
|
||||
|
||||
test_that("outstanding_packages keeps a package CRAN has no version for", {
|
||||
out <- outstanding_packages(
|
||||
"ghost",
|
||||
c(),
|
||||
c(other = "1.0"),
|
||||
c(other_1.0 = "aaaa")
|
||||
)
|
||||
expect_identical(out, "ghost")
|
||||
})
|
||||
|
||||
test_that("outstanding_packages handles an empty list", {
|
||||
expect_identical(
|
||||
outstanding_packages(character(0), c(), c(), c()),
|
||||
character(0)
|
||||
)
|
||||
})
|
||||
|
||||
test_that("parse_rebuild_args defaults the budget", {
|
||||
a <- parse_rebuild_args(c("3", "2"))
|
||||
expect_identical(a$split_into, 3L)
|
||||
expect_identical(a$split_index, 2L)
|
||||
expect_identical(a$budget_hours, 20)
|
||||
|
||||
b <- parse_rebuild_args(c("3", "2", "1.5"))
|
||||
expect_identical(b$budget_hours, 1.5)
|
||||
})
|
||||
|
|
@ -84,9 +84,20 @@ trap 'rm -rf "$project_dir"' EXIT
|
|||
cd "$project_dir"
|
||||
|
||||
"$uvr_bin" init --here --r-version "$r_full"
|
||||
# --no-install: resolve and lock only. The install happens in the sync below,
|
||||
# which is the only command that honours --library.
|
||||
"$uvr_bin" add --no-install "$@"
|
||||
# --no-install resolves and locks only; retry because concurrent shards can
|
||||
# expose short-lived DNS or CRAN-index failures and uvr rolls the manifest back
|
||||
# cleanly after an unsuccessful resolution.
|
||||
add_attempt=1
|
||||
while ! "$uvr_bin" add --no-install "$@"; do
|
||||
if [ "$add_attempt" -ge 4 ]; then
|
||||
echo "error: uvr add failed after ${add_attempt} attempts" >&2
|
||||
exit 1
|
||||
fi
|
||||
add_delay=$((add_attempt * 10))
|
||||
echo "warning: uvr add attempt ${add_attempt} failed; retrying in ${add_delay}s" >&2
|
||||
sleep "$add_delay"
|
||||
add_attempt=$((add_attempt + 1))
|
||||
done
|
||||
|
||||
# TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs
|
||||
# `apt-get install` for every resolved system dependency without refreshing the
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ terraform {
|
|||
required_providers {
|
||||
bunnynet = {
|
||||
source = "registry.terraform.io/BunnyWay/bunnynet"
|
||||
version = "~> 0.17"
|
||||
version = "~> 0.18"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,35 +18,70 @@
|
|||
# objects were replaced. The cost is a cold cache for everything else, which is
|
||||
# why this is not used by the daily update path.
|
||||
#
|
||||
# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io,
|
||||
# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them.
|
||||
# The public hostnames currently use separate pull zones, so callers must pass
|
||||
# every zone that serves the repository. A zone can be identified by its
|
||||
# numeric ID or by one of its hostnames; hostname lookup avoids persisting IDs
|
||||
# that change when a zone is recreated.
|
||||
#
|
||||
# Usage:
|
||||
# purge_cdn_zone.sh <BUNNYNET_API_KEY> <pull_zone_id>
|
||||
# purge_cdn_zone.sh <BUNNYNET_API_KEY> <pull_zone> [<pull_zone>...]
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
if (($# < 2)); then
|
||||
echo "usage: $0 <api_key> <pull_zone_id>" >&2
|
||||
echo "usage: $0 <api_key> <pull_zone> [<pull_zone>...]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
api_key="$1"
|
||||
zone_id="$2"
|
||||
shift
|
||||
|
||||
echo "Purging BunnyCDN pull zone ${zone_id}"
|
||||
resolve_zone_id() {
|
||||
local zone="$1"
|
||||
local response_file
|
||||
local zone_id
|
||||
|
||||
status=$(
|
||||
curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \
|
||||
if [[ "${zone}" =~ ^[0-9]+$ ]]; then
|
||||
echo "${zone}"
|
||||
return
|
||||
fi
|
||||
|
||||
response_file=$(mktemp)
|
||||
curl -sS -o "${response_file}" \
|
||||
-H "AccessKey: ${api_key}" \
|
||||
-H "Content-Length: 0" \
|
||||
"https://api.bunny.net/pullzone/${zone_id}/purgeCache"
|
||||
)
|
||||
"https://api.bunny.net/pullzone"
|
||||
zone_id=$(
|
||||
jq -r --arg hostname "${zone}" \
|
||||
'(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \
|
||||
"${response_file}"
|
||||
)
|
||||
rm -f "${response_file}"
|
||||
|
||||
if [[ "${status}" != "200" && "${status}" != "204" ]]; then
|
||||
echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2
|
||||
cat /tmp/purge_zone_response.txt >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "${zone_id}" ]]; then
|
||||
echo "Could not find BunnyCDN pull zone for hostname ${zone}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Purged pull zone ${zone_id} (HTTP ${status})"
|
||||
echo "${zone_id}"
|
||||
}
|
||||
|
||||
for zone in "$@"; do
|
||||
zone_id=$(resolve_zone_id "${zone}")
|
||||
echo "Purging BunnyCDN pull zone ${zone_id}"
|
||||
|
||||
response_file="/tmp/purge_zone_response_${zone_id}.txt"
|
||||
status=$(
|
||||
curl -sS -o "${response_file}" -w '%{http_code}' -X POST \
|
||||
-H "AccessKey: ${api_key}" \
|
||||
-H "Content-Length: 0" \
|
||||
"https://api.bunny.net/pullzone/${zone_id}/purgeCache"
|
||||
)
|
||||
|
||||
if [[ "${status}" != "200" && "${status}" != "204" ]]; then
|
||||
echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2
|
||||
cat "${response_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Purged pull zone ${zone_id} (HTTP ${status})"
|
||||
done
|
||||
|
|
|
|||
309
scripts/verify-r-minor-routing.sh
Executable file
309
scripts/verify-r-minor-routing.sh
Executable file
|
|
@ -0,0 +1,309 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Verify per-R-minor index routing for cran.rpkgs.com across every published
|
||||
# <arch>/<os> slot.
|
||||
#
|
||||
# The edge router (edge/rpkgs-router.ts) rewrites PACKAGES* requests to
|
||||
# `contrib/<x.y>/` when the slot is listed in UNION_SLOTS and the client's
|
||||
# User-Agent carries an R minor. Two properties have to hold before a slot may
|
||||
# be added to UNION_SLOTS:
|
||||
#
|
||||
# 1. the per-minor index is a UNION of the per-minor and flat slots, so
|
||||
# routing to it hides nothing the flat index carries; and
|
||||
# 2. every R minor a client might report resolves to an index that exists,
|
||||
# because contribPath() does not check existence and has no fallback.
|
||||
#
|
||||
# Modes:
|
||||
# (default) Resolve routing decisions without depending on UNION_SLOTS being
|
||||
# set. Safe to run before enabling: it reads the per-minor indexes
|
||||
# directly and reproduces the router's target path.
|
||||
# --live Additionally drive the real CDN with R User-Agents and assert the
|
||||
# bytes served match the expected index. Only meaningful once the
|
||||
# slot is in UNION_SLOTS.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/verify-r-minor-routing.sh
|
||||
# scripts/verify-r-minor-routing.sh --live
|
||||
# MINORS="4.4 4.5" SAMPLE=10 scripts/verify-r-minor-routing.sh
|
||||
#
|
||||
# Exits non-zero if any check fails.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
BASE=${BASE:-https://cran.rpkgs.com}
|
||||
ARCHES=${ARCHES:-"amd64 arm64"}
|
||||
DISTROS=${DISTROS:-"resolute noble jammy rhel8 rhel9 rhel10 alpine323 alpine324"}
|
||||
# The supported window: the current R minor plus the two previous, matching
|
||||
# build-env-images' R_VERSION_LATEST/PREV1/PREV2 and cdn.tf's
|
||||
# local.rpkgs_supported_minors. Each of these must have a published index.
|
||||
MINORS=${MINORS:-"4.4 4.5 4.6"}
|
||||
# Minors we deliberately do not serve. These must have NO published index and,
|
||||
# once routing is live, must be sent to CRAN for sources rather than 404ing or
|
||||
# being handed binaries built under another minor.
|
||||
EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"}
|
||||
# How many Path: targets to HEAD-check per slot/minor. 0 disables.
|
||||
SAMPLE=${SAMPLE:-5}
|
||||
# Largest package-count shortfall a non-primary minor may have against the best
|
||||
# minor on the same slot before coverage counts as uneven. A slot built under
|
||||
# one R minor carries fewer per-minor binaries for the others; until that gap
|
||||
# closes, "full coverage for ABI-sensitive packages" is not a claim we can make.
|
||||
PARITY_TOLERANCE=${PARITY_TOLERANCE:-25}
|
||||
LIVE=0
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--live) LIVE=1 ;;
|
||||
-h | --help)
|
||||
sed -n '2,32p' "$0"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument: $arg" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
WORK=$(mktemp -d)
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
FAILURES=""
|
||||
|
||||
ok() {
|
||||
PASS=$((PASS + 1))
|
||||
printf ' ok %s\n' "$1"
|
||||
}
|
||||
|
||||
bad() {
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILURES="${FAILURES}\n - $1"
|
||||
printf ' FAIL %s\n' "$1"
|
||||
}
|
||||
|
||||
# Fetch a URL into a file, echoing the HTTP status. Cached per URL.
|
||||
fetch() {
|
||||
local url=$1 dest=$2 ua=${3:-}
|
||||
if [ -s "$dest" ]; then
|
||||
cat "$dest.status"
|
||||
return 0
|
||||
fi
|
||||
local status
|
||||
if [ -n "$ua" ]; then
|
||||
status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
|
||||
else
|
||||
status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
|
||||
fi
|
||||
echo "$status" > "$dest.status"
|
||||
echo "$status"
|
||||
}
|
||||
|
||||
head_status() {
|
||||
curl -sS -o /dev/null -w '%{http_code}' -I --max-time 60 "$1" 2>/dev/null
|
||||
}
|
||||
|
||||
# Package names from a gzipped PACKAGES index, sorted.
|
||||
pkg_names() {
|
||||
gunzip -c "$1" 2>/dev/null | awk '/^Package:/ {print $2}' | sort -u
|
||||
}
|
||||
|
||||
# "<Package> <Path>" pairs for entries that carry a Path: field.
|
||||
path_entries() {
|
||||
gunzip -c "$1" 2>/dev/null | awk '
|
||||
/^Package:/ { pkg = $2; ver = ""; path = "" }
|
||||
/^Version:/ { ver = $2 }
|
||||
/^Path:/ { path = $2 }
|
||||
/^$/ { if (pkg != "" && path != "") print pkg, ver, path; pkg = "" }
|
||||
END { if (pkg != "" && path != "") print pkg, ver, path }
|
||||
'
|
||||
}
|
||||
|
||||
# An R User-Agent of the shape R actually sends.
|
||||
r_user_agent() {
|
||||
printf 'R/%s.0 (Ubuntu 24.04; codename=noble) (x86_64-pc-linux-gnu x86_64 linux-gnu)' "$1"
|
||||
}
|
||||
|
||||
echo "verify-r-minor-routing: $BASE"
|
||||
echo " slots: $(echo "$ARCHES" | wc -w) arch x $(echo "$DISTROS" | wc -w) os"
|
||||
echo " minors: $MINORS (excluded: $EXCLUDED_MINORS)"
|
||||
echo " live: $LIVE"
|
||||
echo
|
||||
|
||||
for arch in $ARCHES; do
|
||||
for distro in $DISTROS; do
|
||||
slot="$arch/$distro"
|
||||
echo "$slot"
|
||||
|
||||
flat_url="$BASE/$slot/latest/src/contrib/PACKAGES.gz"
|
||||
flat_file="$WORK/${arch}-${distro}-flat.gz"
|
||||
flat_status=$(fetch "$flat_url" "$flat_file")
|
||||
|
||||
if [ "$flat_status" != "200" ]; then
|
||||
bad "$slot flat index unreachable (HTTP $flat_status)"
|
||||
continue
|
||||
fi
|
||||
|
||||
pkg_names "$flat_file" > "$flat_file.names"
|
||||
flat_count=$(wc -l < "$flat_file.names")
|
||||
if [ "$flat_count" -lt 1000 ]; then
|
||||
bad "$slot flat index has only $flat_count packages"
|
||||
continue
|
||||
fi
|
||||
ok "$slot flat index: $flat_count packages"
|
||||
|
||||
for minor in $MINORS; do
|
||||
minor_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz"
|
||||
minor_file="$WORK/${arch}-${distro}-${minor}.gz"
|
||||
minor_status=$(fetch "$minor_url" "$minor_file")
|
||||
|
||||
# A minor the router would route to must exist, or clients on that R
|
||||
# version get a 404 and see no packages at all.
|
||||
if [ "$minor_status" != "200" ]; then
|
||||
bad "$slot R $minor index missing (HTTP $minor_status) - routing would 404 for R $minor clients"
|
||||
continue
|
||||
fi
|
||||
|
||||
pkg_names "$minor_file" > "$minor_file.names"
|
||||
minor_count=$(wc -l < "$minor_file.names")
|
||||
|
||||
# Union property: nothing the flat index carries may be missing here.
|
||||
missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5)
|
||||
missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l)
|
||||
if [ "$missing_count" -ne 0 ]; then
|
||||
bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))"
|
||||
else
|
||||
ok "$slot R $minor index: $minor_count packages, union holds"
|
||||
fi
|
||||
|
||||
# Path: entries steer to per-minor binaries; they must resolve.
|
||||
if [ "$SAMPLE" -gt 0 ]; then
|
||||
path_entries "$minor_file" > "$minor_file.paths"
|
||||
total_paths=$(wc -l < "$minor_file.paths")
|
||||
broken=0
|
||||
checked=0
|
||||
while read -r pkg ver path; do
|
||||
[ -z "${pkg:-}" ] && continue
|
||||
tarball="$BASE/$slot/latest/src/contrib/$path/${pkg}_${ver}.tar.gz"
|
||||
status=$(head_status "$tarball")
|
||||
checked=$((checked + 1))
|
||||
if [ "$status" != "200" ]; then
|
||||
broken=$((broken + 1))
|
||||
[ "$broken" -le 2 ] && printf ' broken target: %s (HTTP %s)\n' "$tarball" "$status"
|
||||
fi
|
||||
done < <(shuf -n "$SAMPLE" "$minor_file.paths" 2>/dev/null || head -n "$SAMPLE" "$minor_file.paths")
|
||||
|
||||
if [ "$broken" -ne 0 ]; then
|
||||
bad "$slot R $minor: $broken/$checked sampled Path: targets do not resolve (of $total_paths total)"
|
||||
elif [ "$checked" -gt 0 ]; then
|
||||
ok "$slot R $minor: $checked/$checked sampled Path: targets resolve (of $total_paths total)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Live routing: what a real R client on this minor actually receives.
|
||||
if [ "$LIVE" -eq 1 ]; then
|
||||
ua=$(r_user_agent "$minor")
|
||||
live_file="$WORK/${arch}-${distro}-${minor}-live.gz"
|
||||
live_status=$(fetch "$flat_url" "$live_file" "$ua")
|
||||
if [ "$live_status" != "200" ]; then
|
||||
bad "$slot R $minor live request failed (HTTP $live_status)"
|
||||
elif cmp -s "$live_file" "$minor_file"; then
|
||||
ok "$slot R $minor live request served the per-minor index"
|
||||
elif cmp -s "$live_file" "$flat_file"; then
|
||||
bad "$slot R $minor live request served the FLAT index - slot not in UNION_SLOTS?"
|
||||
else
|
||||
bad "$slot R $minor live request served neither the per-minor nor the flat index"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Coverage parity across minors. The union property only guarantees no
|
||||
# client loses packages relative to the flat index; it says nothing about a
|
||||
# 4.4 client seeing fewer packages than a 4.5 client on the same slot.
|
||||
best=0
|
||||
for minor in $MINORS; do
|
||||
f="$WORK/${arch}-${distro}-${minor}.gz.names"
|
||||
[ -s "$f" ] || continue
|
||||
c=$(wc -l < "$f")
|
||||
[ "$c" -gt "$best" ] && best=$c
|
||||
done
|
||||
if [ "$best" -gt 0 ]; then
|
||||
uneven=""
|
||||
for minor in $MINORS; do
|
||||
f="$WORK/${arch}-${distro}-${minor}.gz.names"
|
||||
[ -s "$f" ] || continue
|
||||
c=$(wc -l < "$f")
|
||||
gap=$((best - c))
|
||||
[ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap"
|
||||
done
|
||||
if [ -n "$uneven" ]; then
|
||||
bad "$slot coverage uneven across minors (vs best $best):$uneven"
|
||||
else
|
||||
ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Excluded minors: no published index, and under --live a redirect to CRAN.
|
||||
for minor in $EXCLUDED_MINORS; do
|
||||
ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz"
|
||||
ex_status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 60 "$ex_url" 2>/dev/null)
|
||||
if [ "$ex_status" = "200" ]; then
|
||||
bad "$slot R $minor is excluded but an index is published - the two lists disagree"
|
||||
else
|
||||
ok "$slot R $minor correctly has no published index"
|
||||
fi
|
||||
|
||||
if [ "$LIVE" -eq 1 ]; then
|
||||
loc=$(curl -sS -o /dev/null -w '%{redirect_url}' -A "$(r_user_agent "$minor")" \
|
||||
--max-time 60 "$flat_url" 2>/dev/null)
|
||||
case "$loc" in
|
||||
https://cran.r-project.org/*)
|
||||
ok "$slot R $minor is sent to CRAN ($loc)"
|
||||
;;
|
||||
"")
|
||||
bad "$slot R $minor was served directly instead of being sent to CRAN"
|
||||
;;
|
||||
*)
|
||||
bad "$slot R $minor redirected somewhere unexpected: $loc"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
done
|
||||
|
||||
# A client whose User-Agent carries no R version must keep getting the flat
|
||||
# index, never a per-minor one.
|
||||
if [ "$LIVE" -eq 1 ]; then
|
||||
plain_file="$WORK/${arch}-${distro}-plain.gz"
|
||||
plain_status=$(fetch "$flat_url" "$plain_file" "curl/8.0.0")
|
||||
if [ "$plain_status" != "200" ]; then
|
||||
bad "$slot non-R User-Agent request failed (HTTP $plain_status)"
|
||||
elif cmp -s "$plain_file" "$flat_file"; then
|
||||
ok "$slot non-R User-Agent still served the flat index"
|
||||
else
|
||||
bad "$slot non-R User-Agent was routed away from the flat index"
|
||||
fi
|
||||
|
||||
# Tarball requests must never be rewritten into a per-minor directory:
|
||||
# flat-slot packages do not live there.
|
||||
sample_pkg=$(gunzip -c "$flat_file" | awk '/^Package:/ {p=$2} /^Version:/ {print p, $2; exit}')
|
||||
if [ -n "$sample_pkg" ]; then
|
||||
# shellcheck disable=SC2086 # deliberate split into $1 (package) and $2 (version)
|
||||
set -- $sample_pkg
|
||||
tb="$BASE/$slot/latest/src/contrib/${1}_${2}.tar.gz"
|
||||
tb_status=$(curl -sS -o /dev/null -w '%{http_code}' -A "$(r_user_agent 4.5)" --max-time 60 "$tb" 2>/dev/null)
|
||||
if [ "$tb_status" = "200" ]; then
|
||||
ok "$slot tarball request under an R User-Agent still resolves"
|
||||
else
|
||||
bad "$slot tarball ${1}_${2}.tar.gz broke under an R User-Agent (HTTP $tb_status)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
echo
|
||||
echo "passed: $PASS failed: $FAIL"
|
||||
if [ "$FAIL" -ne 0 ]; then
|
||||
printf 'failures:%b\n' "$FAILURES"
|
||||
exit 1
|
||||
fi
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue