Compare commits

..
Author SHA1 Message Date
1b6e2b506e
ci(crow): add weekly patch-proposal + feedback-loop pipeline
Add `.crow/weekly-patch-proposals.yaml`, a single (non-matrix) job that runs
the classifier over all of `single_builds` weekly: it posts/updates a Forgejo
tracking issue with the auto-proposable registry entries via
`propose-patches.R --open-issue`, then logs the step-4 metrics via
`proposal-tracking.R`. Clones read-only; the only write is the issue.
Register the `weekly-patch-proposals` cron in the crow UI, or trigger manually
with `task=weekly-patch-proposals`.
2026-07-14 15:05:22 +00:00
8afa584da3
feat(local): auto-propose registry patches and track the feedback loop
Implement steps 3 + 4 of issue #115 on top of the failure classifier, now
that bincraft v4.4.3 applies registry patches/makevars/configure_args to the
target build (not just dependencies), so a trial patched build is meaningful.

- refactor the classify helpers to expose a pure build_triage_report() and a
  list-returning entry builder; failing-builds-report.R now renders from it
- add local/propose-patches.R (step 3, "propose, do not apply"): emit a
  pre-filled registry.json entry for each classified, safe, unregistered
  failure, validate the candidate set against a temporary merged registry,
  and (only on request) --write it plus a proposals ledger, or --open-issue a
  Forgejo tracking issue; the human gate and validator/trial-build acceptance
  stay, and novel source diffs / unknown signatures are never proposed
- add local/trial-build-patch.R: isolated bincraft build of one package with
  the registry applied (no upload/archive/metadata) as the pre-merge gate
- add local/proposal-tracking.R + local/proposal-tracking-lib.R (step 4):
  signature hit rate, proposed-vs-merged, and retirement candidates, with the
  pure helpers covered by tests
- teach validate-patches.R optional PATCH_DIR/REGISTRY_FILE overrides so a
  candidate registry can be validated without touching the real one
- document the propose/trial-build/tracking workflow in local/patches/README.md
2026-07-14 15:01:55 +00:00
55 changed files with 498 additions and 4548 deletions

View file

@ -28,6 +28,10 @@ matrix:
ARCH: amd64
- CODENAME: redhat-10
ARCH: arm64
- CODENAME: alpine321
ARCH: amd64
- CODENAME: alpine321
ARCH: arm64
- CODENAME: alpine322
ARCH: amd64
- CODENAME: alpine322

View file

@ -1,76 +0,0 @@
# Auto-apply registry patches (issue #115, step 3 automation).
# Classifies `single_builds` failures and, for the top-N auto-proposable
# candidates by failure volume, writes the registry entries onto the reused
# `auto/registry-patch-proposals` branch and opens/updates a single PR.
# Nothing merges: the `trial-build-registry` pipeline is the merge gate, and a
# human reviews the PR. Novel source diffs / unknown signatures are never
# proposed. Global across platforms, so a single job -- no matrix.
#
# FORGEJO_TOKEN is used for both the branch push and opening the PR (no separate
# write-scoped secret needed). Register the `auto-apply-patches` cron in the crow
# UI, or run manually:
# crow pipeline create --branch main \
# --var auto_apply_patches=true devxy/build-cran-binaries
#
# The gate variable is `auto_apply_patches`, named after the pipeline: a manual
# run instantiates every pipeline in `.crow/`, so one without its own gate runs
# on *any* manual trigger in this repo. This one pushes a branch and opens a PR,
# so it must stay off unless it is what was asked for.
variables:
auto_apply_patches:
description: 'Run the auto-patch proposer. Also gates this pipeline.'
options:
- 'true'
- 'false'
default: 'false'
patch_limit:
description: 'Max candidates to propose per run (top by failure volume).'
default: '10'
when:
- event: manual
evaluate: 'auto_apply_patches == "true"'
- event: cron
cron: auto-apply-patches
skip_clone: true
labels:
group: rpkgs-amd64
steps:
- name: 'Auto-apply registry patches'
image: reg.devxy.io/rpkgs/build-env-alpine:3.24
pull: true
environment:
PGPASS:
from_secret: PGPASS
REPO_RO_TOKEN:
from_secret: REPO_RO_TOKEN
FORGEJO_TOKEN:
from_secret: FORGEJO_TOKEN
GIT_USER: devxy-bot
GIT_EMAIL: bot@devxy.io
PATCH_LIMIT: ${patch_limit}
R_VERSION: 4.5.3
R_LIBS_USER: /mnt/cache/R-pkgs
commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- mkdir -p /mnt/cache/R-pkgs
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
- /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT
backend_options:
kubernetes:
resources:
requests:
memory: 1Gi
cpu: 2000m
limits:
memory: 2Gi
cpu: 2000m
tolerations:
- key: 'CI'
operator: 'Equal'
value: 'true'
effect: 'NoSchedule'

View file

@ -3,33 +3,29 @@
# Variables are declared so the manual-run form exposes them (crow #1165);
# they are merged with build-all-versions' identical declarations.
variables:
# Gates this pipeline. A manual pipeline creation instantiates every file in
# .crow/, and a declared default is applied even when the run never passed
# this variable, so the default must be a value that matches no matrix row.
target_arch:
description: 'Architecture to build, or "none" to run nothing.'
description: 'Architecture to build.'
options:
- none
- amd64
- arm64
default: none
default: amd64
OS:
description: 'Base OS image name.'
description: "Base OS image name."
options:
- alpine
- redhat
- ubuntu
default: alpine
OS_VERSION:
description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).'
description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)."
options:
- '3.24'
- '8'
- '9'
- '10'
- 'jammy'
- 'noble'
default: '3.24'
- "3.24"
- "8"
- "9"
- "10"
- "jammy"
- "noble"
default: "3.24"
R_VERSION:
description: 'Primary R version under /opt/R.'
options:
@ -70,24 +66,23 @@ steps:
from_secret: B2_S3_SECRET_KEY
PGPASS:
from_secret: PGPASS
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
R_LIBS_USER: /mnt/cache/R-pkgs
# Keep uvr's downloads and extracted-package entries on the persistent
# volume instead of the container-local ~/.uvr default.
UVR_CACHE_DIR: /mnt/cache/uvr/cache
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
CCACHE_DIR: /mnt/cache/ccache
volumes:
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
commands:
# one-time full wipe to fix corrupted .so files from previous failed builds
# - rm -rf /mnt/cache/R-pkgs
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
# Clear churny pkgcache metadata left by a prior crashed run (the "patched"
# repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*).
# Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs.
- rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
# Pin the same bincraft version the build steps use, so the precomputed
# snapshot and the per-agent library stay consistent across the pipeline.
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
- /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")'
- /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))"
backend_options:
docker:

View file

@ -4,36 +4,32 @@
# image and cache volume. Placement is via the group label (rpkgs-amd64/rpkgs-arm64).
# Skip list lives in local/excluded-packages.json (read by local/build-all.R).
variables:
# Gates this pipeline. A manual pipeline creation instantiates every file in
# .crow/, and a declared default is applied even when the run never passed
# this variable, so the default must be a value that matches no matrix row.
target_arch:
description: 'Architecture to build, or "none" to run nothing.'
description: 'Architecture to build.'
options:
- none
- amd64
- arm64
default: none
default: amd64
OS:
description: 'Base OS image name.'
description: "Base OS image name."
options:
- alpine
- redhat
- ubuntu
default: alpine
OS_VERSION:
description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).'
description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)."
options:
- '3.22'
- '3.23'
- '3.24'
- '8'
- '9'
- '10'
- 'jammy'
- 'noble'
- 'resolute'
default: '3.24'
- "3.22"
- "3.23"
- "3.24"
- "8"
- "9"
- "10"
- "jammy"
- "noble"
- "resolute"
default: "3.24"
R_VERSION:
description: 'Primary R version under /opt/R.'
options:
@ -51,49 +47,38 @@ labels:
platform: linux/${ARCH}
group: rpkgs-${ARCH}
# Empty UVR_CACHE_DIR/UVR_PACKAGES_DIR fall back to uvr's container-local
# ~/.uvr defaults; amd64 deliberately does not persist them (as with the
# pkgcache dir it replaces), arm64 does.
matrix:
include:
- ARCH: amd64
UVR_CACHE_DIR: ''
UVR_PACKAGES_DIR: ''
R_PKG_CACHE_DIR: ''
SPLIT_INTO: 4
SPLIT_INDEX: 1
- ARCH: amd64
UVR_CACHE_DIR: ''
UVR_PACKAGES_DIR: ''
R_PKG_CACHE_DIR: ''
SPLIT_INTO: 4
SPLIT_INDEX: 2
- ARCH: amd64
UVR_CACHE_DIR: ''
UVR_PACKAGES_DIR: ''
R_PKG_CACHE_DIR: ''
SPLIT_INTO: 4
SPLIT_INDEX: 3
- ARCH: amd64
UVR_CACHE_DIR: ''
UVR_PACKAGES_DIR: ''
R_PKG_CACHE_DIR: ''
SPLIT_INTO: 4
SPLIT_INDEX: 4
- ARCH: arm64
UVR_CACHE_DIR: /mnt/cache/uvr/cache
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
SPLIT_INTO: 4
SPLIT_INDEX: 1
- ARCH: arm64
UVR_CACHE_DIR: /mnt/cache/uvr/cache
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
SPLIT_INTO: 4
SPLIT_INDEX: 2
- ARCH: arm64
UVR_CACHE_DIR: /mnt/cache/uvr/cache
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
SPLIT_INTO: 4
SPLIT_INDEX: 3
- ARCH: arm64
UVR_CACHE_DIR: /mnt/cache/uvr/cache
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
SPLIT_INTO: 4
SPLIT_INDEX: 4
@ -122,10 +107,8 @@ steps:
from_secret: GITHUB_PAT
# normal env vars
GIT_USER: pat-s
# set the location of uvr's caches, which persist the R package
# dependencies needed to install the packages themselves
UVR_CACHE_DIR: ${UVR_CACHE_DIR}
UVR_PACKAGES_DIR: ${UVR_PACKAGES_DIR}
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR}
R_LIBS_USER: /mnt/cache/R-pkgs
CCACHE_DIR: /mnt/cache/ccache
NCPUS: 2
@ -133,7 +116,12 @@ steps:
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
# Clear churny pkgcache metadata left by a prior crashed run (the "patched"
# repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*).
# Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. Within-run
# growth is bounded separately by trim_pkgcache_metadata() in build-all.R.
- rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
# The primary pass must not rely on build-all-versions-install-deps having
# run on *this* agent: depends_on only orders the steps, but the cache
# volume is per-agent, so a job landing on an agent where install-deps did

View file

@ -3,17 +3,13 @@
# Routing is preserved 1:1:
# - cron: each existing `process-cran-updates-<os>-<arch>` cron fires only
# its matching matrix row (via the per-row `cron:` name filter).
# - manual: pick a target from the `process_cran_updates` dropdown;
# "all" fans out every os/arch as parallel matrix workflows.
# - manual: pick a target from the `process_cran_updates` dropdown
# ("all" = every os/arch).
# Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64).
variables:
# Gates this pipeline. A manual pipeline creation instantiates every file in
# .crow/, and a declared default is applied even when the run never passed
# this variable, so the default must be a value that matches no matrix row.
process_cran_updates:
description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
description: "Manual run target: a specific <os>-<arch>, or 'all' for every os/arch."
options:
- none
- all
- alpine-322-amd64
- alpine-322-arm64
@ -33,7 +29,7 @@ variables:
- ubuntu-2404-arm64
- ubuntu-2604-amd64
- ubuntu-2604-arm64
default: none
default: all
when:
- event: cron
@ -51,111 +47,111 @@ matrix:
- OS: alpine-322
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.22
IMG: alpine:3.24
OS_ID: alpine322
PROCESS_NEW: 'FALSE'
PROCESS_NEW: "FALSE"
- OS: alpine-322
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.23
IMG: alpine:3.24
OS_ID: alpine322
PROCESS_NEW: 'FALSE'
PROCESS_NEW: "FALSE"
- OS: alpine-323
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.23
IMG: alpine:3.24
OS_ID: alpine323
PROCESS_NEW: 'FALSE'
PROCESS_NEW: "FALSE"
- OS: alpine-323
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.23
IMG: alpine:3.24
OS_ID: alpine323
PROCESS_NEW: 'FALSE'
PROCESS_NEW: "FALSE"
- OS: alpine-324
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.24
OS_ID: alpine324
PROCESS_NEW: 'FALSE'
PROCESS_NEW: "FALSE"
- OS: alpine-324
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.24
OS_ID: alpine324
PROCESS_NEW: 'FALSE'
PROCESS_NEW: "FALSE"
- OS: redhat-8
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:8
OS_ID: rhel8
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: redhat-8
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:8
OS_ID: rhel8
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: redhat-9
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:9
OS_ID: rhel9
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: redhat-9
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:9
OS_ID: rhel9
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: redhat-10
ARCH: amd64
R_VERSION: 4.5.3
IMG: redhat:10
OS_ID: rhel10
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: redhat-10
ARCH: arm64
R_VERSION: 4.5.3
IMG: redhat:10
OS_ID: rhel10
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: ubuntu-2204
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
OS_ID: jammy
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: ubuntu-2204
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
OS_ID: jammy
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: ubuntu-2404
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:noble
OS_ID: noble
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: ubuntu-2404
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:noble
OS_ID: noble
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: ubuntu-2604
ARCH: amd64
R_VERSION: 4.5.3
IMG: ubuntu:resolute
OS_ID: resolute
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
- OS: ubuntu-2604
ARCH: arm64
R_VERSION: 4.5.3
IMG: ubuntu:resolute
OS_ID: resolute
PROCESS_NEW: 'TRUE'
PROCESS_NEW: "TRUE"
steps:
- name: 'Processing Updates'
@ -186,10 +182,8 @@ steps:
NTFY_AUTH: TRUE
NTFY_PASSWORD:
from_secret: ntfy_token
# set the location of uvr's caches, which persist the R package
# dependencies needed to install the packages themselves
UVR_CACHE_DIR: /mnt/cache/uvr/cache
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
R_LIBS_USER: /mnt/cache/R-pkgs
R_VERSION: ${R_VERSION}
CCACHE_DIR: /mnt/cache/ccache
@ -200,11 +194,10 @@ steps:
INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
# rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
@ -219,7 +212,6 @@ steps:
LIB="/mnt/cache/R-pkgs-$RMINOR"
mkdir -p "$LIB"
R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true
done
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'

View file

@ -1,110 +0,0 @@
### Manual repair of a slot whose PACKAGES index advertises a broken `Built`
### stamp (e.g. `Built: R 4.5.0; NA; ...`).
#
# uvr matches the stamp's platform triple plus R minor to decide binary vs
# source, so an unusable triple turns a whole slot source-only. See
# local/repair-built-stamp.R for why this patches PACKAGES.db in place instead
# of forcing a full reparse.
#
# Run with `dry_run: true` first: it reports how many entries are broken per
# slot and changes nothing. Pick the R version the slot should advertise, which
# is the R_VERSION its entry in .crow/process-updates.yaml uses.
#
# The gate variable is `repair_built_stamp`, not `target_arch`: `target_arch` is
# what build-all-versions and build-all-versions-install-deps gate on, so a
# manual run passing it would start a full rebuild alongside this repair. Every
# pipeline here gates on a variable named after itself for exactly that reason.
#
# crow pipeline create --branch main \
# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \
# --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries
variables:
# Gates this pipeline. A manual pipeline creation instantiates every file in
# .crow/, and a declared default is applied even when the run never passed
# this variable, so the default must be a value that matches no matrix row.
repair_built_stamp:
description: 'Architecture of the slot to repair, or "none" to run nothing.'
options:
- none
- amd64
- arm64
default: none
OS:
description: 'Base OS image name.'
options:
- alpine
- redhat
- ubuntu
default: alpine
OS_VERSION:
description: 'OS image tag. Must match OS (alpine: 3.22/3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).'
options:
- '3.22'
- '3.23'
- '3.24'
- '8'
- '9'
- '10'
- 'jammy'
- 'noble'
- 'resolute'
default: '3.22'
R_VERSION:
description: 'R version whose stamp the slot should advertise.'
options:
- 4.5.3
- 4.4.3
default: 4.5.3
dry_run:
description: 'Report what would change without writing anything.'
options:
- 'true'
- 'false'
default: 'true'
when:
- event: manual
evaluate: 'repair_built_stamp == "${ARCH}"'
skip_clone: true
labels:
platform: linux/${ARCH}
group: rpkgs-${ARCH}
matrix:
include:
- ARCH: amd64
- ARCH: arm64
steps:
- name: 'Repair Built stamp'
image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}'
pull: true
environment:
B2_S3_ACCESS_KEY:
from_secret: B2_S3_ACCESS_KEY
B2_S3_SECRET_KEY:
from_secret: B2_S3_SECRET_KEY
REPO_RO_TOKEN:
from_secret: REPO_RO_TOKEN
GIT_USER: pat-s
commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
- |
if [ "$dry_run" = "false" ]; then
/opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" --apply
else
/opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH"
fi
backend_options:
docker:
resources:
requests:
memory: 2Gi
cpu: 1000m
limits:
memory: 8Gi
cpu: 2000m

View file

@ -1,160 +0,0 @@
# Merge gate for the auto-patch PR (issue #115, step 3).
# For each platform, trial-builds every registry entry the auto-patch branch
# ADDS (vs main) in that platform's own `reg.devxy.io/rpkgs/build-env-*` image,
# with the registry applied. A row with no new entries for its platform is a
# fast no-op. The pipeline is green only if every new entry builds, so it gates
# the PR before merge. Nothing is uploaded/archived/recorded.
#
# The repo uses no `pull_request` triggers, so this runs manually against the
# branch (or on a cron); point it at the auto-patch branch via `patch_branch`:
# crow pipeline create --branch main --var trial_build_registry=true \
# --var patch_branch=auto/registry-patch-proposals devxy/build-cran-binaries
#
# The gate variable is `trial_build_registry`, named after the pipeline: a
# manual run instantiates every pipeline in `.crow/`, so one without its own
# gate runs on *any* manual trigger in this repo. This one starts a build per
# matrix row on both arches, which is far too expensive to fire by accident.
variables:
trial_build_registry:
description: 'Trial-build the branch new registry entries. Also gates this pipeline.'
options:
- 'true'
- 'false'
default: 'false'
patch_branch:
description: 'Branch whose new registry entries to trial-build.'
default: auto/registry-patch-proposals
when:
- event: manual
evaluate: 'trial_build_registry == "true"'
- event: cron
cron: trial-build-registry
skip_clone: true
labels:
group: rpkgs-${ARCH}
matrix:
include:
- OS: alpine-322
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.22
- OS: alpine-322
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.22
- OS: alpine-323
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.23
- OS: alpine-323
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.23
- OS: alpine-324
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-324
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: redhat-8
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:8
- OS: redhat-8
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:8
- OS: redhat-9
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:9
- OS: redhat-9
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:9
- OS: redhat-10
ARCH: amd64
R_VERSION: 4.5.3
IMG: redhat:10
- OS: redhat-10
ARCH: arm64
R_VERSION: 4.5.3
IMG: redhat:10
- OS: ubuntu-2204
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
- OS: ubuntu-2204
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
- OS: ubuntu-2404
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:noble
- OS: ubuntu-2404
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:noble
- OS: ubuntu-2604
ARCH: amd64
R_VERSION: 4.5.3
IMG: ubuntu:resolute
- OS: ubuntu-2604
ARCH: arm64
R_VERSION: 4.5.3
IMG: ubuntu:resolute
steps:
- name: 'Trial-build new registry entries'
image: reg.devxy.io/rpkgs/build-env-${IMG}
pull: true
environment:
B2_S3_ACCESS_KEY:
from_secret: B2_S3_ACCESS_KEY
B2_S3_SECRET_KEY:
from_secret: B2_S3_SECRET_KEY
REPO_RO_TOKEN:
from_secret: REPO_RO_TOKEN
GITHUB_PAT:
from_secret: GITHUB_PAT
PLATFORM: ${OS}
ARCH: ${ARCH}
R_VERSION: ${R_VERSION}
R_LIBS_USER: /mnt/cache/R-pkgs
# Surface the real compiler error when an isolated patched build fails,
# instead of bincraft's opaque "System command 'R' failed" (needs bincraft
# with BINCRAFT_VERBOSE_PATCH_BUILD support; harmless on older versions).
BINCRAFT_VERBOSE_PATCH_BUILD: 'TRUE'
commands:
# Clone main, then check out the auto-patch branch if it exists. When the
# proposer had no candidates it never (re)creates that branch, so a missing
# branch means "nothing to verify" -- no-op cleanly instead of failing the
# clone.
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- git fetch -q origin main
- 'if git ls-remote --exit-code --heads origin ${patch_branch} >/dev/null 2>&1; then git fetch -q origin ${patch_branch} && git checkout -q FETCH_HEAD; else echo "No ${patch_branch} branch; no pending auto-patch proposals to verify."; exit 0; fi'
- mkdir -p /mnt/cache/R-pkgs
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
- /opt/R/$R_VERSION/bin/Rscript local/trial-build-registry.R origin/main
backend_options:
kubernetes:
resources:
requests:
memory: 2Gi
cpu: 2000m
limits:
memory: 4Gi
cpu: 2000m
tolerations:
- key: 'CI'
operator: 'Equal'
value: 'true'
effect: 'NoSchedule'

View file

@ -3,24 +3,20 @@
# Routing is preserved 1:1:
# - cron: each existing `weekly-audit-missing-<os>-<arch>` cron fires only
# its matching matrix row (via the per-row `cron:` name filter).
# - manual: pick a target from the `weekly_audit_missing` dropdown;
# "all" fans out every os/arch as parallel matrix workflows.
# - manual: pick a target from the `weekly_audit_missing` dropdown
# ("all" = every os/arch).
# Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64).
variables:
# Gates this pipeline. A manual pipeline creation instantiates every file in
# .crow/, and a declared default is applied even when the run never passed
# this variable, so the default must be a value that matches no matrix row.
weekly_audit_missing:
description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
description: "Manual run target: a specific <os>-<arch>, or 'all' for every os/arch."
options:
- none
- all
- alpine-321-amd64
- alpine-321-arm64
- alpine-322-amd64
- alpine-322-arm64
- alpine-323-amd64
- alpine-323-arm64
- alpine-324-amd64
- alpine-324-arm64
- redhat-8-amd64
- redhat-8-arm64
- redhat-9-amd64
@ -31,9 +27,7 @@ variables:
- ubuntu-2204-arm64
- ubuntu-2404-amd64
- ubuntu-2404-arm64
- ubuntu-2604-amd64
- ubuntu-2604-arm64
default: none
default: all
when:
- event: cron
@ -48,27 +42,27 @@ labels:
matrix:
include:
- OS: alpine-322
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.22
- OS: alpine-322
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.22
- OS: alpine-323
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.23
- OS: alpine-323
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.23
- OS: alpine-324
- OS: alpine-321
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-324
- OS: alpine-321
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-322
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-322
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-323
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-323
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.24
@ -112,14 +106,6 @@ matrix:
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:noble
- OS: ubuntu-2604
ARCH: amd64
R_VERSION: 4.5.3
IMG: ubuntu:resolute
- OS: ubuntu-2604
ARCH: arm64
R_VERSION: 4.5.3
IMG: ubuntu:resolute
steps:
- name: 'Audit missing binaries'
@ -146,8 +132,7 @@ steps:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))'
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")'
backend_options:
docker:

View file

@ -1,33 +1,18 @@
# Weekly failure-triage proposals (issue #115, steps 3 + 4).
# Classifies the recorded `single_builds` failures and:
# 1. posts/updates a Forgejo issue with the auto-proposable registry entries
# (human reviews, trial-builds, and opens the PR -- nothing merges),
# 2. posts/updates a second Forgejo issue with the classifier's blind spots
# (unclassified failures = candidates for new signatures, and groups
# blocked on a dependency), and prints the feedback-loop metrics to the log.
# 1. posts/updates a Forgejo tracking issue with the auto-proposable registry
# entries (human reviews, trial-builds, and opens the PR -- nothing merges), and
# 2. prints the feedback-loop metrics (signature hit rate, proposed-vs-merged,
# retirement candidates) to the run log.
# Global across platforms (the classifier groups over all of single_builds), so
# a single job -- no matrix. Clones read-only; the only writes are the two
# Forgejo issues via FORGEJO_TOKEN.
# a single job -- no matrix. Clones read-only; the only write is the Forgejo
# issue via FORGEJO_TOKEN.
#
# Run manually with:
# crow pipeline create --branch main \
# --var weekly_patch_proposals=true devxy/build-cran-binaries
#
# The gate variable is `weekly_patch_proposals`, named after the pipeline: a
# manual run instantiates every pipeline in `.crow/`, so one without its own
# gate runs on *any* manual trigger in this repo. This one posts and edits
# Forgejo issues, so an unrelated manual run must not fire it.
variables:
weekly_patch_proposals:
description: 'Run the weekly failure triage. Also gates this pipeline.'
options:
- 'true'
- 'false'
default: 'false'
# Register the cron in the crow UI as `weekly-patch-proposals`, or run manually:
# woodpecker-cli pipeline create --var task=weekly-patch-proposals --branch=main 7
when:
- event: manual
evaluate: 'weekly_patch_proposals == "true"'
evaluate: 'task == "weekly-patch-proposals"'
- event: cron
cron: weekly-patch-proposals
@ -53,9 +38,9 @@ steps:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- mkdir -p /mnt/cache/R-pkgs
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))'
- /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue
- /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue
- /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R
backend_options:
kubernetes:
resources:

View file

@ -1,36 +1,21 @@
# Consolidated weekly-rebuild-missing pipeline (all platforms, both arches).
# Three matrix rows per OS/arch, one per shard of that slot's rebuild list.
# One matrix row per OS/arch replaces the former per-platform files.
# Routing is preserved 1:1:
# - cron: each existing `weekly-rebuild-missing-<os>-<arch>` cron fires only
# its matching matrix rows (via the per-row `cron:` name filter),
# which is now all three shards of that slot.
# - manual: pick a target from the `weekly_rebuild_missing` dropdown;
# "all" fans out every os/arch and shard as parallel matrix
# workflows, while a single <os>-<arch> runs its three shards.
# its matching matrix row (via the per-row `cron:` name filter).
# - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the
# previous bare manual trigger that ran every os/arch); pick a
# single <os>-<arch> to run just one.
# Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64).
#
# The shard picks up its own slice and re-derives what is still outstanding
# from the bucket, so a restart resumes rather than replaying; see
# local/rebuild-missing.R.
#
# Re-indexing and the CDN purge deliberately do NOT live here. Three shards
# writing one slot's PACKAGES concurrently would race, so they moved to
# .crow/weekly-rebuild-reindex.yaml, which depends on this pipeline.
variables:
# Gates this pipeline. A manual pipeline creation instantiates every file in
# .crow/, and a declared default is applied even when the run never passed
# this variable, so the default must be a value that matches no matrix row.
weekly_rebuild_missing:
description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
description: "Manual run target: a specific <os>-<arch>, or 'all' for every os/arch."
options:
- none
- all
- alpine-322-amd64
- alpine-322-arm64
- alpine-323-amd64
- alpine-323-arm64
- alpine-324-amd64
- alpine-324-arm64
- redhat-8-amd64
- redhat-8-arm64
- redhat-9-amd64
@ -41,9 +26,7 @@ variables:
- ubuntu-2204-arm64
- ubuntu-2404-amd64
- ubuntu-2404-arm64
- ubuntu-2604-amd64
- ubuntu-2604-arm64
default: none
default: all
when:
- event: cron
@ -61,327 +44,59 @@ matrix:
- OS: alpine-322
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.22
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: alpine-322
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.22
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: alpine-322
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.22
SPLIT_INTO: 3
SPLIT_INDEX: 3
IMG: alpine:3.24
- OS: alpine-322
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.22
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: alpine-322
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.22
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: alpine-322
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.22
SPLIT_INTO: 3
SPLIT_INDEX: 3
IMG: alpine:3.24
- OS: alpine-323
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.23
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: alpine-323
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.23
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: alpine-323
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.23
SPLIT_INTO: 3
SPLIT_INDEX: 3
IMG: alpine:3.24
- OS: alpine-323
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.23
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: alpine-323
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.23
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: alpine-323
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.23
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: alpine-324
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.24
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: alpine-324
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.24
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: alpine-324
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.24
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: alpine-324
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.24
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: alpine-324
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.24
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: alpine-324
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.24
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: redhat-8
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:8
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: redhat-8
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:8
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: redhat-8
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:8
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: redhat-8
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:8
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: redhat-8
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:8
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: redhat-8
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:8
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: redhat-9
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:9
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: redhat-9
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:9
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: redhat-9
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:9
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: redhat-9
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:9
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: redhat-9
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:9
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: redhat-9
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:9
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: redhat-10
ARCH: amd64
R_VERSION: 4.5.3
IMG: redhat:10
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: redhat-10
ARCH: amd64
R_VERSION: 4.5.3
IMG: redhat:10
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: redhat-10
ARCH: amd64
R_VERSION: 4.5.3
IMG: redhat:10
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: redhat-10
ARCH: arm64
R_VERSION: 4.5.3
IMG: redhat:10
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: redhat-10
ARCH: arm64
R_VERSION: 4.5.3
IMG: redhat:10
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: redhat-10
ARCH: arm64
R_VERSION: 4.5.3
IMG: redhat:10
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: ubuntu-2204
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: ubuntu-2204
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: ubuntu-2204
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: ubuntu-2204
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: ubuntu-2204
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: ubuntu-2204
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: ubuntu-2404
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:noble
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: ubuntu-2404
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:noble
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: ubuntu-2404
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:noble
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: ubuntu-2404
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:noble
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: ubuntu-2404
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:noble
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: ubuntu-2404
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:noble
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: ubuntu-2604
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:resolute
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: ubuntu-2604
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:resolute
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: ubuntu-2604
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:resolute
SPLIT_INTO: 3
SPLIT_INDEX: 3
- OS: ubuntu-2604
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:resolute
SPLIT_INTO: 3
SPLIT_INDEX: 1
- OS: ubuntu-2604
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:resolute
SPLIT_INTO: 3
SPLIT_INDEX: 2
- OS: ubuntu-2604
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:resolute
SPLIT_INTO: 3
SPLIT_INDEX: 3
steps:
- name: 'Rebuild missing binaries'
@ -406,30 +121,23 @@ steps:
FORGEJO_TOKEN:
from_secret: FORGEJO_TOKEN
GIT_USER: pat-s
UVR_CACHE_DIR: /mnt/cache/uvr/cache
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
R_LIBS_USER: /mnt/cache/R-pkgs
R_VERSION: ${R_VERSION}
CCACHE_DIR: /mnt/cache/ccache
PLATFORM: ${OS}
ARCH: ${ARCH}
NCPUS: 2
SPLIT_INTO: ${SPLIT_INTO}
SPLIT_INDEX: ${SPLIT_INDEX}
# Wall clock after which the shard stops cleanly instead of having to be
# killed. A kill matches neither `success` nor `failure`, so it would skip
# the dependent re-index and leave rebuilt binaries behind a stale edge.
REBUILD_BUDGET_HOURS: 20
commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")'
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
- $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1
backend_options:
docker:
resources:

View file

@ -1,191 +0,0 @@
# Re-index and purge after weekly-rebuild-missing.
#
# weekly-rebuild-missing runs three shards per slot. Each of them replaces
# objects in place, so the slot's index still advertises the old MD5 and, for
# anything that had been served from source, no Built stamp. Re-indexing from
# inside a shard would mean three concurrent `upload_package_index()` calls on
# one prefix: `cranlike::update_PACKAGES()` lists the live bucket, so an early
# lister that uploads last publishes an index missing its siblings' work.
#
# So it happens exactly once per slot, here, after every shard has finished.
# `runs_on: [success, failure]` keeps that true when a shard fails; only an
# explicit cancel skips it, and this pipeline can then be triggered on its own.
variables:
# Mirrors the gate on weekly-rebuild-missing so a manual run re-indexes
# exactly the slots it rebuilt. A manual pipeline creation instantiates every
# file in .crow/, so the default must match no matrix row.
weekly_rebuild_missing:
description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
options:
- none
- all
- alpine-322-amd64
- alpine-322-arm64
- alpine-323-amd64
- alpine-323-arm64
- alpine-324-amd64
- alpine-324-arm64
- redhat-8-amd64
- redhat-8-arm64
- redhat-9-amd64
- redhat-9-arm64
- redhat-10-amd64
- redhat-10-arm64
- ubuntu-2204-amd64
- ubuntu-2204-arm64
- ubuntu-2404-amd64
- ubuntu-2404-arm64
- ubuntu-2604-amd64
- ubuntu-2604-arm64
default: none
when:
- event: cron
cron: weekly-rebuild-missing-${OS}-${ARCH}
- event: manual
evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"'
depends_on:
- weekly-rebuild-missing
runs_on: [success, failure]
skip_clone: true
labels:
group: rpkgs-${ARCH}
matrix:
include:
- OS: alpine-322
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.22
- OS: alpine-322
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.22
- OS: alpine-323
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.23
- OS: alpine-323
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.23
- OS: alpine-324
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-324
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: redhat-8
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:8
- OS: redhat-8
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:8
- OS: redhat-9
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:9
- OS: redhat-9
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:9
- OS: redhat-10
ARCH: amd64
R_VERSION: 4.5.3
IMG: redhat:10
- OS: redhat-10
ARCH: arm64
R_VERSION: 4.5.3
IMG: redhat:10
- OS: ubuntu-2204
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
- OS: ubuntu-2204
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
- OS: ubuntu-2404
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:noble
- OS: ubuntu-2404
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:noble
- OS: ubuntu-2604
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:resolute
- OS: ubuntu-2604
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:resolute
steps:
- name: 'Re-index the slot'
image: reg.devxy.io/rpkgs/build-env-${IMG}
pull: true
environment:
OTEL_R_TRACES_EXPORTER: none
OTEL_R_LOGS_EXPORTER: none
OTEL_R_METRICS_EXPORTER: none
RED_HAT_DEV_PW:
from_secret: RED_HAT_DEV_PW
B2_S3_ACCESS_KEY:
from_secret: B2_S3_ACCESS_KEY
B2_S3_SECRET_KEY:
from_secret: B2_S3_SECRET_KEY
REPO_RO_TOKEN:
from_secret: REPO_RO_TOKEN
GIT_USER: pat-s
R_LIBS_USER: /mnt/cache/R-pkgs
R_VERSION: ${R_VERSION}
PLATFORM: ${OS}
ARCH: ${ARCH}
commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- mkdir -p /mnt/cache/R-pkgs
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
# The codename is detected from the image's /etc/os-release.
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
- |
for RBIN in /opt/R/[0-9]*/bin/R; do
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
done
- name: Purge CDN cache
image: reg.devxy.io/docker.io/library/alpine:3.24
environment:
OTEL_R_TRACES_EXPORTER: none
OTEL_R_LOGS_EXPORTER: none
OTEL_R_METRICS_EXPORTER: none
BUNNYNET_API_KEY:
from_secret: BUNNYNET_API_KEY
# cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
# Bunny pull zones, so both must be purged after the shared origin changes.
BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io'
commands:
- apk add --no-cache -q bash curl jq
# Crow carries the checkout from the re-index step into this step.
- bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
# Runs on every row rather than on one designated slot: a cron fires only
# its own slot's row, so gating on a named slot would leave every other
# slot unpurged. A manual "all" run therefore purges the zone 18 times,
# which is a cheap API call and rare.
#
# Run it even when the re-index above failed: the objects were still
# replaced, and a stale edge is exactly what keeps them hidden.
when:
- status: [success, failure]

View file

@ -7,39 +7,30 @@ exclude: |
benchmark/|
docker/reprex/
)
# The `^local/patches/.*\.patch$` excludes below keep unified diffs byte-exact:
# a context line for a blank line is a single space, and stripping it (or
# appending a newline) makes `git apply` reject the patch, which surfaces as
# "patch did not apply cleanly" at build time rather than as a lint failure
# here. The exclusions are per-hook, not global, so `validate patch registry`
# still runs when a patch changes.
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0
hooks:
- id: end-of-file-fixer
exclude: ^local/patches/.*\.patch$
- id: trailing-whitespace
exclude: ^local/patches/.*\.patch$
args:
- --markdown-linebreak-ext=md
- repo: https://github.com/DavidAnson/markdownlint-cli2
rev: v0.23.2
rev: v0.23.0
hooks:
- id: markdownlint-cli2
- repo: https://github.com/rbubley/mirrors-prettier
rev: v3.9.6
rev: v3.9.4
hooks:
- id: prettier
- repo: https://github.com/posit-dev/air-pre-commit
rev: 0.11.0
rev: 0.10.0
hooks:
- id: air-format
- repo: https://github.com/editorconfig-checker/editorconfig-checker
rev: v3.11.2
rev: v3.8.0
hooks:
- id: editorconfig-checker
exclude: ^local/patches/.*\.patch$
- repo: https://github.com/adrienverge/yamllint.git
rev: v1.38.0
hooks:

134
.terraform.lock.hcl generated
View file

@ -2,79 +2,79 @@
# Manual edits may be lost in future updates.
provider "registry.opentofu.org/hashicorp/http" {
version = "3.6.1"
version = "3.6.0"
hashes = [
"h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=",
"h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=",
"h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=",
"h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=",
"h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=",
"h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=",
"h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=",
"h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=",
"h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=",
"h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=",
"h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=",
"h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=",
"h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=",
"h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=",
"h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=",
"zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9",
"zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6",
"zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab",
"zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b",
"zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c",
"zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4",
"zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502",
"zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69",
"zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6",
"zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279",
"zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6",
"zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7",
"zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df",
"zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621",
"zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424",
"h1:0n4RBz9zNw6TTddh5+x7E8L2+qzPXNwKhK4uoZ/DUwE=",
"h1:22Ob7lpzMBSqdrCvoFN5EgmhGPHPBovV/9qo0c/Cd+A=",
"h1:2IRBvmWOYrq/ooaYYn2i86jZb7iIUvlg0KlmOMfDHoQ=",
"h1:5mucXikk4OcW3un3u94QnMx4AB4Wfih+sXeMd5QxSNk=",
"h1:5oU7Zm+2gAVGmxqtJ9E8uTudUkYy/DEn/y3IWphdv4k=",
"h1:5w0R4b1/VSzpqQF1tXXPr/qmaQLPVRXamOmPKWFcTk4=",
"h1:AEVeJr8xGmwad+JUUQ833C3x5d4W+W2szF5DfwxYppw=",
"h1:CPHJ+0zQbS/cX1m55Y90jIOgf1jV3ocUUnqsXAh+9Eg=",
"h1:JPewnGDOJudNer5+ghqwXoaJkfot3QRq9uiEYvo+JHU=",
"h1:QzbluV2vQLxsJYxjpziQCmPndIoJ/UGS4/UHH/GpwUM=",
"h1:TjUNbUdqweRBq/ycQ4ixpNkx5qaYwpXEOn9QCpqNZP8=",
"h1:XNbcODP60ajj21N/OO7af8bBg1ltIsYkq9egn7BYbiY=",
"h1:tgrbgmX7WYQz9G9ncgu7TkpVB+RlLjJA/Rvp9KPlZH8=",
"h1:vLxthX/ZWsOZ+aHKbAMqmNKqD0K5f4nJ8ppy0Ioyup0=",
"h1:wZOdGBAZkY8OKEPjKz82j1HloAKOmmvtjWyTxM+I110=",
"zh:0f719fa5426bc883e9fa6abf7f6498e48025edafbc29015e2f5c028f1cca3b9d",
"zh:1b4d7dafefd6c61764b2f9ed6943ceb9a200dee3590d18747e3a5f6b20ce85e0",
"zh:1d23a712984866d29f7b07028a4e99c783c71f1a5dddf08bc3d4e7da9d91a1fa",
"zh:257d23d58c3bb024b6bc8eb88736eaf912e934ad47c639d0c3c742bddda849a1",
"zh:479860e1a5468f5e04013b9364c9496d7ed0804bf9a1acd8e07558d57609993d",
"zh:4cb5e681bf599b411b27c4a2c4066a5fb2ed79aaa3a1a3cb5a30002fec062ce9",
"zh:4fb35c3f643dae9f3670d719397a415f815a0b95f8ed7bd8a72f27a94ba78092",
"zh:59ba40825ab38db5b4a0989a2db0df35cc15d8984f898176011ba352f27d77b7",
"zh:61fc1252eb88088638f4c69ea4e2171cde2e5089fa632ac1e943b13787348f73",
"zh:7c5d6dd5f7cbc460e95d368be35c29b4e0402069b8912dbd5d1cd7fa9acef216",
"zh:7f76d756240d4284642f359ad470226e5378670239aadc366ef54d9d914d4d2e",
"zh:8133ad0814098177e0d067c816ccf1bf48bbadacd18f6f2c808c90447505723b",
"zh:c93be06269bb728f1968f8c50506de56c887017ac1d6e4be1f925651d8437eb6",
"zh:ef47b78a10a82e6cf53344a6a85a94041c28286c10a70541c564d762f1cfede0",
"zh:f5796a53a74999135bd9087aff50fddda59129d09b2f9b1902ff8c0c1e047e48",
]
}
provider "registry.terraform.io/bunnyway/bunnynet" {
version = "0.18.2"
constraints = "~> 0.18"
version = "0.15.1"
constraints = "~> 0.15"
hashes = [
"h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=",
"h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=",
"h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=",
"h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=",
"h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=",
"h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=",
"h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=",
"h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=",
"h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=",
"h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=",
"h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=",
"h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=",
"h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=",
"h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=",
"h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=",
"h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=",
"h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=",
"zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c",
"zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178",
"zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd",
"zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef",
"zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738",
"zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc",
"zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c",
"h1:/2NUpbtjkc+w6n5V3kGP0rSzGjN0K2Wdfe2K+CZdmhU=",
"h1:1TOrpmCR0aT5xX1sjt70zqlYkMJnVe6r0nx2B0DS/mE=",
"h1:4uC8r8ILr+vZJ230uGqIUaWQ7uORCrIzoEuYrAq2JuE=",
"h1:6b+2osJUfwcaYZ6mathLPf/58sr/4XkLXQrcSufnkMk=",
"h1:CPldfjf79QS8mIP+GWoS0FVhrFlyjvN2+39zfyP76Ik=",
"h1:EvXICHyGIKpoYlDeKHOPzfmpvdRdhgsOMcR2nb6+tKY=",
"h1:GZLq+nDxS1CyBH0ELGTSQj9X7ozemJ1jpPA4KwbtR+c=",
"h1:ISNFqL745IQgZ6yMLy8ofV8ixbYqZYa9JKdi2W3pmNk=",
"h1:IrNrEuvFd0nYDGQefwmT8d1CSJb9e8LN5w9vw1ODp7E=",
"h1:Ms79slY9bZ94+n4cwIHvI9+/cvbucwo6S1+z5KAiznw=",
"h1:NWA9XSEBcpSkgwwIvl6tHrxGQY3uYhqNS4Vnb9RLyLQ=",
"h1:UjvxxxggicLtiE3yTe1Gx0oLUTeZpWmgIfXuHzWHn1c=",
"h1:VgJjo14DGkU4Jwo4D3GT4/5sq1tdjiZscKS5l3cb890=",
"h1:dBu3AW5YNLIvbBIMNk3wUHKw4TW+BDbj34a+mCqYhWE=",
"h1:i5oGD06nQ3JRsBIa2u4wCej+ETgp970CFl75dOKkHno=",
"h1:mTqR+vD1AWPx+mu7S0/pzBy71z7WOrjH7arOP77PXh4=",
"h1:nmTM61G8vYjpofeEqspMORpsNvTGCNZySGfjdXardL0=",
"zh:0f9bf5aaa47164a4d6ae4433d5e285a9456a5053401b2bad4ed68622f574ddee",
"zh:3039bee421fb8855a919f449fc731d145254371f5f0c39cc4660f3aeed6a8b10",
"zh:36664b08186e0c194747b18dee24ed97327c6e704133d4cf0df27abef1652f86",
"zh:3c7eae99d8c5ff65dfb99c8b9c1980147282d68971e1ef1ff0f126a6bff59d8f",
"zh:5293cc21abf54f4e5745437ca2d40d206aae323b2e1d41cf45dcc63a8868cbb3",
"zh:5994e5145e616e7e881010717e4c7def2945eb6d933f62db4ec3167732ccac84",
"zh:7994db9ed3fdb6cbf21f2154ea962cf82f04e425988bedc6657de2497d6cb6c3",
"zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f",
"zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef",
"zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3",
"zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb",
"zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e",
"zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11",
"zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7",
"zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9",
"zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1",
"zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff",
"zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339",
"zh:93aa863e536ba9376ccf9e614e9edc9b214a2ce8c4316d416a8e249b436f52d2",
"zh:ab5cb4baeda57559686a0ccf0e09158aa64624ee6ba0ef32b769f13b11a43068",
"zh:ae9388b62eede8fd9272407bf75f8241a965bd489d45ec9dd3f9fac696d500e1",
"zh:caa5befd16960e2f69c7ec483e228e5ff43ab0979c17f1b874c9ffaa1c7c0e43",
"zh:cddd3e1067defa06a4e4ad5cb3940c7943e29c42417de57236aa7d3e2aeaae13",
"zh:cdfa44d591d0805116159556947904d70f534c6816188c45cf3a7544d2722ac9",
"zh:d607e9f1f3e09f13404f219e1893e3b3c77aece4afb54e999f934c021f41f576",
"zh:d8a397aca95125c6a0c0c78d2ded5843b9204effa9f5cf7419f017b500ad9228",
"zh:f5499eaff0d221725ad209d27d87c5b46d5c554caad7dc42947c760377abe3b0",
"zh:fc5f5cf433abc83e5169fa222992ec521c0c802075970251e3dd2c1d50c4f5c1",
]
}

View file

@ -56,7 +56,7 @@ For every package+tag combination:
## Patching packages
Some CRAN packages fail to compile on specific platforms due to compiler- or OS-specific issues unrelated to the package itself.
The canonical example is `RcppParallel`, whose bundled TBB is linked in a way that lets a system TBB on the build host shadow it, so the published binary depends on a library the consumer does not have.
The canonical example is `RcppParallel`, whose bundled TBB sources fail on musl (Alpine) and newer compiler/OS combinations.
Because such packages are often transitive dependencies of many others, a single failure cascades: all dependents fail even though nothing is wrong with the dependent itself.
To address this, frequently-failing packages can be "patched" before they are installed — whether as a direct build target or a transitive dependency pulled in by `pak`.
@ -112,9 +112,9 @@ Processing updates with an existing database file takes around 5 minutes.
### Inferring System Dependencies
R package dependencies and their system dependencies are installed through [uvr](https://github.com/nbafrank/uvr).
`uvr` allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`.
Additionally, it automatically infers package dependencies using JSON rules vendored from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements).
R package dependencies and their system dependencies are installed through {pak}.
{pak} allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`.
Additionally, it automatically infers package dependencies using JSON rules from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements).
Not all R packages specify required system dependencies in their DESCRIPTION file, and not all listed dependencies have existing rules in `rstudio/r-system-requirements`.
For Alpine, no rules existed until recently, establishing a foundation for semi-automated package installation on Alpine Linux.
@ -152,13 +152,12 @@ A Shiny dashboard providing a search functionality of the database and grouped s
Is supported by writing `Meta/archive.rds` during each package index update, listing all available archived packages.
### `uvr add package@version`
### `pak::pak(package@version)`
Clients that resolve archived versions search for `Archive/<package>` and can install all versions they find.
This holds for `uvr` as well as for older `pak`-based clients.
`pak` searches for `Archive/<package>` and can install all versions it finds.
Ensure to use a clean cache if other repositories have been used previously.
If in doubt or when testing, call `uvr cache clean`.
If in doubt or when testing, call `pak::meta_clean(force = TRUE)`.
## Lessons Learned
@ -181,7 +180,7 @@ If in doubt or when testing, call `uvr cache clean`.
## URL Composition and Platform Identifiers
Platform identifiers have been aligned with those used in <https://github.com/rstudio/r-system-requirements>, which `uvr` vendors and uses for its automatic syslib dependency installer (`uvr sync --install-system-deps`):
Platform identifiers have been aligned with those used in <https://github.com/rstudio/r-system-requirements> to ensure proper recognition by the automatic syslib dependency installer of `pak`, specifically via the environment variable `PKG_SYSREQS_PLATFORM`:
- redhat-9
- redhat-8
@ -238,8 +237,8 @@ internal error 1 in memDecompress
Solution:
```sh
uvr cache clean
rm -rf /mnt/cache/uvr /root/.cache/R/
rm -rf /mnt/cache/R-pkgs/pak /mnt/cache/pkgcache/ /root/.cache/R/
R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
```
</details>

View file

@ -30,9 +30,8 @@
# from_secret: HETZNER_S3_SECRET_KEY_K3S
# # normal env vars
# GIT_USER: pat-s
# # set the location of uvr's caches, which persist the R package dependencies needed to install the packages themselves
# # UVR_CACHE_DIR: /mnt/cache/uvr/cache
# # UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
# # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
# # R_PKG_CACHE_DIR: /mnt/cache/pkgcache
# R_LIBS_USER: /mnt/cache/R-pkgs
# CCACHE_DIR: /mnt/cache/ccache
# volumes:
@ -40,8 +39,9 @@
# commands:
# - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
# - mkdir -p /mnt/cache/R-pkgs
# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/uvr
# - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft
# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/pkgcache
# - /opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
# - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git", dependencies = TRUE)'
# - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
# # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine322', 'amd64'); saveRDS(pkgs, '/mnt/cache/pkgs_amd64.rds')"
# - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine323', 'arm64'); saveRDS(pkgs, '/mnt/cache/pkgs_arm64.rds')"

103
cdn.tf
View file

@ -32,7 +32,7 @@
# cache_stale = ["offline", "updating"]
# use_background_update = true
# block_ips = var.cdn_block_ips
# block_ips = var.cdn_block_ips
# # 50 TB
# limit_bandwidth = 50000000000000
@ -52,29 +52,6 @@
### cran.rpkgs.com
# The edge middleware that resolves the bare cran.rpkgs.com form to an
# <arch>/<os> slot and routes PACKAGES* to the per-R-minor slot. The source of
# truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release.
#
# The script pre-dates this configuration, so it is adopted rather than created:
# tofu import bunnynet_compute_script.rpkgs_router 29277
resource "bunnynet_compute_script" "rpkgs_router" {
type = "middleware"
name = "rpkgs-router"
content = file("${path.module}/edge/rpkgs-router.ts")
}
# Slots ("<arch>/<os>", comma separated) whose per-minor index bincraft has
# already republished as a union of the per-minor and flat slots. Routing to a
# slot that is not listed here would hide every package the per-minor index does
# not carry, so this stays empty until a slot has been backfilled.
resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" {
script = bunnynet_compute_script.rpkgs_router.id
name = "UNION_SLOTS"
default_value = ""
required = false
}
resource "bunnynet_pullzone" "cran_rpkgs_com" {
name = "cran-rpkgs"
@ -82,12 +59,12 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" {
cache_expiration_time = 31919000
websockets_enabled = false
errorpage_whitelabel = true
errorpage_whitelabel = true
origin {
type = "OriginUrl"
url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com"
middleware_script = bunnynet_compute_script.rpkgs_router.id
middleware_script = 29277
}
routing {
@ -105,21 +82,6 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" {
request_coalescing_enabled = true
block_post_requests = true
# Set on the zone since before this configuration existed; declared here so
# `tofu apply` stops silently removing it.
#
# The router makes it redundant on paper: the only UA-dependent responses it
# produces are redirects, and those carry `Cache-Control: no-store`, while
# their targets are concrete per-slot, per-minor URLs whose content depends
# only on the path. Dropping it would also be a real win, because otherwise
# every distinct R version string keys its own copy of every tarball.
#
# It stays for now anyway: it is the second line of defence against the one
# failure that would be quiet and confusing (an R 4.6 client served the 4.5
# index), and removing it is worth doing on its own once per-minor routing is
# confirmed live, not as a side effect of enabling that routing.
cache_vary_headers = ["User-Agent"]
limit_requests = 5000
limit_connections = 1000
@ -147,65 +109,6 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" {
tls_enabled = true
}
# Alliance SwissPass historically used a separate, manually configured pull
# zone. Adopt it so both public repositories use the same B2 origin, middleware
# release and cache behavior.
import {
to = bunnynet_pullzone.cran_allianceswisspass
id = "3265648"
}
resource "bunnynet_pullzone" "cran_allianceswisspass" {
name = "cran-allianceswisspass"
cache_errors = false
cache_expiration_time = 31919000
websockets_enabled = false
errorpage_whitelabel = true
origin {
type = "OriginUrl"
url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com"
middleware_script = bunnynet_compute_script.rpkgs_router.id
}
routing {
filters = [
"scripting",
]
}
s3_auth_enabled = true
s3_auth_key = var.B2_S3_ACCESS_KEY
s3_auth_secret = var.B2_S3_SECRET_KEY
s3_auth_region = "eu-central-003"
cache_enabled = true
request_coalescing_enabled = true
block_post_requests = true
cache_vary_headers = ["User-Agent"]
limit_requests = 5000
limit_connections = 1000
safehop_enabled = true
add_canonical_header = true
cache_stale = ["offline", "updating"]
block_ips = var.cdn_block_ips
# 50 TB
limit_bandwidth = 50000000000000
block_root_path = true
}
resource "bunnynet_pullzone_hostname" "cran_allianceswisspass" {
pullzone = bunnynet_pullzone.cran_allianceswisspass.id
name = "cran.allianceswisspass.devxy.io"
force_ssl = true
tls_enabled = true
}
# resource "bunnynet_storage_zone" "devxy-r-binaries" {
# name = "devxy-r-binaries-storage"
# region = "DE"

View file

@ -1,15 +1,7 @@
FROM devxygmbh/r-alpine:4.4-3.20 AS build
# ARG GITHUB_PAT
RUN apk add --no-cache curl ca-certificates \
&& curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh \
| UVR_INSTALL_DIR=/usr/local/bin sh
# One uvr project drives both dependency installs below. UVR_LIBRARY points the
# syncs at the image's R library instead of the project-local .uvr/library/, so
# `Rscript app.R` finds the packages without a uvr-aware .Rprofile.
ENV UVR_LIBRARY=/usr/lib/R/library
RUN mkdir -p /uvr && cd /uvr && uvr init --here
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
COPY --link ./DESCRIPTION .
COPY --link ./R ./R
@ -19,19 +11,13 @@ COPY --link ./shiny/app.R /app/app.R
RUN ls -la
# install R package deps. uvr has no equivalent of bare `pak::pak()`, which
# reads the DESCRIPTION in the working directory, so extract the dependency
# names and hand them to `uvr add`.
RUN R -q --no-echo -e "d <- read.dcf('DESCRIPTION'); f <- intersect(colnames(d), c('Depends', 'Imports', 'LinkingTo')); p <- trimws(sub('[(].*', '', unlist(strsplit(paste(d[, f], collapse = ','), ',')))); writeLines(setdiff(p[nzchar(p)], c('R', rownames(installed.packages()))), '/tmp/deps.txt')" \
&& cd /uvr && xargs -r uvr add --no-install < /tmp/deps.txt \
&& uvr sync --install-system-deps
# install R package deps
RUN R -q -e 'pak::pak()'
RUN R CMD INSTALL --no-docs --without-keep.source .
# install shiny app deps
RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); writeLines(pkgs, '/tmp/app-deps.txt')" \
&& cd /uvr && xargs -r uvr add --no-install < /tmp/app-deps.txt \
&& uvr sync --install-system-deps
RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); pak::pak(pkgs)"
ENV PGPASS=""

View file

@ -17,10 +17,7 @@ ARG CACHEBUST
WORKDIR /work
COPY build-one.R /work/build-one.R
# Resolve and install the latest bincraft release dynamically (no hardcoded pin).
# uvr-install.sh lands under /work/local/ because install-bincraft.R looks for it
# there when the working directory is not a repo checkout.
COPY install-bincraft.R /work/install-bincraft.R
COPY uvr-install.sh /work/local/uvr-install.sh
# Ship the patch registry so build-one.R's `patches = "local/patches"` resolves
# (build context is `local/`, CWD is /work).
COPY patches /work/local/patches

View file

@ -1,14 +1,10 @@
docker run --rm -it --platform linux/arm64 alpine sh
apk add --no-cache R R-dev g++ curl ca-certificates
apk add --no-cache R R-dev g++
curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | UVR_INSTALL_DIR=/usr/local/bin sh
R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
# `uvr add` always writes to .uvr/library/; only `uvr sync` honours UVR_LIBRARY,
# so add without installing and let the sync place the packages.
export UVR_LIBRARY=/usr/lib/R/library
mkdir -p /uvr && cd /uvr && uvr init --here
uvr add --no-install gert purrr && uvr sync --install-system-deps
R -q -e 'pak::pak(c("gert", "purrr"))'
R
@ -26,20 +22,14 @@ unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE)
tag <- all_tags$name
package_name <- rep(package_name, length(tag))
# uvr has no `pak::local_install_deps()`; read the DESCRIPTION of the checkout
# and `uvr add` the dependency names instead.
purrr::walk2(package_name[1], tag, \(x, y) {
print(y)
system("git config --global advice.detachedHead false")
src <- sprintf("/tmp/%s_%s", x[1], y)
system2("git", args = c(
"clone", "-q", sprintf("--branch=%s", tail(y, 1)),
sprintf("https://github.com/cran/%s", x[1]), src
sprintf("https://github.com/cran/%s", x[1]), sprintf("/tmp/%s_%s", x[1], y)
))
d <- read.dcf(file.path(src, "DESCRIPTION"))
f <- intersect(colnames(d), c("Depends", "Imports", "LinkingTo"))
deps <- trimws(sub("[(].*", "", unlist(strsplit(paste(d[, f], collapse = ","), ","))))
deps <- setdiff(deps[nzchar(deps)], "R")
system2("uvr", c("add", "--no-install", deps))
system2("uvr", c("sync", "--install-system-deps"))
pak::local_install_deps(sprintf("/tmp/%s_%s", x[1], y))
})

View file

@ -1,167 +0,0 @@
/**
* Routing matrix for `edge/rpkgs-router.ts`.
*
* The script is exercised through the SDK's local server rather than by
* importing its internals, so what is tested is the artifact that gets
* deployed. Requests that the script passes through are proxied to the real
* origin, which keeps the "no redirect" cases honest: they assert that the
* client reached the flat slot, not merely that no `Location` was set.
*
* Run with `just edge-test`.
*/
import { assertEquals } from 'jsr:@std/assert@1';
const SCRIPT = new URL('./rpkgs-router.ts', import.meta.url).pathname;
const BASE = 'http://127.0.0.1:8080';
const UNION_SLOTS = 'amd64/alpine324';
const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)';
const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)';
const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24';
const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)';
const UA_CURL = 'curl/8.0.1';
const SLOT = '/amd64/alpine324/latest/src/contrib';
const OTHER_SLOT = '/amd64/noble/latest/src/contrib';
interface Probe {
status: number;
location: string | null;
cacheControl: string | null;
}
async function probe(path: string, userAgent: string): Promise<Probe> {
const res = await fetch(BASE + path, {
headers: { 'User-Agent': userAgent },
redirect: 'manual',
});
await res.body?.cancel();
return {
status: res.status,
location: res.headers.get('location'),
cacheControl: res.headers.get('cache-control'),
};
}
/** Kill tolerantly: the child has already exited if the script failed to load. */
async function stopServer(child: Deno.ChildProcess): Promise<void> {
try {
child.kill();
} catch {
// already gone
}
await child.status;
}
async function startServer(): Promise<Deno.ChildProcess> {
const child = new Deno.Command(Deno.execPath(), {
args: ['run', '-A', SCRIPT],
env: { UNION_SLOTS },
stdout: 'null',
stderr: 'inherit',
}).spawn();
for (let attempt = 0; attempt < 150; attempt++) {
try {
const res = await fetch(`${BASE}/`, {
headers: { 'User-Agent': UA_CURL },
redirect: 'manual',
});
await res.body?.cancel();
return child;
} catch {
await new Promise((resolve) => setTimeout(resolve, 200));
}
}
await stopServer(child);
throw new Error('edge script did not start listening on ' + BASE);
}
Deno.test('rpkgs-router', async (t) => {
const server = await startServer();
try {
await t.step("routes an index request to the client's R minor", async () => {
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL);
assertEquals(res.status, 302);
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`);
});
await t.step('routes R 4.6 to its own slot', async () => {
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R46_MUSL);
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`);
});
await t.step('routes PACKAGES and PACKAGES.rds too', async () => {
for (const file of ['PACKAGES', 'PACKAGES.rds']) {
const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL);
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/${file}`, `expected ${file} to be routed`);
}
});
await t.step('marks the redirect uncacheable', async () => {
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL);
assertEquals(res.cacheControl, 'no-store');
});
await t.step('leaves a slot outside UNION_SLOTS alone', async () => {
const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R45_MUSL);
assertEquals(res.location, null);
assertEquals(res.status, 200);
});
await t.step('never routes a tarball', async () => {
const res = await probe(`${SLOT}/jsonlite_2.0.0.tar.gz`, UA_R45_MUSL);
assertEquals(res.location, null);
assertEquals(res.status, 200);
});
await t.step('serves an archived binary when it exists', async () => {
const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`;
const res = await probe(path, UA_R45_MUSL);
assertEquals(res.status, 200);
assertEquals(res.location, null);
});
await t.step('does not redirect a path already under a minor', async () => {
const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL);
assertEquals(res.location, null);
assertEquals(res.status, 200);
});
await t.step('leaves a client without an R version alone', async () => {
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_CURL);
assertEquals(res.location, null);
assertEquals(res.status, 200);
});
await t.step('resolves the bare root to slot and minor', async () => {
const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_ALPINE);
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`);
});
await t.step('sends an unidentifiable distro to CRAN', async () => {
const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL);
assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz');
});
await t.step('keeps the macOS rewrite', async () => {
const res = await probe('/src/contrib/foo_1.0.tar.gz', UA_R45_DARWIN);
assertEquals(res.location, 'https://cran.rpkgs.com/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz');
});
await t.step('keeps the macOS binary passthrough to CRAN', async () => {
const path = '/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz';
const res = await probe(path, UA_R45_DARWIN);
assertEquals(res.location, `https://cran.r-project.org${path}`);
});
await t.step('collapses duplicate slashes before matching', async () => {
const res = await probe(`/amd64/alpine324//latest/src/contrib//PACKAGES.gz`, UA_R45_MUSL);
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`);
});
} finally {
await stopServer(server);
}
});

Some files were not shown because too many files have changed in this diff Show more