pat-s
a1c1f5e78f
## Motivation `cran.rpkgs.com` and `cran.allianceswisspass.devxy.io` serve the same B2 repository through separate Bunny pull zones, but only the first zone was managed and purged after weekly reindexing. This allowed the Alliance endpoint to retain stale repository metadata and left locked `renv` restores unable to retrieve versions whose binary archive object was absent. ## Changes - Adopt the Alliance SwissPass pull zone `3265648` into OpenTofu and configure it with the shared B2 origin and middleware script. - Purge both Bunny pull zones after the weekly rebuild reindex. - Preserve the requested public hostname in middleware redirects. - Redirect missing archived binaries to the corresponding CRAN source package, checking whether the version is archived or still current. - Cover the existing archived-binary passthrough behavior in the edge routing matrix. ## Verification - `prek run -a` - `just edge-test` - `crow lint .crow/` - `tofu validate` - `bash -n scripts/purge_cdn_zone.sh` ## Deployment Run `tofu apply` to adopt pull zone `3265648`, publish the middleware release, and align both pull zones. After the apply, rerun the Alliance SwissPass CI restore that requested `cli 3.6.5` and `AzureStor 3.7.1`. Reviewed-on: #165
191 lines
6.1 KiB
YAML
191 lines
6.1 KiB
YAML
# Re-index and purge after weekly-rebuild-missing.
|
|
#
|
|
# weekly-rebuild-missing runs three shards per slot. Each of them replaces
|
|
# objects in place, so the slot's index still advertises the old MD5 and, for
|
|
# anything that had been served from source, no Built stamp. Re-indexing from
|
|
# inside a shard would mean three concurrent `upload_package_index()` calls on
|
|
# one prefix: `cranlike::update_PACKAGES()` lists the live bucket, so an early
|
|
# lister that uploads last publishes an index missing its siblings' work.
|
|
#
|
|
# So it happens exactly once per slot, here, after every shard has finished.
|
|
# `runs_on: [success, failure]` keeps that true when a shard fails; only an
|
|
# explicit cancel skips it, and this pipeline can then be triggered on its own.
|
|
|
|
variables:
|
|
# Mirrors the gate on weekly-rebuild-missing so a manual run re-indexes
|
|
# exactly the slots it rebuilt. A manual pipeline creation instantiates every
|
|
# file in .crow/, so the default must match no matrix row.
|
|
weekly_rebuild_missing:
|
|
description: "Manual run target: a specific <os>-<arch>, 'all' for every os/arch, or 'none' to run nothing."
|
|
options:
|
|
- none
|
|
- all
|
|
- alpine-322-amd64
|
|
- alpine-322-arm64
|
|
- alpine-323-amd64
|
|
- alpine-323-arm64
|
|
- alpine-324-amd64
|
|
- alpine-324-arm64
|
|
- redhat-8-amd64
|
|
- redhat-8-arm64
|
|
- redhat-9-amd64
|
|
- redhat-9-arm64
|
|
- redhat-10-amd64
|
|
- redhat-10-arm64
|
|
- ubuntu-2204-amd64
|
|
- ubuntu-2204-arm64
|
|
- ubuntu-2404-amd64
|
|
- ubuntu-2404-arm64
|
|
- ubuntu-2604-amd64
|
|
- ubuntu-2604-arm64
|
|
default: none
|
|
|
|
when:
|
|
- event: cron
|
|
cron: weekly-rebuild-missing-${OS}-${ARCH}
|
|
- event: manual
|
|
evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"'
|
|
|
|
depends_on:
|
|
- weekly-rebuild-missing
|
|
|
|
runs_on: [success, failure]
|
|
|
|
skip_clone: true
|
|
|
|
labels:
|
|
group: rpkgs-${ARCH}
|
|
|
|
matrix:
|
|
include:
|
|
- OS: alpine-322
|
|
ARCH: amd64
|
|
R_VERSION: 4.5.3
|
|
IMG: alpine:3.22
|
|
- OS: alpine-322
|
|
ARCH: arm64
|
|
R_VERSION: 4.5.3
|
|
IMG: alpine:3.22
|
|
- OS: alpine-323
|
|
ARCH: amd64
|
|
R_VERSION: 4.5.3
|
|
IMG: alpine:3.23
|
|
- OS: alpine-323
|
|
ARCH: arm64
|
|
R_VERSION: 4.5.3
|
|
IMG: alpine:3.23
|
|
- OS: alpine-324
|
|
ARCH: amd64
|
|
R_VERSION: 4.5.3
|
|
IMG: alpine:3.24
|
|
- OS: alpine-324
|
|
ARCH: arm64
|
|
R_VERSION: 4.5.3
|
|
IMG: alpine:3.24
|
|
- OS: redhat-8
|
|
ARCH: amd64
|
|
R_VERSION: 4.4.3
|
|
IMG: redhat:8
|
|
- OS: redhat-8
|
|
ARCH: arm64
|
|
R_VERSION: 4.4.3
|
|
IMG: redhat:8
|
|
- OS: redhat-9
|
|
ARCH: amd64
|
|
R_VERSION: 4.4.3
|
|
IMG: redhat:9
|
|
- OS: redhat-9
|
|
ARCH: arm64
|
|
R_VERSION: 4.4.3
|
|
IMG: redhat:9
|
|
- OS: redhat-10
|
|
ARCH: amd64
|
|
R_VERSION: 4.5.3
|
|
IMG: redhat:10
|
|
- OS: redhat-10
|
|
ARCH: arm64
|
|
R_VERSION: 4.5.3
|
|
IMG: redhat:10
|
|
- OS: ubuntu-2204
|
|
ARCH: amd64
|
|
R_VERSION: 4.4.3
|
|
IMG: ubuntu:jammy
|
|
- OS: ubuntu-2204
|
|
ARCH: arm64
|
|
R_VERSION: 4.4.3
|
|
IMG: ubuntu:jammy
|
|
- OS: ubuntu-2404
|
|
ARCH: amd64
|
|
R_VERSION: 4.4.3
|
|
IMG: ubuntu:noble
|
|
- OS: ubuntu-2404
|
|
ARCH: arm64
|
|
R_VERSION: 4.4.3
|
|
IMG: ubuntu:noble
|
|
- OS: ubuntu-2604
|
|
ARCH: amd64
|
|
R_VERSION: 4.4.3
|
|
IMG: ubuntu:resolute
|
|
- OS: ubuntu-2604
|
|
ARCH: arm64
|
|
R_VERSION: 4.4.3
|
|
IMG: ubuntu:resolute
|
|
|
|
steps:
|
|
- name: 'Re-index the slot'
|
|
image: reg.devxy.io/rpkgs/build-env-${IMG}
|
|
pull: true
|
|
environment:
|
|
OTEL_R_TRACES_EXPORTER: none
|
|
OTEL_R_LOGS_EXPORTER: none
|
|
OTEL_R_METRICS_EXPORTER: none
|
|
RED_HAT_DEV_PW:
|
|
from_secret: RED_HAT_DEV_PW
|
|
B2_S3_ACCESS_KEY:
|
|
from_secret: B2_S3_ACCESS_KEY
|
|
B2_S3_SECRET_KEY:
|
|
from_secret: B2_S3_SECRET_KEY
|
|
REPO_RO_TOKEN:
|
|
from_secret: REPO_RO_TOKEN
|
|
GIT_USER: pat-s
|
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
|
R_VERSION: ${R_VERSION}
|
|
PLATFORM: ${OS}
|
|
ARCH: ${ARCH}
|
|
commands:
|
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
|
- mkdir -p /mnt/cache/R-pkgs
|
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
|
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
|
# The codename is detected from the image's /etc/os-release.
|
|
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
|
- |
|
|
for RBIN in /opt/R/[0-9]*/bin/R; do
|
|
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
|
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
|
|
done
|
|
|
|
- name: Purge CDN cache
|
|
image: reg.devxy.io/docker.io/library/alpine:3.24
|
|
environment:
|
|
OTEL_R_TRACES_EXPORTER: none
|
|
OTEL_R_LOGS_EXPORTER: none
|
|
OTEL_R_METRICS_EXPORTER: none
|
|
BUNNYNET_API_KEY:
|
|
from_secret: BUNNYNET_API_KEY
|
|
# cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
|
|
# Bunny pull zones, so both must be purged after the shared origin changes.
|
|
BUNNY_PULLZONES: '3857050 3265648'
|
|
commands:
|
|
- apk add --no-cache -q bash curl
|
|
# Crow carries the checkout from the re-index step into this step.
|
|
- bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
|
|
# Runs on every row rather than on one designated slot: a cron fires only
|
|
# its own slot's row, so gating on a named slot would leave every other
|
|
# slot unpurged. A manual "all" run therefore purges the zone 18 times,
|
|
# which is a cheap API call and rare.
|
|
#
|
|
# Run it even when the re-index above failed: the objects were still
|
|
# replaced, and a stale edge is exactly what keeps them hidden.
|
|
when:
|
|
- status: [success, failure]
|