From a1c1f5e78f2e22db0497fd749067661f063f35ff Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 13 Aug 2026 14:08:10 +0000 Subject: [PATCH] fix(cdn): align repository routing across pull zones (#165) ## Motivation `cran.rpkgs.com` and `cran.allianceswisspass.devxy.io` serve the same B2 repository through separate Bunny pull zones, but only the first zone was managed and purged after weekly reindexing. This allowed the Alliance endpoint to retain stale repository metadata and left locked `renv` restores unable to retrieve versions whose binary archive object was absent. ## Changes - Adopt the Alliance SwissPass pull zone `3265648` into OpenTofu and configure it with the shared B2 origin and middleware script. - Purge both Bunny pull zones after the weekly rebuild reindex. - Preserve the requested public hostname in middleware redirects. - Redirect missing archived binaries to the corresponding CRAN source package, checking whether the version is archived or still current. - Cover the existing archived-binary passthrough behavior in the edge routing matrix. ## Verification - `prek run -a` - `just edge-test` - `crow lint .crow/` - `tofu validate` - `bash -n scripts/purge_cdn_zone.sh` ## Deployment Run `tofu apply` to adopt pull zone `3265648`, publish the middleware release, and align both pull zones. After the apply, rerun the Alliance SwissPass CI restore that requested `cli 3.6.5` and `AzureStor 3.7.1`. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/165 --- .crow/weekly-rebuild-reindex.yaml | 8 ++--- cdn.tf | 56 +++++++++++++++++++++++++++++-- edge/rpkgs-router.test.ts | 7 ++++ edge/rpkgs-router.ts | 35 +++++++++++++++---- scripts/purge_cdn_zone.sh | 39 +++++++++++---------- 5 files changed, 115 insertions(+), 30 deletions(-) diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index a67875d..bb00332 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -173,13 +173,13 @@ steps: OTEL_R_METRICS_EXPORTER: none BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY - # All hostnames on the zone share this id, so one purge covers - # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. - BUNNY_PULLZONE: '3857050' + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + BUNNY_PULLZONES: '3857050 3265648' commands: - apk add --no-cache -q bash curl # Crow carries the checkout from the re-index step into this step. - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES # Runs on every row rather than on one designated slot: a cron fires only # its own slot's row, so gating on a named slot would leave every other # slot unpurged. A manual "all" run therefore purges the zone 18 times, diff --git a/cdn.tf b/cdn.tf index 5e8899d..01191a5 100644 --- a/cdn.tf +++ b/cdn.tf @@ -32,7 +32,7 @@ # cache_stale = ["offline", "updating"] # use_background_update = true - # block_ips = var.cdn_block_ips +# block_ips = var.cdn_block_ips # # 50 TB # limit_bandwidth = 50000000000000 @@ -82,7 +82,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { cache_expiration_time = 31919000 websockets_enabled = false - errorpage_whitelabel = true + errorpage_whitelabel = true origin { type = "OriginUrl" @@ -147,6 +147,58 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } +# Alliance SwissPass historically used a separate, manually configured pull +# zone. Adopt it so both public repositories use the same B2 origin, middleware +# release and cache behavior. +import { + to = bunnynet_pullzone.cran_allianceswisspass + id = "3265648" +} + +resource "bunnynet_pullzone" "cran_allianceswisspass" { + name = "cran-allianceswisspass" + + cache_errors = false + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + + origin { + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router.id + } + + routing { + filters = [ + "scripting", + ] + } + + s3_auth_enabled = true + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" + + cache_enabled = true + request_coalescing_enabled = true + block_post_requests = true + cache_vary_headers = ["User-Agent"] + + limit_requests = 5000 + limit_connections = 1000 + + safehop_enabled = true + add_canonical_header = true + cache_stale = ["offline", "updating"] + block_ips = var.cdn_block_ips + + # 50 TB + limit_bandwidth = 50000000000000 + + block_root_path = true +} + # resource "bunnynet_storage_zone" "devxy-r-binaries" { # name = "devxy-r-binaries-storage" # region = "DE" diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 553185d..9493f8a 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -117,6 +117,13 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.status, 200); }); + await t.step('serves an archived binary when it exists', async () => { + const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`; + const res = await probe(path, UA_R45_MUSL); + assertEquals(res.status, 200); + assertEquals(res.location, null); + }); + await t.step('does not redirect a path already under a minor', async () => { const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); assertEquals(res.location, null); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index 9cd410b..9278d4a 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -27,6 +27,7 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; const CRAN_ORIGIN = 'https://cran.r-project.org'; +const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']); /** Slots ("/", comma separated) whose per-minor index is a union. */ const UNION_SLOTS = new Set( @@ -47,6 +48,10 @@ const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; +/** A binary archive URL whose upstream source counterpart CRAN can serve. */ +const ARCHIVE_TARBALL_REGEX = + /^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/; + const MACOS_BIN_REGEX = /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; @@ -85,6 +90,10 @@ function redirectTo(location: string, status = 302): Response { }); } +function publicCdnOrigin(url: URL): string { + return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; +} + function extractRMinor(userAgent: string): string | null { for (const regex of R_MINOR_REGEXES) { const match = userAgent.match(regex); @@ -181,15 +190,14 @@ BunnySDK.net.http const url = new URL(ctx.request.url); const path = normalizePathname(url.pathname); const userAgent = ctx.request.headers.get('User-Agent') || ''; + const publicOrigin = publicCdnOrigin(url); // macOS clients are served from CRAN's own binary tree. const srcContrib = path.match(SRC_CONTRIB_REGEX); if (srcContrib && /darwin/.test(userAgent)) { const mac = parseMacUserAgent(userAgent); if (mac) { - return Promise.resolve( - redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`), - ); + return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`)); } } @@ -213,7 +221,7 @@ BunnySDK.net.http if (target === path) { return Promise.resolve(ctx.request); } - return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`)); + return Promise.resolve(redirectTo(`${publicOrigin}${target}`)); } // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. @@ -224,12 +232,27 @@ BunnySDK.net.http } const rest = srcContrib ? srcContrib[1] : ''; - return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`)); + return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); } return Promise.resolve(ctx.request); }) - .onOriginResponse((ctx) => { + .onOriginResponse(async (ctx) => { + const path = normalizePathname(new URL(ctx.request.url).pathname); + const archive = path.match(ARCHIVE_TARBALL_REGEX); + + // Binary archives can be incomplete when an older build never succeeded. + // Preserve renv/remotes version restores by falling back to CRAN's source + // package only for an absent archived tarball. A requested version can be + // either archived upstream or still current, so probe the archive first. + // Other 404s remain visible. + if (ctx.response.status === 404 && archive) { + const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`; + const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' }); + const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`; + return redirectTo(sourceUrl); + } + ctx.response.headers.append('X-Via', 'MyMiddleware'); return Promise.resolve(ctx.response); }); diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 391a814..4853245 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -18,35 +18,38 @@ # objects were replaced. The cost is a cold cache for everything else, which is # why this is not used by the daily update path. # -# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io, -# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them. +# The public hostnames currently use separate pull zones, so callers must pass +# every zone that serves the repository. # # Usage: -# purge_cdn_zone.sh +# purge_cdn_zone.sh [...] # set -euo pipefail if (($# < 2)); then - echo "usage: $0 " >&2 + echo "usage: $0 [...]" >&2 exit 2 fi api_key="$1" -zone_id="$2" +shift -echo "Purging BunnyCDN pull zone ${zone_id}" +for zone_id in "$@"; do + echo "Purging BunnyCDN pull zone ${zone_id}" -status=$( - curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \ - -H "AccessKey: ${api_key}" \ - -H "Content-Length: 0" \ - "https://api.bunny.net/pullzone/${zone_id}/purgeCache" -) + response_file="/tmp/purge_zone_response_${zone_id}.txt" + status=$( + curl -sS -o "${response_file}" -w '%{http_code}' -X POST \ + -H "AccessKey: ${api_key}" \ + -H "Content-Length: 0" \ + "https://api.bunny.net/pullzone/${zone_id}/purgeCache" + ) -if [[ "${status}" != "200" && "${status}" != "204" ]]; then - echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 - cat /tmp/purge_zone_response.txt >&2 - exit 1 -fi + if [[ "${status}" != "200" && "${status}" != "204" ]]; then + echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 + cat "${response_file}" >&2 + exit 1 + fi -echo "Purged pull zone ${zone_id} (HTTP ${status})" + echo "Purged pull zone ${zone_id} (HTTP ${status})" +done