diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index a67875d..bb00332 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -173,13 +173,13 @@ steps: OTEL_R_METRICS_EXPORTER: none BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY - # All hostnames on the zone share this id, so one purge covers - # cran.devxy.io, cran.allianceswisspass.devxy.io and cran.rpkgs.com. - BUNNY_PULLZONE: '3857050' + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + BUNNY_PULLZONES: '3857050 3265648' commands: - apk add --no-cache -q bash curl # Crow carries the checkout from the re-index step into this step. - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" "$BUNNY_PULLZONE" + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES # Runs on every row rather than on one designated slot: a cron fires only # its own slot's row, so gating on a named slot would leave every other # slot unpurged. A manual "all" run therefore purges the zone 18 times, diff --git a/cdn.tf b/cdn.tf index 5e8899d..01191a5 100644 --- a/cdn.tf +++ b/cdn.tf @@ -32,7 +32,7 @@ # cache_stale = ["offline", "updating"] # use_background_update = true - # block_ips = var.cdn_block_ips +# block_ips = var.cdn_block_ips # # 50 TB # limit_bandwidth = 50000000000000 @@ -82,7 +82,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { cache_expiration_time = 31919000 websockets_enabled = false - errorpage_whitelabel = true + errorpage_whitelabel = true origin { type = "OriginUrl" @@ -147,6 +147,58 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } +# Alliance SwissPass historically used a separate, manually configured pull +# zone. Adopt it so both public repositories use the same B2 origin, middleware +# release and cache behavior. +import { + to = bunnynet_pullzone.cran_allianceswisspass + id = "3265648" +} + +resource "bunnynet_pullzone" "cran_allianceswisspass" { + name = "cran-allianceswisspass" + + cache_errors = false + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + + origin { + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router.id + } + + routing { + filters = [ + "scripting", + ] + } + + s3_auth_enabled = true + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" + + cache_enabled = true + request_coalescing_enabled = true + block_post_requests = true + cache_vary_headers = ["User-Agent"] + + limit_requests = 5000 + limit_connections = 1000 + + safehop_enabled = true + add_canonical_header = true + cache_stale = ["offline", "updating"] + block_ips = var.cdn_block_ips + + # 50 TB + limit_bandwidth = 50000000000000 + + block_root_path = true +} + # resource "bunnynet_storage_zone" "devxy-r-binaries" { # name = "devxy-r-binaries-storage" # region = "DE" diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 553185d..9493f8a 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -117,6 +117,13 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.status, 200); }); + await t.step('serves an archived binary when it exists', async () => { + const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`; + const res = await probe(path, UA_R45_MUSL); + assertEquals(res.status, 200); + assertEquals(res.location, null); + }); + await t.step('does not redirect a path already under a minor', async () => { const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); assertEquals(res.location, null); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index 9cd410b..9278d4a 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -27,6 +27,7 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; const CRAN_ORIGIN = 'https://cran.r-project.org'; +const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']); /** Slots ("/", comma separated) whose per-minor index is a union. */ const UNION_SLOTS = new Set( @@ -47,6 +48,10 @@ const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; +/** A binary archive URL whose upstream source counterpart CRAN can serve. */ +const ARCHIVE_TARBALL_REGEX = + /^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/; + const MACOS_BIN_REGEX = /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; @@ -85,6 +90,10 @@ function redirectTo(location: string, status = 302): Response { }); } +function publicCdnOrigin(url: URL): string { + return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; +} + function extractRMinor(userAgent: string): string | null { for (const regex of R_MINOR_REGEXES) { const match = userAgent.match(regex); @@ -181,15 +190,14 @@ BunnySDK.net.http const url = new URL(ctx.request.url); const path = normalizePathname(url.pathname); const userAgent = ctx.request.headers.get('User-Agent') || ''; + const publicOrigin = publicCdnOrigin(url); // macOS clients are served from CRAN's own binary tree. const srcContrib = path.match(SRC_CONTRIB_REGEX); if (srcContrib && /darwin/.test(userAgent)) { const mac = parseMacUserAgent(userAgent); if (mac) { - return Promise.resolve( - redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`), - ); + return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`)); } } @@ -213,7 +221,7 @@ BunnySDK.net.http if (target === path) { return Promise.resolve(ctx.request); } - return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`)); + return Promise.resolve(redirectTo(`${publicOrigin}${target}`)); } // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. @@ -224,12 +232,27 @@ BunnySDK.net.http } const rest = srcContrib ? srcContrib[1] : ''; - return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`)); + return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); } return Promise.resolve(ctx.request); }) - .onOriginResponse((ctx) => { + .onOriginResponse(async (ctx) => { + const path = normalizePathname(new URL(ctx.request.url).pathname); + const archive = path.match(ARCHIVE_TARBALL_REGEX); + + // Binary archives can be incomplete when an older build never succeeded. + // Preserve renv/remotes version restores by falling back to CRAN's source + // package only for an absent archived tarball. A requested version can be + // either archived upstream or still current, so probe the archive first. + // Other 404s remain visible. + if (ctx.response.status === 404 && archive) { + const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`; + const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' }); + const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`; + return redirectTo(sourceUrl); + } + ctx.response.headers.append('X-Via', 'MyMiddleware'); return Promise.resolve(ctx.response); }); diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 391a814..4853245 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -18,35 +18,38 @@ # objects were replaced. The cost is a cold cache for everything else, which is # why this is not used by the daily update path. # -# All hostnames on the zone (cran.devxy.io, cran.allianceswisspass.devxy.io, -# cran.rpkgs.com) share pull zone 3857050, so one purge covers all of them. +# The public hostnames currently use separate pull zones, so callers must pass +# every zone that serves the repository. # # Usage: -# purge_cdn_zone.sh +# purge_cdn_zone.sh [...] # set -euo pipefail if (($# < 2)); then - echo "usage: $0 " >&2 + echo "usage: $0 [...]" >&2 exit 2 fi api_key="$1" -zone_id="$2" +shift -echo "Purging BunnyCDN pull zone ${zone_id}" +for zone_id in "$@"; do + echo "Purging BunnyCDN pull zone ${zone_id}" -status=$( - curl -sS -o /tmp/purge_zone_response.txt -w '%{http_code}' -X POST \ - -H "AccessKey: ${api_key}" \ - -H "Content-Length: 0" \ - "https://api.bunny.net/pullzone/${zone_id}/purgeCache" -) + response_file="/tmp/purge_zone_response_${zone_id}.txt" + status=$( + curl -sS -o "${response_file}" -w '%{http_code}' -X POST \ + -H "AccessKey: ${api_key}" \ + -H "Content-Length: 0" \ + "https://api.bunny.net/pullzone/${zone_id}/purgeCache" + ) -if [[ "${status}" != "200" && "${status}" != "204" ]]; then - echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 - cat /tmp/purge_zone_response.txt >&2 - exit 1 -fi + if [[ "${status}" != "200" && "${status}" != "204" ]]; then + echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 + cat "${response_file}" >&2 + exit 1 + fi -echo "Purged pull zone ${zone_id} (HTTP ${status})" + echo "Purged pull zone ${zone_id} (HTTP ${status})" +done