From 4c1e9b773c8d973f851119b2df6552a4ecda9fa4 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 15:24:43 +0000 Subject: [PATCH 1/2] feat(edge): gate per-minor routing on published minors and add a staging zone Enabling UNION_SLOTS today would break every client on an R minor we do not publish. contribPath() redirects on any minor the User-Agent carries, without checking that the target exists and without a fallback, and only 4.4, 4.5 and 4.6 are published: a 4.3 client would be sent to a 404 and see no packages at all. - Gate routing on KNOWN_MINORS, falling back to the flat index otherwise. - Honour EXTRA_PUBLIC_HOSTS so the same script can run on a staging zone and redirect within itself instead of into production. - Add the cran-rpkgs-test pull zone with UNION_SLOTS pre-enabled, served on the bunny default hostname so it needs no DNS record. - Add scripts/verify-r-minor-routing.sh, covering all 16 slots: index reachability, the union property against flat, Path: target resolution, coverage parity across minors, and (--live) real User-Agent routing. - Cover the fallback in the edge test suite. --- cdn.tf | 91 ++++++++++ edge/rpkgs-router.test.ts | 27 ++- edge/rpkgs-router.ts | 33 +++- scripts/verify-r-minor-routing.sh | 277 ++++++++++++++++++++++++++++++ 4 files changed, 416 insertions(+), 12 deletions(-) create mode 100755 scripts/verify-r-minor-routing.sh diff --git a/cdn.tf b/cdn.tf index 96877b2..ffe9b21 100644 --- a/cdn.tf +++ b/cdn.tf @@ -147,6 +147,97 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } +### Staging zone for edge-router changes + +# Every published / slot. The staging zone enables per-minor routing +# for all of them at once; production adopts the same list only after +# `scripts/verify-r-minor-routing.sh --live` passes against staging. +locals { + rpkgs_slots = [ + for pair in setproduct( + ["amd64", "arm64"], + ["resolute", "noble", "jammy", "rhel8", "rhel9", "rhel10", "alpine323", "alpine324"] + ) : "${pair[0]}/${pair[1]}" + ] + + # bunny.net serves every pull zone on .b-cdn.net, so staging needs no + # DNS record and is never advertised. + rpkgs_test_hostname = "cran-rpkgs-test.b-cdn.net" +} + +# A second copy of the same router, bound to the same B2 origin, so UNION_SLOTS +# can be exercised end to end before production is touched. +resource "bunnynet_compute_script" "rpkgs_router_test" { + type = "middleware" + name = "rpkgs-router-test" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +resource "bunnynet_compute_script_variable" "rpkgs_router_test_union_slots" { + script = bunnynet_compute_script.rpkgs_router_test.id + name = "UNION_SLOTS" + default_value = join(",", local.rpkgs_slots) + required = false +} + +# Without this the staging zone rewrites to PUBLIC_CDN_ORIGIN, so its redirects +# land on production and the test silently measures the wrong system. +resource "bunnynet_compute_script_variable" "rpkgs_router_test_extra_hosts" { + script = bunnynet_compute_script.rpkgs_router_test.id + name = "EXTRA_PUBLIC_HOSTS" + default_value = local.rpkgs_test_hostname + required = false +} + +resource "bunnynet_pullzone" "cran_rpkgs_test" { + name = "cran-rpkgs-test" + + cache_errors = false + + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + + origin { + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router_test.id + } + + routing { + filters = [ + "scripting", + ] + } + + s3_auth_enabled = true + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" + + cache_enabled = true + request_coalescing_enabled = true + block_post_requests = true + + cache_vary_headers = ["User-Agent"] + + # Staging carries only synthetic verification traffic, so the production + # ceilings would be pure headroom. + limit_requests = 500 + limit_connections = 100 + + safehop_enabled = true + add_canonical_header = true + cache_stale = ["offline", "updating"] + block_ips = var.cdn_block_ips + + # 1 TB + limit_bandwidth = 1000000000000 + + block_root_path = true +} + + # Alliance SwissPass historically used a separate, manually configured pull # zone. Adopt it so both public repositories use the same B2 origin, middleware # release and cache behavior. diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 3020b2d..4bfaa28 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -17,6 +17,8 @@ const UNION_SLOTS = 'amd64/alpine324'; const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R43_MUSL = 'R (4.3.3 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R47_MUSL = 'R (4.7.0 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; const UA_R45_RESOLUTE = 'R/4.5.3 (Ubuntu 26.04) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; const UA_R45_FUTURE_UBUNTU = @@ -96,6 +98,21 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`); }); + // No per-minor index is published for 4.3, and contribPath() cannot probe + // the origin. Routing it would send the client to a 404 and it would see no + // packages at all, so an unpublished minor must fall through to flat. + await t.step('falls back to flat for an R minor that is not published', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R43_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('falls back to flat for a future R minor', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R47_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + await t.step('routes PACKAGES and PACKAGES.rds too', async () => { for (const file of ['PACKAGES', 'PACKAGES.rds']) { const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL); @@ -146,18 +163,12 @@ Deno.test('rpkgs-router', async (t) => { await t.step('resolves Ubuntu 26.04 to the resolute slot', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_RESOLUTE); - assertEquals( - res.location, - 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz', - ); + assertEquals(res.location, 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz'); }); await t.step('resolves a future Ubuntu release from its codename', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_FUTURE_UBUNTU); - assertEquals( - res.location, - 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz', - ); + assertEquals(res.location, 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz'); }); await t.step('sends an unidentifiable distro to CRAN', async () => { diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index cdeac44..e9aec22 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -27,7 +27,17 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; const CRAN_ORIGIN = 'https://cran.r-project.org'; -const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']); +const PUBLIC_CDN_HOSTS = new Set([ + 'cran.rpkgs.com', + 'cran.allianceswisspass.devxy.io', + // Staging hostnames, so the identical script can run on a test pull zone and + // redirect within itself. Without this a test zone rewrites to + // PUBLIC_CDN_ORIGIN, quietly exercising production instead of itself. + ...(Deno.env.get('EXTRA_PUBLIC_HOSTS') ?? '') + .split(',') + .map((host) => host.trim()) + .filter((host) => host.length > 0), +]); /** Slots ("/", comma separated) whose per-minor index is a union. */ const UNION_SLOTS = new Set( @@ -37,6 +47,21 @@ const UNION_SLOTS = new Set( .filter((slot) => slot.length > 0), ); +/** + * R minors for which a per-minor index is actually published. + * + * contribPath() has no way to probe the origin, so a minor that is not + * published here must fall back to the flat index. Routing an unlisted minor + * would send that client to a 404 and it would see no packages at all - a + * silent, total failure rather than a degraded one. + */ +const KNOWN_MINORS = new Set( + (Deno.env.get('KNOWN_MINORS') ?? '4.4,4.5,4.6') + .split(',') + .map((minor) => minor.trim()) + .filter((minor) => minor.length > 0), +); + /** `///latest/src/contrib[/]` */ const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/; @@ -177,8 +202,8 @@ function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: * The contrib path a request should be served from, relative to the slot. * * Returns the per-minor path for an index file when the slot is known to carry - * a union index and the client's R minor is known; otherwise the flat path, - * which is what every client sees today. + * a union index and the client's R minor is one we publish; otherwise the flat + * path, which is what every client sees today. */ function contribPath(slot: string, rest: string, userAgent: string): string { const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`; @@ -188,7 +213,7 @@ function contribPath(slot: string, rest: string, userAgent: string): string { } const rMinor = extractRMinor(userAgent); - return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; + return rMinor && KNOWN_MINORS.has(rMinor) ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; } BunnySDK.net.http diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh new file mode 100755 index 0000000..0096b0e --- /dev/null +++ b/scripts/verify-r-minor-routing.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +# +# Verify per-R-minor index routing for cran.rpkgs.com across every published +# / slot. +# +# The edge router (edge/rpkgs-router.ts) rewrites PACKAGES* requests to +# `contrib//` when the slot is listed in UNION_SLOTS and the client's +# User-Agent carries an R minor. Two properties have to hold before a slot may +# be added to UNION_SLOTS: +# +# 1. the per-minor index is a UNION of the per-minor and flat slots, so +# routing to it hides nothing the flat index carries; and +# 2. every R minor a client might report resolves to an index that exists, +# because contribPath() does not check existence and has no fallback. +# +# Modes: +# (default) Resolve routing decisions without depending on UNION_SLOTS being +# set. Safe to run before enabling: it reads the per-minor indexes +# directly and reproduces the router's target path. +# --live Additionally drive the real CDN with R User-Agents and assert the +# bytes served match the expected index. Only meaningful once the +# slot is in UNION_SLOTS. +# +# Usage: +# scripts/verify-r-minor-routing.sh +# scripts/verify-r-minor-routing.sh --live +# MINORS="4.4 4.5" SAMPLE=10 scripts/verify-r-minor-routing.sh +# +# Exits non-zero if any check fails. + +set -uo pipefail + +BASE=${BASE:-https://cran.rpkgs.com} +ARCHES=${ARCHES:-"amd64 arm64"} +DISTROS=${DISTROS:-"resolute noble jammy rhel8 rhel9 rhel10 alpine323 alpine324"} +# Minors a client may plausibly report. 4.3 is listed because the published +# support notes still claim it. +MINORS=${MINORS:-"4.3 4.4 4.5 4.6"} +# How many Path: targets to HEAD-check per slot/minor. 0 disables. +SAMPLE=${SAMPLE:-5} +# Largest package-count shortfall a non-primary minor may have against the best +# minor on the same slot before coverage counts as uneven. A slot built under +# one R minor carries fewer per-minor binaries for the others; until that gap +# closes, "full coverage for ABI-sensitive packages" is not a claim we can make. +PARITY_TOLERANCE=${PARITY_TOLERANCE:-25} +LIVE=0 + +for arg in "$@"; do + case "$arg" in + --live) LIVE=1 ;; + -h | --help) + sed -n '2,32p' "$0" + exit 0 + ;; + *) + echo "unknown argument: $arg" >&2 + exit 2 + ;; + esac +done + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +PASS=0 +FAIL=0 +FAILURES="" + +ok() { + PASS=$((PASS + 1)) + printf ' ok %s\n' "$1" +} + +bad() { + FAIL=$((FAIL + 1)) + FAILURES="${FAILURES}\n - $1" + printf ' FAIL %s\n' "$1" +} + +# Fetch a URL into a file, echoing the HTTP status. Cached per URL. +fetch() { + local url=$1 dest=$2 ua=${3:-} + if [ -s "$dest" ]; then + cat "$dest.status" + return 0 + fi + local status + if [ -n "$ua" ]; then + status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + else + status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + fi + echo "$status" > "$dest.status" + echo "$status" +} + +head_status() { + curl -sS -o /dev/null -w '%{http_code}' -I --max-time 60 "$1" 2>/dev/null +} + +# Package names from a gzipped PACKAGES index, sorted. +pkg_names() { + gunzip -c "$1" 2>/dev/null | awk '/^Package:/ {print $2}' | sort -u +} + +# " " pairs for entries that carry a Path: field. +path_entries() { + gunzip -c "$1" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; ver = ""; path = "" } + /^Version:/ { ver = $2 } + /^Path:/ { path = $2 } + /^$/ { if (pkg != "" && path != "") print pkg, ver, path; pkg = "" } + END { if (pkg != "" && path != "") print pkg, ver, path } + ' +} + +# An R User-Agent of the shape R actually sends. +r_user_agent() { + printf 'R/%s.0 (Ubuntu 24.04; codename=noble) (x86_64-pc-linux-gnu x86_64 linux-gnu)' "$1" +} + +echo "verify-r-minor-routing: $BASE" +echo " slots: $(echo "$ARCHES" | wc -w) arch x $(echo "$DISTROS" | wc -w) os" +echo " minors: $MINORS" +echo " live: $LIVE" +echo + +for arch in $ARCHES; do + for distro in $DISTROS; do + slot="$arch/$distro" + echo "$slot" + + flat_url="$BASE/$slot/latest/src/contrib/PACKAGES.gz" + flat_file="$WORK/${arch}-${distro}-flat.gz" + flat_status=$(fetch "$flat_url" "$flat_file") + + if [ "$flat_status" != "200" ]; then + bad "$slot flat index unreachable (HTTP $flat_status)" + continue + fi + + pkg_names "$flat_file" > "$flat_file.names" + flat_count=$(wc -l < "$flat_file.names") + if [ "$flat_count" -lt 1000 ]; then + bad "$slot flat index has only $flat_count packages" + continue + fi + ok "$slot flat index: $flat_count packages" + + for minor in $MINORS; do + minor_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" + minor_file="$WORK/${arch}-${distro}-${minor}.gz" + minor_status=$(fetch "$minor_url" "$minor_file") + + # A minor the router would route to must exist, or clients on that R + # version get a 404 and see no packages at all. + if [ "$minor_status" != "200" ]; then + bad "$slot R $minor index missing (HTTP $minor_status) - routing would 404 for R $minor clients" + continue + fi + + pkg_names "$minor_file" > "$minor_file.names" + minor_count=$(wc -l < "$minor_file.names") + + # Union property: nothing the flat index carries may be missing here. + missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5) + missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l) + if [ "$missing_count" -ne 0 ]; then + bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))" + else + ok "$slot R $minor index: $minor_count packages, union holds" + fi + + # Path: entries steer to per-minor binaries; they must resolve. + if [ "$SAMPLE" -gt 0 ]; then + path_entries "$minor_file" > "$minor_file.paths" + total_paths=$(wc -l < "$minor_file.paths") + broken=0 + checked=0 + while read -r pkg ver path; do + [ -z "${pkg:-}" ] && continue + tarball="$BASE/$slot/latest/src/contrib/$path/${pkg}_${ver}.tar.gz" + status=$(head_status "$tarball") + checked=$((checked + 1)) + if [ "$status" != "200" ]; then + broken=$((broken + 1)) + [ "$broken" -le 2 ] && printf ' broken target: %s (HTTP %s)\n' "$tarball" "$status" + fi + done < <(shuf -n "$SAMPLE" "$minor_file.paths" 2>/dev/null || head -n "$SAMPLE" "$minor_file.paths") + + if [ "$broken" -ne 0 ]; then + bad "$slot R $minor: $broken/$checked sampled Path: targets do not resolve (of $total_paths total)" + elif [ "$checked" -gt 0 ]; then + ok "$slot R $minor: $checked/$checked sampled Path: targets resolve (of $total_paths total)" + fi + fi + + # Live routing: what a real R client on this minor actually receives. + if [ "$LIVE" -eq 1 ]; then + ua=$(r_user_agent "$minor") + live_file="$WORK/${arch}-${distro}-${minor}-live.gz" + live_status=$(fetch "$flat_url" "$live_file" "$ua") + if [ "$live_status" != "200" ]; then + bad "$slot R $minor live request failed (HTTP $live_status)" + elif cmp -s "$live_file" "$minor_file"; then + ok "$slot R $minor live request served the per-minor index" + elif cmp -s "$live_file" "$flat_file"; then + bad "$slot R $minor live request served the FLAT index - slot not in UNION_SLOTS?" + else + bad "$slot R $minor live request served neither the per-minor nor the flat index" + fi + fi + done + + # Coverage parity across minors. The union property only guarantees no + # client loses packages relative to the flat index; it says nothing about a + # 4.4 client seeing fewer packages than a 4.5 client on the same slot. + best=0 + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz.names" + [ -s "$f" ] || continue + c=$(wc -l < "$f") + [ "$c" -gt "$best" ] && best=$c + done + if [ "$best" -gt 0 ]; then + uneven="" + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz.names" + [ -s "$f" ] || continue + c=$(wc -l < "$f") + gap=$((best - c)) + [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap" + done + if [ -n "$uneven" ]; then + bad "$slot coverage uneven across minors (vs best $best):$uneven" + else + ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)" + fi + fi + + # A client whose User-Agent carries no R version must keep getting the flat + # index, never a per-minor one. + if [ "$LIVE" -eq 1 ]; then + plain_file="$WORK/${arch}-${distro}-plain.gz" + plain_status=$(fetch "$flat_url" "$plain_file" "curl/8.0.0") + if [ "$plain_status" != "200" ]; then + bad "$slot non-R User-Agent request failed (HTTP $plain_status)" + elif cmp -s "$plain_file" "$flat_file"; then + ok "$slot non-R User-Agent still served the flat index" + else + bad "$slot non-R User-Agent was routed away from the flat index" + fi + + # Tarball requests must never be rewritten into a per-minor directory: + # flat-slot packages do not live there. + sample_pkg=$(gunzip -c "$flat_file" | awk '/^Package:/ {p=$2} /^Version:/ {print p, $2; exit}') + if [ -n "$sample_pkg" ]; then + # shellcheck disable=SC2086 # deliberate split into $1 (package) and $2 (version) + set -- $sample_pkg + tb="$BASE/$slot/latest/src/contrib/${1}_${2}.tar.gz" + tb_status=$(curl -sS -o /dev/null -w '%{http_code}' -A "$(r_user_agent 4.5)" --max-time 60 "$tb" 2>/dev/null) + if [ "$tb_status" = "200" ]; then + ok "$slot tarball request under an R User-Agent still resolves" + else + bad "$slot tarball ${1}_${2}.tar.gz broke under an R User-Agent (HTTP $tb_status)" + fi + fi + fi + done +done + +echo +echo "passed: $PASS failed: $FAIL" +if [ "$FAIL" -ne 0 ]; then + printf 'failures:%b\n' "$FAILURES" + exit 1 +fi -- 2.54.0 From 771d3a776864ce46ce36ef1748e5cf62eae5c3dd Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 30 Aug 2026 15:44:14 +0000 Subject: [PATCH 2/2] feat(edge): send unsupported R minors to CRAN instead of serving them Falling back to the flat index for an excluded minor is the silent case: risky packages there are built under another minor and fail at load time, far from the cause. Send those clients to CRAN for sources instead, which is what the router already does for an unidentifiable distro. The whole interaction has to move, not just the index. R resolves tarball URLs against the repo it was configured with, so serving the index from CRAN and tarballs from here would hand R a binary where it expects a source tarball. A client reporting no R minor at all is not excluded: mirror scripts and image builds keep getting the flat slot. - Declare the supported window once in cdn.tf as local.rpkgs_supported_minors and set KNOWN_MINORS from it on both zones, so the router cannot drift from build-env-images' LATEST/PREV1/PREV2 unnoticed. - Split the verification script into supported and excluded minors: the former must have published indexes, the latter must have none and must redirect to CRAN under --live. --- cdn.tf | 46 +++++++++++++++++++++++-------- edge/rpkgs-router.test.ts | 26 ++++++++++++----- edge/rpkgs-router.ts | 26 +++++++++++++++++ scripts/verify-r-minor-routing.sh | 40 ++++++++++++++++++++++++--- 4 files changed, 115 insertions(+), 23 deletions(-) diff --git a/cdn.tf b/cdn.tf index ffe9b21..349486e 100644 --- a/cdn.tf +++ b/cdn.tf @@ -52,6 +52,26 @@ ### cran.rpkgs.com +locals { + rpkgs_slots = [ + for pair in setproduct( + ["amd64", "arm64"], + ["resolute", "noble", "jammy", "rhel8", "rhel9", "rhel10", "alpine323", "alpine324"] + ) : "${pair[0]}/${pair[1]}" + ] + + # The supported R minors: the current one plus the two previous, which is + # exactly what build-env-images installs as R_VERSION_LATEST / PREV1 / PREV2. + # These must stay in step. A minor listed here without a published index + # sends those clients to a 404; a published minor missing from this list + # sends them to CRAN for sources instead of serving the binaries we built. + rpkgs_supported_minors = ["4.4", "4.5", "4.6"] + + # bunny.net serves every pull zone on .b-cdn.net, so staging needs no + # DNS record and is never advertised. + rpkgs_test_hostname = "cran-rpkgs-test.b-cdn.net" +} + # The edge middleware that resolves the bare cran.rpkgs.com form to an # / slot and routes PACKAGES* to the per-R-minor slot. The source of # truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release. @@ -75,6 +95,13 @@ resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { required = false } +resource "bunnynet_compute_script_variable" "rpkgs_router_known_minors" { + script = bunnynet_compute_script.rpkgs_router.id + name = "KNOWN_MINORS" + default_value = join(",", local.rpkgs_supported_minors) + required = false +} + resource "bunnynet_pullzone" "cran_rpkgs_com" { name = "cran-rpkgs" @@ -152,18 +179,6 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { # Every published / slot. The staging zone enables per-minor routing # for all of them at once; production adopts the same list only after # `scripts/verify-r-minor-routing.sh --live` passes against staging. -locals { - rpkgs_slots = [ - for pair in setproduct( - ["amd64", "arm64"], - ["resolute", "noble", "jammy", "rhel8", "rhel9", "rhel10", "alpine323", "alpine324"] - ) : "${pair[0]}/${pair[1]}" - ] - - # bunny.net serves every pull zone on .b-cdn.net, so staging needs no - # DNS record and is never advertised. - rpkgs_test_hostname = "cran-rpkgs-test.b-cdn.net" -} # A second copy of the same router, bound to the same B2 origin, so UNION_SLOTS # can be exercised end to end before production is touched. @@ -189,6 +204,13 @@ resource "bunnynet_compute_script_variable" "rpkgs_router_test_extra_hosts" { required = false } +resource "bunnynet_compute_script_variable" "rpkgs_router_test_known_minors" { + script = bunnynet_compute_script.rpkgs_router_test.id + name = "KNOWN_MINORS" + default_value = join(",", local.rpkgs_supported_minors) + required = false +} + resource "bunnynet_pullzone" "cran_rpkgs_test" { name = "cran-rpkgs-test" diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 4bfaa28..3e2fe8b 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -98,17 +98,29 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`); }); - // No per-minor index is published for 4.3, and contribPath() cannot probe - // the origin. Routing it would send the client to a 404 and it would see no - // packages at all, so an unpublished minor must fall through to flat. - await t.step('falls back to flat for an R minor that is not published', async () => { + // We publish binaries only for the supported window. An excluded minor has + // no slot we can serve safely, so it goes to CRAN for sources rather than + // to a 404 or to binaries built under another minor. + await t.step('sends an excluded R minor to CRAN for the index', async () => { const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R43_MUSL); - assertEquals(res.location, null); - assertEquals(res.status, 200); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); }); - await t.step('falls back to flat for a future R minor', async () => { + await t.step('sends a future R minor to CRAN too', async () => { const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R47_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); + }); + + // The index and the tarballs R resolves against it have to come from the + // same place. Serving one from CRAN and the other from here would hand R a + // binary where it expects a source tarball. + await t.step('sends an excluded minor to CRAN for tarballs as well', async () => { + const res = await probe(`${SLOT}/foo_1.0.tar.gz`, UA_R43_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/foo_1.0.tar.gz'); + }); + + await t.step('leaves an excluded minor alone on a slot outside UNION_SLOTS', async () => { + const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R43_MUSL); assertEquals(res.location, null); assertEquals(res.status, 200); }); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index e9aec22..05b6e18 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -121,6 +121,18 @@ function publicCdnOrigin(url: URL): string { return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; } +/** + * True when the client reports an R minor that we deliberately do not serve. + * + * A client that reports no minor at all is not "unsupported": non-R fetchers + * (mirror scripts, image builds) must keep getting the flat slot. Only a + * known-and-excluded minor falls through to CRAN. + */ +function isExcludedMinor(userAgent: string): boolean { + const rMinor = extractRMinor(userAgent); + return rMinor !== null && !KNOWN_MINORS.has(rMinor); +} + function extractRMinor(userAgent: string): string | null { for (const regex of R_MINOR_REGEXES) { const match = userAgent.match(regex); @@ -249,6 +261,16 @@ BunnySDK.net.http return Promise.resolve(ctx.request); } + // An R minor outside the supported window has no binaries we can safely + // serve, so the whole interaction goes to CRAN: the index and the + // tarballs R will resolve against it. Serving the index from CRAN but + // tarballs from here would hand R a binary where it expects a source + // tarball, which fails in a far more confusing way than not being + // served at all. + if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}/src/contrib/${rest}`)); + } + const target = contribPath(slot, rest, userAgent); if (target === path) { return Promise.resolve(ctx.request); @@ -263,6 +285,10 @@ BunnySDK.net.http return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); } + if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + const rest = srcContrib ? srcContrib[1] : ''; return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); } diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh index 0096b0e..28acd92 100755 --- a/scripts/verify-r-minor-routing.sh +++ b/scripts/verify-r-minor-routing.sh @@ -33,9 +33,14 @@ set -uo pipefail BASE=${BASE:-https://cran.rpkgs.com} ARCHES=${ARCHES:-"amd64 arm64"} DISTROS=${DISTROS:-"resolute noble jammy rhel8 rhel9 rhel10 alpine323 alpine324"} -# Minors a client may plausibly report. 4.3 is listed because the published -# support notes still claim it. -MINORS=${MINORS:-"4.3 4.4 4.5 4.6"} +# The supported window: the current R minor plus the two previous, matching +# build-env-images' R_VERSION_LATEST/PREV1/PREV2 and cdn.tf's +# local.rpkgs_supported_minors. Each of these must have a published index. +MINORS=${MINORS:-"4.4 4.5 4.6"} +# Minors we deliberately do not serve. These must have NO published index and, +# once routing is live, must be sent to CRAN for sources rather than 404ing or +# being handed binaries built under another minor. +EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"} # How many Path: targets to HEAD-check per slot/minor. 0 disables. SAMPLE=${SAMPLE:-5} # Largest package-count shortfall a non-primary minor may have against the best @@ -121,7 +126,7 @@ r_user_agent() { echo "verify-r-minor-routing: $BASE" echo " slots: $(echo "$ARCHES" | wc -w) arch x $(echo "$DISTROS" | wc -w) os" -echo " minors: $MINORS" +echo " minors: $MINORS (excluded: $EXCLUDED_MINORS)" echo " live: $LIVE" echo @@ -238,6 +243,33 @@ for arch in $ARCHES; do fi fi + # Excluded minors: no published index, and under --live a redirect to CRAN. + for minor in $EXCLUDED_MINORS; do + ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" + ex_status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 60 "$ex_url" 2>/dev/null) + if [ "$ex_status" = "200" ]; then + bad "$slot R $minor is excluded but an index is published - the two lists disagree" + else + ok "$slot R $minor correctly has no published index" + fi + + if [ "$LIVE" -eq 1 ]; then + loc=$(curl -sS -o /dev/null -w '%{redirect_url}' -A "$(r_user_agent "$minor")" \ + --max-time 60 "$flat_url" 2>/dev/null) + case "$loc" in + https://cran.r-project.org/*) + ok "$slot R $minor is sent to CRAN ($loc)" + ;; + "") + bad "$slot R $minor was served directly instead of being sent to CRAN" + ;; + *) + bad "$slot R $minor redirected somewhere unexpected: $loc" + ;; + esac + fi + done + # A client whose User-Agent carries no R version must keep getting the flat # index, never a per-minor one. if [ "$LIVE" -eq 1 ]; then -- 2.54.0