feat(edge): send unsupported R minors to CRAN instead of serving them
Falling back to the flat index for an excluded minor is the silent case: risky packages there are built under another minor and fail at load time, far from the cause. Send those clients to CRAN for sources instead, which is what the router already does for an unidentifiable distro. The whole interaction has to move, not just the index. R resolves tarball URLs against the repo it was configured with, so serving the index from CRAN and tarballs from here would hand R a binary where it expects a source tarball. A client reporting no R minor at all is not excluded: mirror scripts and image builds keep getting the flat slot. - Declare the supported window once in cdn.tf as local.rpkgs_supported_minors and set KNOWN_MINORS from it on both zones, so the router cannot drift from build-env-images' LATEST/PREV1/PREV2 unnoticed. - Split the verification script into supported and excluded minors: the former must have published indexes, the latter must have none and must redirect to CRAN under --live.
This commit is contained in:
parent
4c1e9b773c
commit
771d3a7768
4 changed files with 115 additions and 23 deletions
46
cdn.tf
46
cdn.tf
|
|
@ -52,6 +52,26 @@
|
||||||
|
|
||||||
### cran.rpkgs.com
|
### cran.rpkgs.com
|
||||||
|
|
||||||
|
locals {
|
||||||
|
rpkgs_slots = [
|
||||||
|
for pair in setproduct(
|
||||||
|
["amd64", "arm64"],
|
||||||
|
["resolute", "noble", "jammy", "rhel8", "rhel9", "rhel10", "alpine323", "alpine324"]
|
||||||
|
) : "${pair[0]}/${pair[1]}"
|
||||||
|
]
|
||||||
|
|
||||||
|
# The supported R minors: the current one plus the two previous, which is
|
||||||
|
# exactly what build-env-images installs as R_VERSION_LATEST / PREV1 / PREV2.
|
||||||
|
# These must stay in step. A minor listed here without a published index
|
||||||
|
# sends those clients to a 404; a published minor missing from this list
|
||||||
|
# sends them to CRAN for sources instead of serving the binaries we built.
|
||||||
|
rpkgs_supported_minors = ["4.4", "4.5", "4.6"]
|
||||||
|
|
||||||
|
# bunny.net serves every pull zone on <name>.b-cdn.net, so staging needs no
|
||||||
|
# DNS record and is never advertised.
|
||||||
|
rpkgs_test_hostname = "cran-rpkgs-test.b-cdn.net"
|
||||||
|
}
|
||||||
|
|
||||||
# The edge middleware that resolves the bare cran.rpkgs.com form to an
|
# The edge middleware that resolves the bare cran.rpkgs.com form to an
|
||||||
# <arch>/<os> slot and routes PACKAGES* to the per-R-minor slot. The source of
|
# <arch>/<os> slot and routes PACKAGES* to the per-R-minor slot. The source of
|
||||||
# truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release.
|
# truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release.
|
||||||
|
|
@ -75,6 +95,13 @@ resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" {
|
||||||
required = false
|
required = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resource "bunnynet_compute_script_variable" "rpkgs_router_known_minors" {
|
||||||
|
script = bunnynet_compute_script.rpkgs_router.id
|
||||||
|
name = "KNOWN_MINORS"
|
||||||
|
default_value = join(",", local.rpkgs_supported_minors)
|
||||||
|
required = false
|
||||||
|
}
|
||||||
|
|
||||||
resource "bunnynet_pullzone" "cran_rpkgs_com" {
|
resource "bunnynet_pullzone" "cran_rpkgs_com" {
|
||||||
name = "cran-rpkgs"
|
name = "cran-rpkgs"
|
||||||
|
|
||||||
|
|
@ -152,18 +179,6 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" {
|
||||||
# Every published <arch>/<os> slot. The staging zone enables per-minor routing
|
# Every published <arch>/<os> slot. The staging zone enables per-minor routing
|
||||||
# for all of them at once; production adopts the same list only after
|
# for all of them at once; production adopts the same list only after
|
||||||
# `scripts/verify-r-minor-routing.sh --live` passes against staging.
|
# `scripts/verify-r-minor-routing.sh --live` passes against staging.
|
||||||
locals {
|
|
||||||
rpkgs_slots = [
|
|
||||||
for pair in setproduct(
|
|
||||||
["amd64", "arm64"],
|
|
||||||
["resolute", "noble", "jammy", "rhel8", "rhel9", "rhel10", "alpine323", "alpine324"]
|
|
||||||
) : "${pair[0]}/${pair[1]}"
|
|
||||||
]
|
|
||||||
|
|
||||||
# bunny.net serves every pull zone on <name>.b-cdn.net, so staging needs no
|
|
||||||
# DNS record and is never advertised.
|
|
||||||
rpkgs_test_hostname = "cran-rpkgs-test.b-cdn.net"
|
|
||||||
}
|
|
||||||
|
|
||||||
# A second copy of the same router, bound to the same B2 origin, so UNION_SLOTS
|
# A second copy of the same router, bound to the same B2 origin, so UNION_SLOTS
|
||||||
# can be exercised end to end before production is touched.
|
# can be exercised end to end before production is touched.
|
||||||
|
|
@ -189,6 +204,13 @@ resource "bunnynet_compute_script_variable" "rpkgs_router_test_extra_hosts" {
|
||||||
required = false
|
required = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resource "bunnynet_compute_script_variable" "rpkgs_router_test_known_minors" {
|
||||||
|
script = bunnynet_compute_script.rpkgs_router_test.id
|
||||||
|
name = "KNOWN_MINORS"
|
||||||
|
default_value = join(",", local.rpkgs_supported_minors)
|
||||||
|
required = false
|
||||||
|
}
|
||||||
|
|
||||||
resource "bunnynet_pullzone" "cran_rpkgs_test" {
|
resource "bunnynet_pullzone" "cran_rpkgs_test" {
|
||||||
name = "cran-rpkgs-test"
|
name = "cran-rpkgs-test"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -98,17 +98,29 @@ Deno.test('rpkgs-router', async (t) => {
|
||||||
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`);
|
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`);
|
||||||
});
|
});
|
||||||
|
|
||||||
// No per-minor index is published for 4.3, and contribPath() cannot probe
|
// We publish binaries only for the supported window. An excluded minor has
|
||||||
// the origin. Routing it would send the client to a 404 and it would see no
|
// no slot we can serve safely, so it goes to CRAN for sources rather than
|
||||||
// packages at all, so an unpublished minor must fall through to flat.
|
// to a 404 or to binaries built under another minor.
|
||||||
await t.step('falls back to flat for an R minor that is not published', async () => {
|
await t.step('sends an excluded R minor to CRAN for the index', async () => {
|
||||||
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R43_MUSL);
|
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R43_MUSL);
|
||||||
assertEquals(res.location, null);
|
assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz');
|
||||||
assertEquals(res.status, 200);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
await t.step('falls back to flat for a future R minor', async () => {
|
await t.step('sends a future R minor to CRAN too', async () => {
|
||||||
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R47_MUSL);
|
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R47_MUSL);
|
||||||
|
assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz');
|
||||||
|
});
|
||||||
|
|
||||||
|
// The index and the tarballs R resolves against it have to come from the
|
||||||
|
// same place. Serving one from CRAN and the other from here would hand R a
|
||||||
|
// binary where it expects a source tarball.
|
||||||
|
await t.step('sends an excluded minor to CRAN for tarballs as well', async () => {
|
||||||
|
const res = await probe(`${SLOT}/foo_1.0.tar.gz`, UA_R43_MUSL);
|
||||||
|
assertEquals(res.location, 'https://cran.r-project.org/src/contrib/foo_1.0.tar.gz');
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('leaves an excluded minor alone on a slot outside UNION_SLOTS', async () => {
|
||||||
|
const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R43_MUSL);
|
||||||
assertEquals(res.location, null);
|
assertEquals(res.location, null);
|
||||||
assertEquals(res.status, 200);
|
assertEquals(res.status, 200);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -121,6 +121,18 @@ function publicCdnOrigin(url: URL): string {
|
||||||
return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN;
|
return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* True when the client reports an R minor that we deliberately do not serve.
|
||||||
|
*
|
||||||
|
* A client that reports no minor at all is not "unsupported": non-R fetchers
|
||||||
|
* (mirror scripts, image builds) must keep getting the flat slot. Only a
|
||||||
|
* known-and-excluded minor falls through to CRAN.
|
||||||
|
*/
|
||||||
|
function isExcludedMinor(userAgent: string): boolean {
|
||||||
|
const rMinor = extractRMinor(userAgent);
|
||||||
|
return rMinor !== null && !KNOWN_MINORS.has(rMinor);
|
||||||
|
}
|
||||||
|
|
||||||
function extractRMinor(userAgent: string): string | null {
|
function extractRMinor(userAgent: string): string | null {
|
||||||
for (const regex of R_MINOR_REGEXES) {
|
for (const regex of R_MINOR_REGEXES) {
|
||||||
const match = userAgent.match(regex);
|
const match = userAgent.match(regex);
|
||||||
|
|
@ -249,6 +261,16 @@ BunnySDK.net.http
|
||||||
return Promise.resolve(ctx.request);
|
return Promise.resolve(ctx.request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An R minor outside the supported window has no binaries we can safely
|
||||||
|
// serve, so the whole interaction goes to CRAN: the index and the
|
||||||
|
// tarballs R will resolve against it. Serving the index from CRAN but
|
||||||
|
// tarballs from here would hand R a binary where it expects a source
|
||||||
|
// tarball, which fails in a far more confusing way than not being
|
||||||
|
// served at all.
|
||||||
|
if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) {
|
||||||
|
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}/src/contrib/${rest}`));
|
||||||
|
}
|
||||||
|
|
||||||
const target = contribPath(slot, rest, userAgent);
|
const target = contribPath(slot, rest, userAgent);
|
||||||
if (target === path) {
|
if (target === path) {
|
||||||
return Promise.resolve(ctx.request);
|
return Promise.resolve(ctx.request);
|
||||||
|
|
@ -263,6 +285,10 @@ BunnySDK.net.http
|
||||||
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`));
|
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) {
|
||||||
|
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`));
|
||||||
|
}
|
||||||
|
|
||||||
const rest = srcContrib ? srcContrib[1] : '';
|
const rest = srcContrib ? srcContrib[1] : '';
|
||||||
return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`));
|
return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -33,9 +33,14 @@ set -uo pipefail
|
||||||
BASE=${BASE:-https://cran.rpkgs.com}
|
BASE=${BASE:-https://cran.rpkgs.com}
|
||||||
ARCHES=${ARCHES:-"amd64 arm64"}
|
ARCHES=${ARCHES:-"amd64 arm64"}
|
||||||
DISTROS=${DISTROS:-"resolute noble jammy rhel8 rhel9 rhel10 alpine323 alpine324"}
|
DISTROS=${DISTROS:-"resolute noble jammy rhel8 rhel9 rhel10 alpine323 alpine324"}
|
||||||
# Minors a client may plausibly report. 4.3 is listed because the published
|
# The supported window: the current R minor plus the two previous, matching
|
||||||
# support notes still claim it.
|
# build-env-images' R_VERSION_LATEST/PREV1/PREV2 and cdn.tf's
|
||||||
MINORS=${MINORS:-"4.3 4.4 4.5 4.6"}
|
# local.rpkgs_supported_minors. Each of these must have a published index.
|
||||||
|
MINORS=${MINORS:-"4.4 4.5 4.6"}
|
||||||
|
# Minors we deliberately do not serve. These must have NO published index and,
|
||||||
|
# once routing is live, must be sent to CRAN for sources rather than 404ing or
|
||||||
|
# being handed binaries built under another minor.
|
||||||
|
EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"}
|
||||||
# How many Path: targets to HEAD-check per slot/minor. 0 disables.
|
# How many Path: targets to HEAD-check per slot/minor. 0 disables.
|
||||||
SAMPLE=${SAMPLE:-5}
|
SAMPLE=${SAMPLE:-5}
|
||||||
# Largest package-count shortfall a non-primary minor may have against the best
|
# Largest package-count shortfall a non-primary minor may have against the best
|
||||||
|
|
@ -121,7 +126,7 @@ r_user_agent() {
|
||||||
|
|
||||||
echo "verify-r-minor-routing: $BASE"
|
echo "verify-r-minor-routing: $BASE"
|
||||||
echo " slots: $(echo "$ARCHES" | wc -w) arch x $(echo "$DISTROS" | wc -w) os"
|
echo " slots: $(echo "$ARCHES" | wc -w) arch x $(echo "$DISTROS" | wc -w) os"
|
||||||
echo " minors: $MINORS"
|
echo " minors: $MINORS (excluded: $EXCLUDED_MINORS)"
|
||||||
echo " live: $LIVE"
|
echo " live: $LIVE"
|
||||||
echo
|
echo
|
||||||
|
|
||||||
|
|
@ -238,6 +243,33 @@ for arch in $ARCHES; do
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Excluded minors: no published index, and under --live a redirect to CRAN.
|
||||||
|
for minor in $EXCLUDED_MINORS; do
|
||||||
|
ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz"
|
||||||
|
ex_status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 60 "$ex_url" 2>/dev/null)
|
||||||
|
if [ "$ex_status" = "200" ]; then
|
||||||
|
bad "$slot R $minor is excluded but an index is published - the two lists disagree"
|
||||||
|
else
|
||||||
|
ok "$slot R $minor correctly has no published index"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$LIVE" -eq 1 ]; then
|
||||||
|
loc=$(curl -sS -o /dev/null -w '%{redirect_url}' -A "$(r_user_agent "$minor")" \
|
||||||
|
--max-time 60 "$flat_url" 2>/dev/null)
|
||||||
|
case "$loc" in
|
||||||
|
https://cran.r-project.org/*)
|
||||||
|
ok "$slot R $minor is sent to CRAN ($loc)"
|
||||||
|
;;
|
||||||
|
"")
|
||||||
|
bad "$slot R $minor was served directly instead of being sent to CRAN"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
bad "$slot R $minor redirected somewhere unexpected: $loc"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
# A client whose User-Agent carries no R version must keep getting the flat
|
# A client whose User-Agent carries no R version must keep getting the flat
|
||||||
# index, never a per-minor one.
|
# index, never a per-minor one.
|
||||||
if [ "$LIVE" -eq 1 ]; then
|
if [ "$LIVE" -eq 1 ]; then
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue