feat(edge): route PACKAGES requests to the per-R-minor slot #152
5 changed files with 440 additions and 22 deletions
feat(edge): route PACKAGES requests to the per-R-minor slot
- add edge/rpkgs-router.ts, which redirects PACKAGES, PACKAGES.gz and PACKAGES.rds into …/src/contrib/<x.y>/ for slots listed in UNION_SLOTS - leave tarballs alone: R keeps the pre-redirect contrib URL, so the union index steers per-minor tarballs with a Path field instead - stop resolving an unidentifiable distro to a phantom linux-gnu/linux-musl slot and send those clients to CRAN - mark every redirect no-store, since the target depends on the User-Agent - cover the routing matrix in edge/rpkgs-router.test.ts, run by just edge-test - correct the spec and plan: the union index carries Path: <x.y> on per-minor records, not Path: .. on flat ones
commit
628d19d650
160
edge/rpkgs-router.test.ts
Normal file
160
edge/rpkgs-router.test.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
||||||
|
/**
|
||||||
|
* Routing matrix for `edge/rpkgs-router.ts`.
|
||||||
|
*
|
||||||
|
* The script is exercised through the SDK's local server rather than by
|
||||||
|
* importing its internals, so what is tested is the artifact that gets
|
||||||
|
* deployed. Requests that the script passes through are proxied to the real
|
||||||
|
* origin, which keeps the "no redirect" cases honest: they assert that the
|
||||||
|
* client reached the flat slot, not merely that no `Location` was set.
|
||||||
|
*
|
||||||
|
* Run with `just edge-test`.
|
||||||
|
*/
|
||||||
|
import { assertEquals } from 'jsr:@std/assert@1';
|
||||||
|
|
||||||
|
const SCRIPT = new URL('./rpkgs-router.ts', import.meta.url).pathname;
|
||||||
|
const BASE = 'http://127.0.0.1:8080';
|
||||||
|
const UNION_SLOTS = 'amd64/alpine324';
|
||||||
|
|
||||||
|
const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)';
|
||||||
|
const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)';
|
||||||
|
const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24';
|
||||||
|
const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)';
|
||||||
|
const UA_CURL = 'curl/8.0.1';
|
||||||
|
|
||||||
|
const SLOT = '/amd64/alpine324/latest/src/contrib';
|
||||||
|
const OTHER_SLOT = '/amd64/noble/latest/src/contrib';
|
||||||
|
|
||||||
|
interface Probe {
|
||||||
|
status: number;
|
||||||
|
location: string | null;
|
||||||
|
cacheControl: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function probe(path: string, userAgent: string): Promise<Probe> {
|
||||||
|
const res = await fetch(BASE + path, {
|
||||||
|
headers: { 'User-Agent': userAgent },
|
||||||
|
redirect: 'manual',
|
||||||
|
});
|
||||||
|
await res.body?.cancel();
|
||||||
|
return {
|
||||||
|
status: res.status,
|
||||||
|
location: res.headers.get('location'),
|
||||||
|
cacheControl: res.headers.get('cache-control'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Kill tolerantly: the child has already exited if the script failed to load. */
|
||||||
|
async function stopServer(child: Deno.ChildProcess): Promise<void> {
|
||||||
|
try {
|
||||||
|
child.kill();
|
||||||
|
} catch {
|
||||||
|
// already gone
|
||||||
|
}
|
||||||
|
await child.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startServer(): Promise<Deno.ChildProcess> {
|
||||||
|
const child = new Deno.Command(Deno.execPath(), {
|
||||||
|
args: ['run', '-A', SCRIPT],
|
||||||
|
env: { UNION_SLOTS },
|
||||||
|
stdout: 'null',
|
||||||
|
stderr: 'inherit',
|
||||||
|
}).spawn();
|
||||||
|
|
||||||
|
for (let attempt = 0; attempt < 150; attempt++) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${BASE}/`, {
|
||||||
|
headers: { 'User-Agent': UA_CURL },
|
||||||
|
redirect: 'manual',
|
||||||
|
});
|
||||||
|
await res.body?.cancel();
|
||||||
|
return child;
|
||||||
|
} catch {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await stopServer(child);
|
||||||
|
throw new Error('edge script did not start listening on ' + BASE);
|
||||||
|
}
|
||||||
|
|
||||||
|
Deno.test('rpkgs-router', async (t) => {
|
||||||
|
const server = await startServer();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await t.step("routes an index request to the client's R minor", async () => {
|
||||||
|
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL);
|
||||||
|
assertEquals(res.status, 302);
|
||||||
|
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`);
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('routes R 4.6 to its own slot', async () => {
|
||||||
|
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R46_MUSL);
|
||||||
|
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`);
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('routes PACKAGES and PACKAGES.rds too', async () => {
|
||||||
|
for (const file of ['PACKAGES', 'PACKAGES.rds']) {
|
||||||
|
const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL);
|
||||||
|
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/${file}`, `expected ${file} to be routed`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('marks the redirect uncacheable', async () => {
|
||||||
|
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL);
|
||||||
|
assertEquals(res.cacheControl, 'no-store');
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('leaves a slot outside UNION_SLOTS alone', async () => {
|
||||||
|
const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R45_MUSL);
|
||||||
|
assertEquals(res.location, null);
|
||||||
|
assertEquals(res.status, 200);
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('never routes a tarball', async () => {
|
||||||
|
const res = await probe(`${SLOT}/jsonlite_2.0.0.tar.gz`, UA_R45_MUSL);
|
||||||
|
assertEquals(res.location, null);
|
||||||
|
assertEquals(res.status, 200);
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('does not redirect a path already under a minor', async () => {
|
||||||
|
const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL);
|
||||||
|
assertEquals(res.location, null);
|
||||||
|
assertEquals(res.status, 200);
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('leaves a client without an R version alone', async () => {
|
||||||
|
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_CURL);
|
||||||
|
assertEquals(res.location, null);
|
||||||
|
assertEquals(res.status, 200);
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('resolves the bare root to slot and minor', async () => {
|
||||||
|
const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_ALPINE);
|
||||||
|
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`);
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('sends an unidentifiable distro to CRAN', async () => {
|
||||||
|
const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL);
|
||||||
|
assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz');
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('keeps the macOS rewrite', async () => {
|
||||||
|
const res = await probe('/src/contrib/foo_1.0.tar.gz', UA_R45_DARWIN);
|
||||||
|
assertEquals(res.location, 'https://cran.rpkgs.com/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz');
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('keeps the macOS binary passthrough to CRAN', async () => {
|
||||||
|
const path = '/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz';
|
||||||
|
const res = await probe(path, UA_R45_DARWIN);
|
||||||
|
assertEquals(res.location, `https://cran.r-project.org${path}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.step('collapses duplicate slashes before matching', async () => {
|
||||||
|
const res = await probe(`/amd64/alpine324//latest/src/contrib//PACKAGES.gz`, UA_R45_MUSL);
|
||||||
|
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`);
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
await stopServer(server);
|
||||||
|
}
|
||||||
|
});
|
||||||
235
edge/rpkgs-router.ts
Normal file
235
edge/rpkgs-router.ts
Normal file
|
|
@ -0,0 +1,235 @@
|
||||||
|
/**
|
||||||
|
* Edge middleware for cran.rpkgs.com.
|
||||||
|
*
|
||||||
|
* Two jobs:
|
||||||
|
*
|
||||||
|
* 1. Resolve the bare `https://cran.rpkgs.com` form to a concrete
|
||||||
|
* `<arch>/<os>` slot from the User-Agent, or send the client to CRAN when
|
||||||
|
* the distro cannot be identified.
|
||||||
|
* 2. Route `PACKAGES*` requests to the per-R-minor slot
|
||||||
|
* (`…/latest/src/contrib/<x.y>/`), so a stock `install.packages()` sees the
|
||||||
|
* packages that only exist there.
|
||||||
|
*
|
||||||
|
* Only index files are routed. Tarballs are deliberately left alone: R keeps
|
||||||
|
* the contrib URL it asked for, not the one it was redirected to, so every
|
||||||
|
* tarball URL is resolved against the flat directory and the union index steers
|
||||||
|
* the per-minor ones with a `Path: <x.y>` field. Rewriting a tarball request
|
||||||
|
* here would send flat-slot packages into a directory that does not hold them.
|
||||||
|
*
|
||||||
|
* Routing is gated on UNION_SLOTS. The raw per-minor index holds only the
|
||||||
|
* ABI-sensitive subset of a slot; it is safe to route to it only once bincraft
|
||||||
|
* has republished it as a union of the per-minor and flat slots.
|
||||||
|
*
|
||||||
|
* Deployed by OpenTofu from this file (`bunnynet_compute_script.rpkgs_router`).
|
||||||
|
* Test with `just edge-test`.
|
||||||
|
*/
|
||||||
|
import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12';
|
||||||
|
|
||||||
|
const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com';
|
||||||
|
const CRAN_ORIGIN = 'https://cran.r-project.org';
|
||||||
|
|
||||||
|
/** Slots ("<arch>/<os>", comma separated) whose per-minor index is a union. */
|
||||||
|
const UNION_SLOTS = new Set(
|
||||||
|
(Deno.env.get('UNION_SLOTS') ?? '')
|
||||||
|
.split(',')
|
||||||
|
.map((slot) => slot.trim())
|
||||||
|
.filter((slot) => slot.length > 0),
|
||||||
|
);
|
||||||
|
|
||||||
|
/** `/<arch>/<os>/latest/src/contrib[/<rest>]` */
|
||||||
|
const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/;
|
||||||
|
|
||||||
|
/** A path that already sits in a per-minor slot, e.g. `4.5/PACKAGES.gz`. */
|
||||||
|
const MINOR_DIR_REGEX = /^\d+\.\d+\//;
|
||||||
|
|
||||||
|
/** The only files this script routes. */
|
||||||
|
const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/;
|
||||||
|
|
||||||
|
const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/;
|
||||||
|
|
||||||
|
const MACOS_BIN_REGEX =
|
||||||
|
/^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/;
|
||||||
|
|
||||||
|
const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i;
|
||||||
|
|
||||||
|
const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i;
|
||||||
|
const UBUNTU_CODENAMES: Record<string, string> = {
|
||||||
|
'24.04': 'noble',
|
||||||
|
'22.04': 'jammy',
|
||||||
|
};
|
||||||
|
|
||||||
|
const ALPINE_REGEX = /(?:Alpine Linux(?:\s+VERSION_ID=)?|alpine-)\s*(\d+)\.(\d+)/i;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* R's own User-Agent is `R (4.5.3 x86_64-pc-linux-musl …)`; the Posit-style one
|
||||||
|
* some sites configure is `R/4.5.3 R (…)`. Both carry the minor, which is why
|
||||||
|
* per-minor routing works without the distro being identifiable.
|
||||||
|
*/
|
||||||
|
const R_MINOR_REGEXES = [/\bR\/(\d+)\.(\d+)/, /\bR \((\d+)\.(\d+)/];
|
||||||
|
|
||||||
|
function normalizePathname(pathname: string): string {
|
||||||
|
return pathname.replace(/\/{2,}/g, '/');
|
||||||
|
}
|
||||||
|
|
||||||
|
function redirectTo(location: string, status = 302): Response {
|
||||||
|
return new Response(null, {
|
||||||
|
status,
|
||||||
|
headers: {
|
||||||
|
Location: location,
|
||||||
|
// The target depends on the User-Agent, so the redirect itself must
|
||||||
|
// never be cached; only its target is a cacheable, UA-independent URL.
|
||||||
|
'Cache-Control': 'no-store',
|
||||||
|
'X-Via': 'MyMiddleware',
|
||||||
|
'X-Rewritten-By': 'rpkgs-edge-middleware',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractRMinor(userAgent: string): string | null {
|
||||||
|
for (const regex of R_MINOR_REGEXES) {
|
||||||
|
const match = userAgent.match(regex);
|
||||||
|
if (match) {
|
||||||
|
return `${match[1]}.${match[2]}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mapArch(arch: string): string {
|
||||||
|
if (arch === 'aarch64') return 'arm64';
|
||||||
|
if (arch === 'x86_64') return 'amd64';
|
||||||
|
return arch;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractArch(userAgent: string): string {
|
||||||
|
const match = userAgent.match(/(x86_64|aarch64|arm64|i386|i686)/);
|
||||||
|
return match ? mapArch(match[1]) : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Identify the `<arch>/<os>` slot from the User-Agent, or null.
|
||||||
|
*
|
||||||
|
* A stock R User-Agent carries only `linux-gnu` / `linux-musl`, which are not
|
||||||
|
* slot names: returning them produced redirects into slots that do not exist
|
||||||
|
* (`/amd64/linux-musl/latest/…`, a guaranteed 404). An unidentifiable distro
|
||||||
|
* is reported as such so the caller can fall back to CRAN.
|
||||||
|
*/
|
||||||
|
function parseSlot(userAgent: string): string | null {
|
||||||
|
const arch = extractArch(userAgent);
|
||||||
|
if (!arch) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const rhel = userAgent.match(RHEL_REGEX);
|
||||||
|
if (rhel) {
|
||||||
|
return `${arch}/rhel${rhel[2]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ubuntu = userAgent.match(UBUNTU_REGEX);
|
||||||
|
if (ubuntu) {
|
||||||
|
const codename = UBUNTU_CODENAMES[ubuntu[1]];
|
||||||
|
if (codename) {
|
||||||
|
return `${arch}/${codename}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const alpine = userAgent.match(ALPINE_REGEX);
|
||||||
|
if (alpine) {
|
||||||
|
return `${arch}/alpine${alpine[1]}${alpine[2]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: string } | null {
|
||||||
|
const rverMatch = userAgent.match(/R \((\d+)\.(\d+)/);
|
||||||
|
const archMatch = userAgent.match(/(aarch64|arm64|x86_64)/);
|
||||||
|
const osMatch = userAgent.match(/darwin(\d+)/);
|
||||||
|
|
||||||
|
if (!rverMatch || !archMatch || !osMatch) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const arch = archMatch[1] === 'aarch64' ? 'arm64' : archMatch[1];
|
||||||
|
const darwinVer = parseInt(osMatch[1], 10);
|
||||||
|
const os = darwinVer >= 21 && darwinVer < 22 ? `monterey-${arch}` : `big-sur-${arch}`;
|
||||||
|
|
||||||
|
return { os, arch, rver: `${rverMatch[1]}.${rverMatch[2]}` };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The contrib path a request should be served from, relative to the slot.
|
||||||
|
*
|
||||||
|
* Returns the per-minor path for an index file when the slot is known to carry
|
||||||
|
* a union index and the client's R minor is known; otherwise the flat path,
|
||||||
|
* which is what every client sees today.
|
||||||
|
*/
|
||||||
|
function contribPath(slot: string, rest: string, userAgent: string): string {
|
||||||
|
const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`;
|
||||||
|
|
||||||
|
if (!INDEX_FILE_REGEX.test(rest) || !UNION_SLOTS.has(slot)) {
|
||||||
|
return flat;
|
||||||
|
}
|
||||||
|
|
||||||
|
const rMinor = extractRMinor(userAgent);
|
||||||
|
return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat;
|
||||||
|
}
|
||||||
|
|
||||||
|
BunnySDK.net.http
|
||||||
|
.servePullZone({ url: 'https://cran.rpkgs.com/' })
|
||||||
|
.onOriginRequest((ctx) => {
|
||||||
|
const url = new URL(ctx.request.url);
|
||||||
|
const path = normalizePathname(url.pathname);
|
||||||
|
const userAgent = ctx.request.headers.get('User-Agent') || '';
|
||||||
|
|
||||||
|
// macOS clients are served from CRAN's own binary tree.
|
||||||
|
const srcContrib = path.match(SRC_CONTRIB_REGEX);
|
||||||
|
if (srcContrib && /darwin/.test(userAgent)) {
|
||||||
|
const mac = parseMacUserAgent(userAgent);
|
||||||
|
if (mac) {
|
||||||
|
return Promise.resolve(
|
||||||
|
redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (MACOS_BIN_REGEX.test(path)) {
|
||||||
|
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Already-qualified slot URLs: what the runtime images have baked in.
|
||||||
|
const slotPath = path.match(SLOT_PATH_REGEX);
|
||||||
|
if (slotPath) {
|
||||||
|
const slot = `${slotPath[1]}/${slotPath[2]}`;
|
||||||
|
const rest = slotPath[3];
|
||||||
|
|
||||||
|
// Never rewrite a request that is already in a per-minor slot, or the
|
||||||
|
// redirect would chase its own tail.
|
||||||
|
if (MINOR_DIR_REGEX.test(rest)) {
|
||||||
|
return Promise.resolve(ctx.request);
|
||||||
|
}
|
||||||
|
|
||||||
|
const target = contribPath(slot, rest, userAgent);
|
||||||
|
if (target === path) {
|
||||||
|
return Promise.resolve(ctx.request);
|
||||||
|
}
|
||||||
|
return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bare `https://cran.rpkgs.com` form, resolved from the User-Agent.
|
||||||
|
if (path === '/' || path === '/src/contrib' || path.startsWith('/src/contrib/')) {
|
||||||
|
const slot = parseSlot(userAgent);
|
||||||
|
if (!slot) {
|
||||||
|
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
const rest = srcContrib ? srcContrib[1] : '';
|
||||||
|
return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
return Promise.resolve(ctx.request);
|
||||||
|
})
|
||||||
|
.onOriginResponse((ctx) => {
|
||||||
|
ctx.response.headers.append('X-Via', 'MyMiddleware');
|
||||||
|
return Promise.resolve(ctx.response);
|
||||||
|
});
|
||||||
14
justfile
14
justfile
|
|
@ -73,3 +73,17 @@ rebuild os tag arch package *versions:
|
||||||
--build-arg CACHEBUST="$(date +%s)" \
|
--build-arg CACHEBUST="$(date +%s)" \
|
||||||
-f docker/build-one.Dockerfile \
|
-f docker/build-one.Dockerfile \
|
||||||
local
|
local
|
||||||
|
|
||||||
|
# run the edge middleware routing matrix (uses a local deno, else the deno image)
|
||||||
|
edge-test:
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
if command -v deno >/dev/null 2>&1; then
|
||||||
|
deno test -A edge/rpkgs-router.test.ts
|
||||||
|
else
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/w" -w /w \
|
||||||
|
-v deno-cache:/deno-dir \
|
||||||
|
denoland/deno:latest \
|
||||||
|
deno test -A edge/rpkgs-router.test.ts
|
||||||
|
fi
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ Spec: `specs/2026-08-07-per-minor-edge-routing-design.md`
|
||||||
|
|
||||||
## Global constraints
|
## Global constraints
|
||||||
|
|
||||||
- Redirect only `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds`; never a tarball, because `Path: ..` entries arrive already normalised to the flat path.
|
- Redirect only `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds`; never a tarball, because the union index already carries the correct tarball URL for both classes of package.
|
||||||
- Every redirect carries `Cache-Control: no-store`; redirect targets stay UA-independent.
|
- Every redirect carries `Cache-Control: no-store`; redirect targets stay UA-independent.
|
||||||
- `UNION_SLOTS` is empty by default, so deploying the script is a no-op until a slot is backfilled.
|
- `UNION_SLOTS` is empty by default, so deploying the script is a no-op until a slot is backfilled.
|
||||||
- A slot is `<arch>/<os>`, e.g. `amd64/alpine324`.
|
- A slot is `<arch>/<os>`, e.g. `amd64/alpine324`.
|
||||||
|
|
@ -138,9 +138,9 @@ git commit -m "feat(cdn): manage the edge middleware script from this repo"
|
||||||
|
|
||||||
- [ ] **Step 1: write the failing tests**
|
- [ ] **Step 1: write the failing tests**
|
||||||
|
|
||||||
- a package present in both slots keeps the per-minor record and gains no `Path`
|
- a package present in both slots keeps the per-minor record, with `Path = "4.5"`
|
||||||
- a package only in the flat slot survives with `Path = ".."`
|
- a package only in the flat slot survives with no `Path`
|
||||||
- a package only in the per-minor slot survives unchanged
|
- a package only in the per-minor slot survives with `Path = "4.5"`
|
||||||
- a union smaller than the flat input raises an error rather than returning
|
- a union smaller than the flat input raises an error rather than returning
|
||||||
|
|
||||||
- [ ] **Step 2: run them and watch them fail**
|
- [ ] **Step 2: run them and watch them fail**
|
||||||
|
|
@ -149,7 +149,7 @@ git commit -m "feat(cdn): manage the edge middleware script from this repo"
|
||||||
|
|
||||||
- [ ] **Step 3: implement `write_union_index()` and call it from `upload_package_index()`**
|
- [ ] **Step 3: implement `write_union_index()` and call it from `upload_package_index()`**
|
||||||
|
|
||||||
After `update_PACKAGES()` has written the per-minor index, read the flat slot's `PACKAGES.rds`, drop packages already in the per-minor index, set `Path = ".."` on the rest, and rewrite `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` in the per-minor slot.
|
After `update_PACKAGES()` has written the per-minor index, read the flat slot's `PACKAGES.rds`, set `Path = <r_minor>` on the per-minor records, drop the flat records for packages the per-minor slot already has, and rewrite `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` in the per-minor slot.
|
||||||
|
|
||||||
- [ ] **Step 4: run the tests until they pass**
|
- [ ] **Step 4: run the tests until they pass**
|
||||||
|
|
||||||
|
|
@ -159,7 +159,7 @@ After `update_PACKAGES()` has written the per-minor index, read the flat slot's
|
||||||
|
|
||||||
## Task 4: Roll out slot by slot
|
## Task 4: Roll out slot by slot
|
||||||
|
|
||||||
- [ ] Re-index one slot (`amd64/alpine324`, R 4.5) and confirm the union index lists both `curl` (per-minor) and `jsonlite` (flat, `Path: ..`).
|
- [ ] Re-index one slot (`amd64/alpine324`, R 4.5) and confirm the union index lists both `curl` (per-minor, `Path: 4.5`) and `jsonlite` (flat, no `Path`).
|
||||||
- [ ] Set `UNION_SLOTS = "amd64/alpine324"` and confirm in `reg.devxy.io/r/r-alpine:4.5-3.24` that `available.packages()` returns the union count and `"curl" %in% rownames(...)`.
|
- [ ] Set `UNION_SLOTS = "amd64/alpine324"` and confirm in `reg.devxy.io/r/r-alpine:4.5-3.24` that `available.packages()` returns the union count and `"curl" %in% rownames(...)`.
|
||||||
- [ ] Add `arm64/alpine324`, then the remaining slots.
|
- [ ] Add `arm64/alpine324`, then the remaining slots.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -35,30 +35,39 @@ Let a stock `install.packages()` see one complete package list for its own R min
|
||||||
|
|
||||||
## Key constraint that drives the design
|
## Key constraint that drives the design
|
||||||
|
|
||||||
R resolves a package's download URL from the index, not from the request path.
|
R resolves a package's download URL from the index, not from the request path, and it keeps the `contriburl` it _asked for_ rather than the one it was redirected to.
|
||||||
`available.packages()` honours a `Path:` field in `PACKAGES` and folds it into the `Repository` column, and libcurl normalises dot segments **before** the request leaves the client.
|
Measured with `options(repos = …/latest)` against a middleware that redirects the index into `4.5/`:
|
||||||
|
|
||||||
Verified against the live CDN:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
# Path: .. in a per-minor index resolves to the flat slot
|
curl available: TRUE
|
||||||
curl "https://cran.rpkgs.com/amd64/alpine324/latest/src/contrib/4.5/../jsonlite_2.0.0.tar.gz"
|
curl repo: …/latest/src/contrib # the flat URL, not the 4.5 one it was served from
|
||||||
# -> 200, effective URL .../latest/src/contrib/jsonlite_2.0.0.tar.gz
|
|
||||||
```
|
```
|
||||||
|
|
||||||
So a per-minor index can point back at flat-slot tarballs at zero storage cost, and the edge never sees a `/4.5/../` path.
|
So the union index is always addressed relative to the **flat** directory, whatever path it was fetched from.
|
||||||
The corollary is that the edge script must **not** rewrite tarball requests: a flat-slot tarball arrives already normalised to the flat path, and redirecting it into `<x.y>/` would 404 exactly the packages `Path: ..` was meant to serve.
|
`available.packages()` honours a `Path:` field and folds it into the `Repository` column, which gives the whole routing for free:
|
||||||
|
|
||||||
|
- a per-minor record carries `Path: <x.y>`, so its tarball is fetched from `…/src/contrib/<x.y>/`
|
||||||
|
- a flat record carries no `Path`, so its tarball is fetched from `…/src/contrib/`
|
||||||
|
|
||||||
|
Verified end to end against the live CDN with a locally built union index for `amd64/alpine324` (31 507 records):
|
||||||
|
|
||||||
|
```
|
||||||
|
curl: 7.1.0 -> …/latest/src/contrib/4.5 -> curl_7.1.0.tar.gz 717 725 B
|
||||||
|
jsonlite: 2.0.0 -> …/latest/src/contrib -> jsonlite_2.0.0.tar.gz 1 055 849 B
|
||||||
|
```
|
||||||
|
|
||||||
|
The corollary is that the edge script must **not** rewrite tarball requests: every tarball URL is already correct when it leaves the client, and redirecting one into `<x.y>/` would break exactly the flat packages the union is meant to preserve.
|
||||||
|
|
||||||
The complementary trick does not work: R's `gzcon()` reads only the first member of a concatenated gzip stream (10 291 of an expected 31 931 records), so an edge-side merge would have to fully decompress and recompress both indices and additionally 404 `PACKAGES.rds` to stop R preferring it.
|
The complementary trick does not work: R's `gzcon()` reads only the first member of a concatenated gzip stream (10 291 of an expected 31 931 records), so an edge-side merge would have to fully decompress and recompress both indices and additionally 404 `PACKAGES.rds` to stop R preferring it.
|
||||||
That is why the union is produced in `bincraft`, not at the edge.
|
That is why the union is produced in `bincraft`, not at the edge.
|
||||||
|
|
||||||
## Approaches considered
|
## Approaches considered
|
||||||
|
|
||||||
| Approach | Where the union lives | Verdict |
|
| Approach | Where the union lives | Verdict |
|
||||||
| ----------------------------------------------------------- | ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- |
|
| ----------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- |
|
||||||
| **A. Union index written by `bincraft` (chosen)** | per-minor `PACKAGES*`, flat entries carry `Path: ..` | Edge does one redirect; `PACKAGES.rds` stays correct; no duplication |
|
| **A. Union index written by `bincraft` (chosen)** | per-minor `PACKAGES*`, per-minor entries carry `Path: <x.y>` | Edge does one redirect; `PACKAGES.rds` stays correct; no duplication |
|
||||||
| B. Merge at the edge | middleware fetches both indices, recompresses | ~2 MB decompress/recompress per cache fill, cache key must include the R minor, breaks R's `.rds` fast path |
|
| B. Merge at the edge | middleware fetches both indices, recompresses | ~2 MB decompress/recompress per cache fill, cache key must include the R minor, breaks R's `.rds` fast path |
|
||||||
| C. Move the minor up the path (`latest/<x.y>/src/contrib/`) | addressable by `options(repos)` directly | No edge logic at all, but a full layout migration and breaks the published URL contract |
|
| C. Move the minor up the path (`latest/<x.y>/src/contrib/`) | addressable by `options(repos)` directly | No edge logic at all, but a full layout migration and breaks the published URL contract |
|
||||||
|
|
||||||
Chosen: **A**.
|
Chosen: **A**.
|
||||||
|
|
||||||
|
|
@ -69,8 +78,8 @@ Chosen: **A**.
|
||||||
After writing a per-minor index, republish it as a union of that slot and the flat slot:
|
After writing a per-minor index, republish it as a union of that slot and the flat slot:
|
||||||
|
|
||||||
1. Read the flat slot's `PACKAGES.rds` and the per-minor slot's own records.
|
1. Read the flat slot's `PACKAGES.rds` and the per-minor slot's own records.
|
||||||
2. Drop every flat record whose package is already present in the per-minor slot, so the per-minor build always wins.
|
2. Set `Path: <x.y>` on every per-minor record, so its tarball resolves into the per-minor directory.
|
||||||
3. Add `Path: ..` to the surviving flat records.
|
3. Drop every flat record whose package is already present in the per-minor slot, so the per-minor build always wins, and leave the survivors without a `Path`.
|
||||||
4. Write the merged `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib/<x.y>/`.
|
4. Write the merged `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib/<x.y>/`.
|
||||||
|
|
||||||
Guard: refuse to publish a union with fewer records than the flat index it was built from.
|
Guard: refuse to publish a union with fewer records than the flat index it was built from.
|
||||||
|
|
@ -150,7 +159,7 @@ Local, before any apply: `deno run -A edge/rpkgs-router.ts` serves the middlewar
|
||||||
After apply, a smoke test against `cran.rpkgs.com`:
|
After apply, a smoke test against `cran.rpkgs.com`:
|
||||||
|
|
||||||
- `available.packages()` inside `reg.devxy.io/r/r-alpine:4.5-3.24` returns the union count, and `"curl" %in% rownames(...)` is `TRUE`.
|
- `available.packages()` inside `reg.devxy.io/r/r-alpine:4.5-3.24` returns the union count, and `"curl" %in% rownames(...)` is `TRUE`.
|
||||||
- A flat-slot package still downloads (`Path: ..` path), and a per-minor package downloads from `<x.y>/`.
|
- A flat-slot package still downloads from `…/src/contrib/`, and a per-minor package downloads from `…/src/contrib/<x.y>/`.
|
||||||
|
|
||||||
## Out of scope
|
## Out of scope
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue