feat(edge): route PACKAGES requests to the per-R-minor slot #152

Merged
pat-s merged 4 commits from feat/per-minor-edge-routing into main 2026-08-07 14:12:22 +00:00
6 changed files with 773 additions and 1 deletions

25
cdn.tf
View file

@ -52,6 +52,29 @@
### cran.rpkgs.com
# The edge middleware that resolves the bare cran.rpkgs.com form to an
# <arch>/<os> slot and routes PACKAGES* to the per-R-minor slot. The source of
# truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release.
#
# The script pre-dates this configuration, so it is adopted rather than created:
# tofu import bunnynet_compute_script.rpkgs_router 29277
resource "bunnynet_compute_script" "rpkgs_router" {
type = "middleware"
name = "rpkgs-router"
content = file("${path.module}/edge/rpkgs-router.ts")
}
# Slots ("<arch>/<os>", comma separated) whose per-minor index bincraft has
# already republished as a union of the per-minor and flat slots. Routing to a
# slot that is not listed here would hide every package the per-minor index does
# not carry, so this stays empty until a slot has been backfilled.
resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" {
script = bunnynet_compute_script.rpkgs_router.id
name = "UNION_SLOTS"
default_value = ""
required = false
}
resource "bunnynet_pullzone" "cran_rpkgs_com" {
name = "cran-rpkgs"
@ -64,7 +87,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" {
origin {
type = "OriginUrl"
url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com"
middleware_script = 29277
middleware_script = bunnynet_compute_script.rpkgs_router.id
}
routing {

160
edge/rpkgs-router.test.ts Normal file
View file

@ -0,0 +1,160 @@
/**
* Routing matrix for `edge/rpkgs-router.ts`.
*
* The script is exercised through the SDK's local server rather than by
* importing its internals, so what is tested is the artifact that gets
* deployed. Requests that the script passes through are proxied to the real
* origin, which keeps the "no redirect" cases honest: they assert that the
* client reached the flat slot, not merely that no `Location` was set.
*
* Run with `just edge-test`.
*/
import { assertEquals } from 'jsr:@std/assert@1';
const SCRIPT = new URL('./rpkgs-router.ts', import.meta.url).pathname;
const BASE = 'http://127.0.0.1:8080';
const UNION_SLOTS = 'amd64/alpine324';
const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)';
const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)';
const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24';
const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)';
const UA_CURL = 'curl/8.0.1';
const SLOT = '/amd64/alpine324/latest/src/contrib';
const OTHER_SLOT = '/amd64/noble/latest/src/contrib';
interface Probe {
status: number;
location: string | null;
cacheControl: string | null;
}
async function probe(path: string, userAgent: string): Promise<Probe> {
const res = await fetch(BASE + path, {
headers: { 'User-Agent': userAgent },
redirect: 'manual',
});
await res.body?.cancel();
return {
status: res.status,
location: res.headers.get('location'),
cacheControl: res.headers.get('cache-control'),
};
}
/** Kill tolerantly: the child has already exited if the script failed to load. */
async function stopServer(child: Deno.ChildProcess): Promise<void> {
try {
child.kill();
} catch {
// already gone
}
await child.status;
}
async function startServer(): Promise<Deno.ChildProcess> {
const child = new Deno.Command(Deno.execPath(), {
args: ['run', '-A', SCRIPT],
env: { UNION_SLOTS },
stdout: 'null',
stderr: 'inherit',
}).spawn();
for (let attempt = 0; attempt < 150; attempt++) {
try {
const res = await fetch(`${BASE}/`, {
headers: { 'User-Agent': UA_CURL },
redirect: 'manual',
});
await res.body?.cancel();
return child;
} catch {
await new Promise((resolve) => setTimeout(resolve, 200));
}
}
await stopServer(child);
throw new Error('edge script did not start listening on ' + BASE);
}
Deno.test('rpkgs-router', async (t) => {
const server = await startServer();
try {
await t.step("routes an index request to the client's R minor", async () => {
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL);
assertEquals(res.status, 302);
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`);
});
await t.step('routes R 4.6 to its own slot', async () => {
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R46_MUSL);
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`);
});
await t.step('routes PACKAGES and PACKAGES.rds too', async () => {
for (const file of ['PACKAGES', 'PACKAGES.rds']) {
const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL);
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/${file}`, `expected ${file} to be routed`);
}
});
await t.step('marks the redirect uncacheable', async () => {
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL);
assertEquals(res.cacheControl, 'no-store');
});
await t.step('leaves a slot outside UNION_SLOTS alone', async () => {
const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R45_MUSL);
assertEquals(res.location, null);
assertEquals(res.status, 200);
});
await t.step('never routes a tarball', async () => {
const res = await probe(`${SLOT}/jsonlite_2.0.0.tar.gz`, UA_R45_MUSL);
assertEquals(res.location, null);
assertEquals(res.status, 200);
});
await t.step('does not redirect a path already under a minor', async () => {
const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL);
assertEquals(res.location, null);
assertEquals(res.status, 200);
});
await t.step('leaves a client without an R version alone', async () => {
const res = await probe(`${SLOT}/PACKAGES.gz`, UA_CURL);
assertEquals(res.location, null);
assertEquals(res.status, 200);
});
await t.step('resolves the bare root to slot and minor', async () => {
const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_ALPINE);
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`);
});
await t.step('sends an unidentifiable distro to CRAN', async () => {
const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL);
assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz');
});
await t.step('keeps the macOS rewrite', async () => {
const res = await probe('/src/contrib/foo_1.0.tar.gz', UA_R45_DARWIN);
assertEquals(res.location, 'https://cran.rpkgs.com/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz');
});
await t.step('keeps the macOS binary passthrough to CRAN', async () => {
const path = '/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz';
const res = await probe(path, UA_R45_DARWIN);
assertEquals(res.location, `https://cran.r-project.org${path}`);
});
await t.step('collapses duplicate slashes before matching', async () => {
const res = await probe(`/amd64/alpine324//latest/src/contrib//PACKAGES.gz`, UA_R45_MUSL);
assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`);
});
} finally {
await stopServer(server);
}
});

235
edge/rpkgs-router.ts Normal file
View file

@ -0,0 +1,235 @@
/**
* Edge middleware for cran.rpkgs.com.
*
* Two jobs:
*
* 1. Resolve the bare `https://cran.rpkgs.com` form to a concrete
* `<arch>/<os>` slot from the User-Agent, or send the client to CRAN when
* the distro cannot be identified.
* 2. Route `PACKAGES*` requests to the per-R-minor slot
* (`…/latest/src/contrib/<x.y>/`), so a stock `install.packages()` sees the
* packages that only exist there.
*
* Only index files are routed. Tarballs are deliberately left alone: R keeps
* the contrib URL it asked for, not the one it was redirected to, so every
* tarball URL is resolved against the flat directory and the union index steers
* the per-minor ones with a `Path: <x.y>` field. Rewriting a tarball request
* here would send flat-slot packages into a directory that does not hold them.
*
* Routing is gated on UNION_SLOTS. The raw per-minor index holds only the
* ABI-sensitive subset of a slot; it is safe to route to it only once bincraft
* has republished it as a union of the per-minor and flat slots.
*
* Deployed by OpenTofu from this file (`bunnynet_compute_script.rpkgs_router`).
* Test with `just edge-test`.
*/
import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12';
const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com';
const CRAN_ORIGIN = 'https://cran.r-project.org';
/** Slots ("<arch>/<os>", comma separated) whose per-minor index is a union. */
const UNION_SLOTS = new Set(
(Deno.env.get('UNION_SLOTS') ?? '')
.split(',')
.map((slot) => slot.trim())
.filter((slot) => slot.length > 0),
);
/** `/<arch>/<os>/latest/src/contrib[/<rest>]` */
const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/;
/** A path that already sits in a per-minor slot, e.g. `4.5/PACKAGES.gz`. */
const MINOR_DIR_REGEX = /^\d+\.\d+\//;
/** The only files this script routes. */
const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/;
const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/;
const MACOS_BIN_REGEX =
/^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/;
const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i;
const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i;
const UBUNTU_CODENAMES: Record<string, string> = {
'24.04': 'noble',
'22.04': 'jammy',
};
const ALPINE_REGEX = /(?:Alpine Linux(?:\s+VERSION_ID=)?|alpine-)\s*(\d+)\.(\d+)/i;
/**
* R's own User-Agent is `R (4.5.3 x86_64-pc-linux-musl …)`; the Posit-style one
* some sites configure is `R/4.5.3 R (…)`. Both carry the minor, which is why
* per-minor routing works without the distro being identifiable.
*/
const R_MINOR_REGEXES = [/\bR\/(\d+)\.(\d+)/, /\bR \((\d+)\.(\d+)/];
function normalizePathname(pathname: string): string {
return pathname.replace(/\/{2,}/g, '/');
}
function redirectTo(location: string, status = 302): Response {
return new Response(null, {
status,
headers: {
Location: location,
// The target depends on the User-Agent, so the redirect itself must
// never be cached; only its target is a cacheable, UA-independent URL.
'Cache-Control': 'no-store',
'X-Via': 'MyMiddleware',
'X-Rewritten-By': 'rpkgs-edge-middleware',
},
});
}
function extractRMinor(userAgent: string): string | null {
for (const regex of R_MINOR_REGEXES) {
const match = userAgent.match(regex);
if (match) {
return `${match[1]}.${match[2]}`;
}
}
return null;
}
function mapArch(arch: string): string {
if (arch === 'aarch64') return 'arm64';
if (arch === 'x86_64') return 'amd64';
return arch;
}
function extractArch(userAgent: string): string {
const match = userAgent.match(/(x86_64|aarch64|arm64|i386|i686)/);
return match ? mapArch(match[1]) : '';
}
/**
* Identify the `<arch>/<os>` slot from the User-Agent, or null.
*
* A stock R User-Agent carries only `linux-gnu` / `linux-musl`, which are not
* slot names: returning them produced redirects into slots that do not exist
* (`/amd64/linux-musl/latest/…`, a guaranteed 404). An unidentifiable distro
* is reported as such so the caller can fall back to CRAN.
*/
function parseSlot(userAgent: string): string | null {
const arch = extractArch(userAgent);
if (!arch) {
return null;
}
const rhel = userAgent.match(RHEL_REGEX);
if (rhel) {
return `${arch}/rhel${rhel[2]}`;
}
const ubuntu = userAgent.match(UBUNTU_REGEX);
if (ubuntu) {
const codename = UBUNTU_CODENAMES[ubuntu[1]];
if (codename) {
return `${arch}/${codename}`;
}
}
const alpine = userAgent.match(ALPINE_REGEX);
if (alpine) {
return `${arch}/alpine${alpine[1]}${alpine[2]}`;
}
return null;
}
function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: string } | null {
const rverMatch = userAgent.match(/R \((\d+)\.(\d+)/);
const archMatch = userAgent.match(/(aarch64|arm64|x86_64)/);
const osMatch = userAgent.match(/darwin(\d+)/);
if (!rverMatch || !archMatch || !osMatch) {
return null;
}
const arch = archMatch[1] === 'aarch64' ? 'arm64' : archMatch[1];
const darwinVer = parseInt(osMatch[1], 10);
const os = darwinVer >= 21 && darwinVer < 22 ? `monterey-${arch}` : `big-sur-${arch}`;
return { os, arch, rver: `${rverMatch[1]}.${rverMatch[2]}` };
}
/**
* The contrib path a request should be served from, relative to the slot.
*
* Returns the per-minor path for an index file when the slot is known to carry
* a union index and the client's R minor is known; otherwise the flat path,
* which is what every client sees today.
*/
function contribPath(slot: string, rest: string, userAgent: string): string {
const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`;
if (!INDEX_FILE_REGEX.test(rest) || !UNION_SLOTS.has(slot)) {
return flat;
}
const rMinor = extractRMinor(userAgent);
return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat;
}
BunnySDK.net.http
.servePullZone({ url: 'https://cran.rpkgs.com/' })
.onOriginRequest((ctx) => {
const url = new URL(ctx.request.url);
const path = normalizePathname(url.pathname);
const userAgent = ctx.request.headers.get('User-Agent') || '';
// macOS clients are served from CRAN's own binary tree.
const srcContrib = path.match(SRC_CONTRIB_REGEX);
if (srcContrib && /darwin/.test(userAgent)) {
const mac = parseMacUserAgent(userAgent);
if (mac) {
return Promise.resolve(
redirectTo(`${PUBLIC_CDN_ORIGIN}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`),
);
}
}
if (MACOS_BIN_REGEX.test(path)) {
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`));
}
// Already-qualified slot URLs: what the runtime images have baked in.
const slotPath = path.match(SLOT_PATH_REGEX);
if (slotPath) {
const slot = `${slotPath[1]}/${slotPath[2]}`;
const rest = slotPath[3];
// Never rewrite a request that is already in a per-minor slot, or the
// redirect would chase its own tail.
if (MINOR_DIR_REGEX.test(rest)) {
return Promise.resolve(ctx.request);
}
const target = contribPath(slot, rest, userAgent);
if (target === path) {
return Promise.resolve(ctx.request);
}
return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${target}`));
}
// The bare `https://cran.rpkgs.com` form, resolved from the User-Agent.
if (path === '/' || path === '/src/contrib' || path.startsWith('/src/contrib/')) {
const slot = parseSlot(userAgent);
if (!slot) {
return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`));
}
const rest = srcContrib ? srcContrib[1] : '';
return Promise.resolve(redirectTo(`${PUBLIC_CDN_ORIGIN}${contribPath(slot, rest, userAgent)}`));
}
return Promise.resolve(ctx.request);
})
.onOriginResponse((ctx) => {
ctx.response.headers.append('X-Via', 'MyMiddleware');
return Promise.resolve(ctx.response);
});

View file

@ -73,3 +73,17 @@ rebuild os tag arch package *versions:
--build-arg CACHEBUST="$(date +%s)" \
-f docker/build-one.Dockerfile \
local
# run the edge middleware routing matrix (uses a local deno, else the deno image)
edge-test:
#!/usr/bin/env bash
set -euo pipefail
if command -v deno >/dev/null 2>&1; then
deno test -A edge/rpkgs-router.test.ts
else
docker run --rm \
-v "$PWD:/w" -w /w \
-v deno-cache:/deno-dir \
denoland/deno:latest \
deno test -A edge/rpkgs-router.test.ts
fi

View file

@ -0,0 +1,166 @@
# Per-R-minor edge routing implementation plan
Spec: `specs/2026-08-07-per-minor-edge-routing-design.md`
**Goal:** let a stock `install.packages()` see the per-minor packages by routing `PACKAGES*` requests to `…/src/contrib/<x.y>/`, where `bincraft` publishes a union index.
**Architecture:** the union is built in `bincraft`; the edge script only redirects index requests, gated on a `UNION_SLOTS` script variable; the script lives in this repo and is applied by OpenTofu.
**Tech stack:** Deno / TypeScript (Bunny Edge Scripting, SDK 0.12), OpenTofu with `BunnyWay/bunnynet` 0.17, R (bincraft).
## Global constraints
- Redirect only `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds`; never a tarball, because the union index already carries the correct tarball URL for both classes of package.
- Every redirect carries `Cache-Control: no-store`; redirect targets stay UA-independent.
- `UNION_SLOTS` is empty by default, so deploying the script is a no-op until a slot is backfilled.
- A slot is `<arch>/<os>`, e.g. `amd64/alpine324`.
- Verified prerequisites: `PACKAGES*` is served `cdn-cache: BYPASS`, so the script sees every index request; `Deno.env.get()` reads script variables; the SDK local server listens on `127.0.0.1:8080`.
---
## Task 1: Edge script and its test matrix
**Files:**
- Create: `edge/rpkgs-router.ts`
- Create: `edge/rpkgs-router.test.ts`
- Modify: `justfile` (add `edge-test`)
**Produces:** a single-file script deployable as `bunnynet_compute_script.content`, reading `UNION_SLOTS` from the environment.
- [ ] **Step 1: write the test matrix first**
`edge/rpkgs-router.test.ts` spawns `deno run -A edge/rpkgs-router.ts` with `UNION_SLOTS=amd64/alpine324`, waits for `127.0.0.1:8080`, and issues requests with `redirect: "manual"`.
Cases, asserted on the `location` header (or its absence):
| # | path | User-Agent | expectation |
| --- | ------------------------------------------------------ | --------------------------------------------- | ------------------------------------------------------------ |
| 1 | `/amd64/alpine324/latest/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)` | 302 → `…/src/contrib/4.5/PACKAGES.gz` |
| 2 | same | `R (4.6.0 …)` | 302 → `…/src/contrib/4.6/PACKAGES.gz` |
| 3 | same, but slot `amd64/noble` | `R (4.5.3 …)` | no redirect (slot not in `UNION_SLOTS`) |
| 4 | `…/src/contrib/curl_7.1.0.tar.gz` | `R (4.5.3 …)` | no redirect |
| 5 | `…/src/contrib/4.5/PACKAGES.gz` | `R (4.5.3 …)` | no redirect (loop guard) |
| 6 | `…/src/contrib/PACKAGES.gz` | `curl/8.0` | no redirect (no R minor) |
| 7 | `/src/contrib/PACKAGES.gz` | alpine UA with `Alpine Linux … 3.24` | 302 → `/amd64/alpine324/latest/src/contrib/4.5/PACKAGES.gz` |
| 8 | `/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)`, no distro | 302 → `cran.r-project.org`, **not** a `linux-musl` slot |
| 9 | `/src/contrib/foo_1.0.tar.gz` | `R (4.5.1 aarch64-apple-darwin20 …)` | 302 → `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` |
| 10 | `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | any | 302 → `cran.r-project.org` |
| 11 | any redirect above | — | `cache-control: no-store` |
- [ ] **Step 2: run the tests and watch them fail**
`just edge-test` → every case fails, because `edge/rpkgs-router.ts` does not exist.
- [ ] **Step 3: write `edge/rpkgs-router.ts`**
Order of evaluation in `onOriginRequest`:
1. normalise `//` runs in the path
2. darwin `/src/contrib/*``/bin/macosx/<flavour>/contrib/<x.y>/`
3. `/bin/macosx/**` → CRAN
4. `/{arch}/{os}/latest/src/contrib/<rest>`: pass through if `rest` already starts with `<x.y>/`, or is not an index file, or the slot is not in `UNION_SLOTS`, or the UA has no R minor; otherwise redirect into `<x.y>/`
5. `/`, `/src/contrib`, `/src/contrib/**`: resolve arch+os from the UA, redirect to CRAN when the distro is unidentifiable, otherwise redirect to the qualified path, adding `<x.y>/` under the same index-file rule
6. anything else: pass through
The R minor comes from either `R/4.5.3` or `R (4.5.3 …)`, so a stock UA is enough. The `linux-gnu` / `linux-musl` fallback in `parseUserAgent` is deleted: those are not slot names.
- [ ] **Step 4: run the tests until they pass**
`just edge-test`
- [ ] **Step 5: commit**
```bash
git add edge/rpkgs-router.ts edge/rpkgs-router.test.ts justfile
git commit -m "feat(edge): route PACKAGES requests to the per-R-minor slot"
```
---
## Task 2: Manage the script from OpenTofu
**Files:**
- Modify: `cdn.tf`
**Consumes:** `edge/rpkgs-router.ts` from Task 1.
- [ ] **Step 1: add the resources**
```terraform
resource "bunnynet_compute_script" "rpkgs_router" {
type = "middleware"
name = "rpkgs-router"
content = file("${path.module}/edge/rpkgs-router.ts")
}
resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" {
script = bunnynet_compute_script.rpkgs_router.id
name = "UNION_SLOTS"
default_value = ""
required = false
}
```
and replace `middleware_script = 29277` with `middleware_script = bunnynet_compute_script.rpkgs_router.id`.
- [ ] **Step 2: validate**
`tofu init -backend=false && tofu validate`
- [ ] **Step 3: import the existing script (needs `BUNNYNET_API_KEY`)**
```bash
tofu import bunnynet_compute_script.rpkgs_router 29277
tofu plan
```
The plan must show an in-place `content` update and **no** replacement of the pull zone. A replacement means the import did not take.
- [ ] **Step 4: commit**
```bash
git add cdn.tf
git commit -m "feat(cdn): manage the edge middleware script from this repo"
```
---
## Task 3: Union index writer in bincraft
**Files (repo `codefloe.com/rpkgs/bincraft`):**
- Modify: `R/package_index.R`
- Test: `tests/testthat/test-package_index.R`
**Produces:** `write_union_index(flat_records, minor_records)` returning the merged records, called from `upload_package_index()` when `r_minor` is set.
- [ ] **Step 1: write the failing tests**
- a package present in both slots keeps the per-minor record, with `Path = "4.5"`
- a package only in the flat slot survives with no `Path`
- a package only in the per-minor slot survives with `Path = "4.5"`
- a union smaller than the flat input raises an error rather than returning
- [ ] **Step 2: run them and watch them fail**
`Rscript -e 'testthat::test_file("tests/testthat/test-package_index.R")'`
- [ ] **Step 3: implement `write_union_index()` and call it from `upload_package_index()`**
After `update_PACKAGES()` has written the per-minor index, read the flat slot's `PACKAGES.rds`, set `Path = <r_minor>` on the per-minor records, drop the flat records for packages the per-minor slot already has, and rewrite `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` in the per-minor slot.
- [ ] **Step 4: run the tests until they pass**
- [ ] **Step 5: commit and open the PR against bincraft**
---
## Task 4: Roll out slot by slot
- [ ] Re-index one slot (`amd64/alpine324`, R 4.5) and confirm the union index lists both `curl` (per-minor, `Path: 4.5`) and `jsonlite` (flat, no `Path`).
- [ ] Set `UNION_SLOTS = "amd64/alpine324"` and confirm in `reg.devxy.io/r/r-alpine:4.5-3.24` that `available.packages()` returns the union count and `"curl" %in% rownames(...)`.
- [ ] Add `arm64/alpine324`, then the remaining slots.
`install.packages("curl")` will still fail to build on `alpine324` until that slot's source tarballs are replaced with real binaries. That is tracked separately.

View file

@ -0,0 +1,174 @@
# Design: Routing clients to per-R-minor binary slots
Date: 2026-08-07
Status: Approved (pending spec review)
## Problem
`bincraft` routes ABI-"risky" packages to a per-minor slot `…/latest/src/contrib/<x.y>/` and indexes every directory independently (`upload_package_index()` calls `cranlike::update_PACKAGES()` on one prefix at a time).
Nothing unions those indices, and `contrib.url()` only ever yields `<repos>/src/contrib`, so no value of `options(repos)` can address a per-minor slot.
Only `uvr` resolves per-minor URLs, which means the per-minor slots are invisible to `install.packages()` by construction.
Measured on 2026-08-07:
| slot | flat `src/contrib` | `src/contrib/4.5` | unique packages only in the per-minor slot |
| ----------------- | ------------------ | ----------------- | ------------------------------------------ |
| `amd64/alpine324` | 21 640 | 3 310 | 2 886 |
| `amd64/noble` | 24 495 | 398 | 23 |
This is what issue #63 records as "missing binaries" on `alpine324`.
The packages are not missing; they are in a directory base R cannot reach.
The user-visible symptom in `reg.devxy.io/r/r-alpine:4.5-3.24` is:
```
> install.packages("curl")
Warning message:
package 'curl' is not available for this version of R
```
A second, unrelated defect exists on the same slot and is **out of scope here**: many `alpine324` tarballs are byte-identical CRAN _source_ tarballs that the index nevertheless stamps `Built: R 4.5.3; …-linux-musl`.
Routing exposes `curl`; only a rebuild of that slot makes it install.
## Goal
Let a stock `install.packages()` see one complete package list for its own R minor, without duplicating tarballs and without an R-version-varying cache key anywhere in the CDN.
## Key constraint that drives the design
R resolves a package's download URL from the index, not from the request path, and it keeps the `contriburl` it _asked for_ rather than the one it was redirected to.
Measured with `options(repos = …/latest)` against a middleware that redirects the index into `4.5/`:
```
curl available: TRUE
curl repo: …/latest/src/contrib # the flat URL, not the 4.5 one it was served from
```
So the union index is always addressed relative to the **flat** directory, whatever path it was fetched from.
`available.packages()` honours a `Path:` field and folds it into the `Repository` column, which gives the whole routing for free:
- a per-minor record carries `Path: <x.y>`, so its tarball is fetched from `…/src/contrib/<x.y>/`
- a flat record carries no `Path`, so its tarball is fetched from `…/src/contrib/`
Verified end to end against the live CDN with a locally built union index for `amd64/alpine324` (31 507 records):
```
curl: 7.1.0 -> …/latest/src/contrib/4.5 -> curl_7.1.0.tar.gz 717 725 B
jsonlite: 2.0.0 -> …/latest/src/contrib -> jsonlite_2.0.0.tar.gz 1 055 849 B
```
The corollary is that the edge script must **not** rewrite tarball requests: every tarball URL is already correct when it leaves the client, and redirecting one into `<x.y>/` would break exactly the flat packages the union is meant to preserve.
The complementary trick does not work: R's `gzcon()` reads only the first member of a concatenated gzip stream (10 291 of an expected 31 931 records), so an edge-side merge would have to fully decompress and recompress both indices and additionally 404 `PACKAGES.rds` to stop R preferring it.
That is why the union is produced in `bincraft`, not at the edge.
## Approaches considered
| Approach | Where the union lives | Verdict |
| ----------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- |
| **A. Union index written by `bincraft` (chosen)** | per-minor `PACKAGES*`, per-minor entries carry `Path: <x.y>` | Edge does one redirect; `PACKAGES.rds` stays correct; no duplication |
| B. Merge at the edge | middleware fetches both indices, recompresses | ~2 MB decompress/recompress per cache fill, cache key must include the R minor, breaks R's `.rds` fast path |
| C. Move the minor up the path (`latest/<x.y>/src/contrib/`) | addressable by `options(repos)` directly | No edge logic at all, but a full layout migration and breaks the published URL contract |
Chosen: **A**.
## Architecture
### bincraft: union index (separate PR)
After writing a per-minor index, republish it as a union of that slot and the flat slot:
1. Read the flat slot's `PACKAGES.rds` and the per-minor slot's own records.
2. Set `Path: <x.y>` on every per-minor record, so its tarball resolves into the per-minor directory.
3. Drop every flat record whose package is already present in the per-minor slot, so the per-minor build always wins, and leave the survivors without a `Path`.
4. Write the merged `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib/<x.y>/`.
Guard: refuse to publish a union with fewer records than the flat index it was built from.
A truncated union is worse than no union, because it silently removes packages from every client on that minor.
### Edge script (this repo)
The script routes `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` requests, and nothing else.
```
normalize path
parseClient(UA) -> { rMinor, arch, os } # rMinor from "R (4.5.3 …)" or "R/4.5.3"
darwin branches # unchanged
if path is /{arch}/{os}/latest/src/contrib/PACKAGES*
already under /<x.y>/ ? pass through # loop guard
rMinor known && slot in UNION_SLOTS ? 302 -> …/src/contrib/<rMinor>/PACKAGES*
else pass through # flat slot, today's behaviour
if path is /src/contrib/… # bare root
resolve arch + os; unknown -> 302 to CRAN
then apply the same PACKAGES* rule
else pass through
```
Redirects carry `Cache-Control: no-store`.
Every cacheable URL is therefore UA-independent, and no cache key has to vary by R version.
### Repaired bare-root detection
The bare `https://cran.rpkgs.com` form is currently broken for every Linux client that uses a stock R user agent.
`ALPINE_REGEX`, `UBUNTU_REGEX` and `RHEL_REGEX` only match a Posit-style user agent the user has to set by hand; stock R never carries the distro, so the script falls through to `extractOs()` and redirects to a slot that does not exist:
```
UA: R (4.5.3 x86_64-pc-linux-musl …) -> 302 /amd64/linux-musl/latest/… (404)
UA: R (4.5.3 x86_64-pc-linux-gnu …) -> 302 /amd64/linux-gnu/latest/… (404)
```
The fallback to a phantom `linux-musl` / `linux-gnu` slot is removed.
An unidentifiable distro redirects to CRAN, which is the existing behaviour for an unparseable user agent.
The R _minor_ is always present in a stock user agent, so per-minor routing itself does not depend on distro detection.
### Rollout gate
`UNION_SLOTS` is a `bunnynet_compute_script_variable` listing the slots whose per-minor index is already a union.
It is empty by default, so deploying the script changes nothing until `bincraft` has backfilled a slot, and a rollback is a variable edit rather than a code deploy.
All slots currently carry `4.4`, `4.5` and `4.6`; a client on any other minor falls through to the flat slot.
### Deployment from this repo
The script is a file in the repo, applied by the existing OpenTofu configuration:
```
edge/rpkgs-router.ts # the script
edge/rpkgs-router.test.ts # UA x path -> expected Location matrix
cdn.tf # bunnynet_compute_script + _variable
```
Provider `BunnyWay/bunnynet` v0.17.0 (already pinned) ships `bunnynet_compute_script` with `content` loadable via `file()`, plus `bunnynet_compute_script_variable`.
`middleware_script = bunnynet_compute_script.rpkgs_router.id` replaces the hard-coded `29277`, after a one-time `tofu import` of the existing script.
## Error handling
- Unknown R minor, or a slot not listed in `UNION_SLOTS`: pass through to the flat slot.
The client sees exactly today's behaviour.
- Unparseable distro on the bare-root form: redirect to CRAN.
- A request already under `…/src/contrib/<x.y>/`: pass through, so a redirect can never loop.
- A per-minor slot that does not exist for a listed minor: the client gets the origin's 404.
`UNION_SLOTS` is the operator's assertion that the slot is ready, so this is a configuration error, not a runtime condition to paper over.
## Testing
Local, before any apply: `deno run -A edge/rpkgs-router.ts` serves the middleware against the real origin, so `edge/rpkgs-router.test.ts` drives the whole matrix against that local server.
- User agent matrix: R 4.4 / 4.5 / 4.6 on musl and gnu, both arches, Posit-style and stock forms, plus a darwin UA and a non-R UA.
- Path matrix: `PACKAGES`, `PACKAGES.gz`, `PACKAGES.rds`, a tarball, a path already under `4.5/`, and `/src/contrib/…` on the bare root.
- Assertion is the `Location` header (or its absence), not the body.
After apply, a smoke test against `cran.rpkgs.com`:
- `available.packages()` inside `reg.devxy.io/r/r-alpine:4.5-3.24` returns the union count, and `"curl" %in% rownames(...)` is `TRUE`.
- A flat-slot package still downloads from `…/src/contrib/`, and a per-minor package downloads from `…/src/contrib/<x.y>/`.
## Out of scope
- `Meta/archive.rds` stays flat-only, so `remotes::install_version()` does not see per-minor archives.
- The `alpine324` source-tarball defect: that slot serves CRAN sources stamped as binaries, and needs a rebuild independent of this work.
- Any change to how `uvr` resolves per-minor URLs; it already addresses the slots directly.
## Split of work
1. `bincraft`: union index writer plus its guard, and a re-index of one slot to validate.
2. This repo: `edge/rpkgs-router.ts`, its test matrix, and the `cdn.tf` resources with `UNION_SLOTS` empty.
3. Enable `UNION_SLOTS` slot by slot as `bincraft` backfills them.