Compare commits
| Author | SHA1 | Date |
|---|---|---|
| 2614d40bd9 |
13 changed files with 2220 additions and 33 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -102,7 +102,3 @@ docs/
|
||||||
|
|
||||||
# Superpowers SDD scratch (briefs, reports, ledger)
|
# Superpowers SDD scratch (briefs, reports, ledger)
|
||||||
.superpowers/
|
.superpowers/
|
||||||
|
|
||||||
# Internal design docs (plans and specs) are kept out of the repo.
|
|
||||||
plans/
|
|
||||||
specs/
|
|
||||||
|
|
|
||||||
|
|
@ -2,3 +2,6 @@
|
||||||
ignores:
|
ignores:
|
||||||
- LICENSE.md
|
- LICENSE.md
|
||||||
- docs/superpowers/**
|
- docs/superpowers/**
|
||||||
|
# Internal design docs (specs/plans) are not user-facing reference material.
|
||||||
|
- specs/**
|
||||||
|
- plans/**
|
||||||
|
|
|
||||||
|
|
@ -73,6 +73,8 @@ Patching uses a two-tier approach:
|
||||||
The system is implemented in `bincraft`: when a package needs patching, `bincraft` pre-builds it with the patch and serves the patched binary to `pak`, ensuring transitive dependents receive the fixed package.
|
The system is implemented in `bincraft`: when a package needs patching, `bincraft` pre-builds it with the patch and serves the patched binary to `pak`, ensuring transitive dependents receive the fixed package.
|
||||||
This way, the fix cascades to all packages that depend on it.
|
This way, the fix cascades to all packages that depend on it.
|
||||||
|
|
||||||
|
For the design rationale and architecture, see `specs/2026-06-30-package-patching-design.md`.
|
||||||
|
|
||||||
## Build Environment
|
## Build Environment
|
||||||
|
|
||||||
Binaries are built on a mixed-architecture Kubernetes cluster using CI.
|
Binaries are built on a mixed-architecture Kubernetes cluster using CI.
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,8 @@ s3fs::s3_file_system(
|
||||||
refresh = T
|
refresh = T
|
||||||
)
|
)
|
||||||
|
|
||||||
arch = "arm64"
|
arch <- "arm64"
|
||||||
os = "alpine321"
|
os <- "alpine321"
|
||||||
|
|
||||||
files <- s3fs::s3_dir_ls(sprintf(
|
files <- s3fs::s3_dir_ls(sprintf(
|
||||||
"devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib",
|
"devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib",
|
||||||
|
|
@ -17,11 +17,11 @@ files <- s3fs::s3_dir_ls(sprintf(
|
||||||
os
|
os
|
||||||
))
|
))
|
||||||
length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1])))
|
length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1])))
|
||||||
non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) {
|
non_archived <- unique(sapply(strsplit(basename(files), "_"), function(x) {
|
||||||
x[1]
|
x[1]
|
||||||
})[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))])
|
})[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))])
|
||||||
# RInno is not built because it only exists for Windows
|
# RInno is not built because it only exists for Windows
|
||||||
non_archived = setdiff(non_archived, "RInno")
|
non_archived <- setdiff(non_archived, "RInno")
|
||||||
# future::plan("sequential", workers = 8)
|
# future::plan("sequential", workers = 8)
|
||||||
future::plan("sequential", workers = 1)
|
future::plan("sequential", workers = 1)
|
||||||
# future.apply::future_lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch))
|
# future.apply::future_lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch))
|
||||||
|
|
|
||||||
|
|
@ -1,22 +1,30 @@
|
||||||
### Scope: Check whether there are any incomplete entries in PACKAGES.rds. These will result in NA when calling available.packages()
|
### Scope: Check whether there are any incomplete entries in PACKAGES.rds. These will result in NA when calling available.packages()
|
||||||
foo = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib")
|
foo <- available.packages(
|
||||||
|
"https://cran.devxy.io/amd64/jammy/latest/src/contrib"
|
||||||
|
)
|
||||||
sum((is.na(foo[, "Version"])))
|
sum((is.na(foo[, "Version"])))
|
||||||
which((is.na(foo[, "Version"])))
|
which((is.na(foo[, "Version"])))
|
||||||
|
|
||||||
foo = available.packages("https://cran.devxy.io/amd64/noble/latest/src/contrib")
|
foo <- available.packages(
|
||||||
|
"https://cran.devxy.io/amd64/noble/latest/src/contrib"
|
||||||
|
)
|
||||||
sum((is.na(foo[, "Version"])))
|
sum((is.na(foo[, "Version"])))
|
||||||
which((is.na(foo[, "Version"])))
|
which((is.na(foo[, "Version"])))
|
||||||
|
|
||||||
|
|
||||||
foo = available.packages("https://cran.devxy.io/amd64/rhel8/latest/src/contrib")
|
foo <- available.packages(
|
||||||
|
"https://cran.devxy.io/amd64/rhel8/latest/src/contrib"
|
||||||
|
)
|
||||||
sum((is.na(foo[, "Version"])))
|
sum((is.na(foo[, "Version"])))
|
||||||
which((is.na(foo[, "Version"])))
|
which((is.na(foo[, "Version"])))
|
||||||
|
|
||||||
foo = available.packages("https://cran.devxy.io/amd64/rhel9/latest/src/contrib")
|
foo <- available.packages(
|
||||||
|
"https://cran.devxy.io/amd64/rhel9/latest/src/contrib"
|
||||||
|
)
|
||||||
sum((is.na(foo[, "Version"])))
|
sum((is.na(foo[, "Version"])))
|
||||||
which((is.na(foo[, "Version"])))
|
which((is.na(foo[, "Version"])))
|
||||||
|
|
||||||
foo = available.packages(
|
foo <- available.packages(
|
||||||
"https://cran.devxy.io/amd64/alpine320/latest/src/contrib"
|
"https://cran.devxy.io/amd64/alpine320/latest/src/contrib"
|
||||||
)
|
)
|
||||||
sum((is.na(foo[, "Version"])))
|
sum((is.na(foo[, "Version"])))
|
||||||
|
|
@ -24,24 +32,32 @@ which((is.na(foo[, "Version"])))
|
||||||
|
|
||||||
# arm64
|
# arm64
|
||||||
|
|
||||||
foo = available.packages("https://cran.devxy.io/arm64/jammy/latest/src/contrib")
|
foo <- available.packages(
|
||||||
|
"https://cran.devxy.io/arm64/jammy/latest/src/contrib"
|
||||||
|
)
|
||||||
sum((is.na(foo[, "Version"])))
|
sum((is.na(foo[, "Version"])))
|
||||||
which((is.na(foo[, "Version"])))
|
which((is.na(foo[, "Version"])))
|
||||||
|
|
||||||
foo = available.packages("https://cran.devxy.io/arm64/noble/latest/src/contrib")
|
foo <- available.packages(
|
||||||
|
"https://cran.devxy.io/arm64/noble/latest/src/contrib"
|
||||||
|
)
|
||||||
sum((is.na(foo[, "Version"])))
|
sum((is.na(foo[, "Version"])))
|
||||||
which((is.na(foo[, "Version"])))
|
which((is.na(foo[, "Version"])))
|
||||||
|
|
||||||
|
|
||||||
foo = available.packages("https://cran.devxy.io/arm64/rhel8/latest/src/contrib")
|
foo <- available.packages(
|
||||||
|
"https://cran.devxy.io/arm64/rhel8/latest/src/contrib"
|
||||||
|
)
|
||||||
sum((is.na(foo[, "Version"])))
|
sum((is.na(foo[, "Version"])))
|
||||||
which((is.na(foo[, "Version"])))
|
which((is.na(foo[, "Version"])))
|
||||||
|
|
||||||
foo = available.packages("https://cran.devxy.io/arm64/rhel9/latest/src/contrib")
|
foo <- available.packages(
|
||||||
|
"https://cran.devxy.io/arm64/rhel9/latest/src/contrib"
|
||||||
|
)
|
||||||
sum((is.na(foo[, "Version"])))
|
sum((is.na(foo[, "Version"])))
|
||||||
which((is.na(foo[, "Version"])))
|
which((is.na(foo[, "Version"])))
|
||||||
|
|
||||||
foo = available.packages(
|
foo <- available.packages(
|
||||||
"https://cran.devxy.io/arm64/alpine320/latest/src/contrib"
|
"https://cran.devxy.io/arm64/alpine320/latest/src/contrib"
|
||||||
)
|
)
|
||||||
sum((is.na(foo[, "Version"])))
|
sum((is.na(foo[, "Version"])))
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
library(future)
|
library(future)
|
||||||
future::plan(multisession)
|
future::plan(multisession)
|
||||||
future::plan(sequential)
|
future::plan(sequential)
|
||||||
time = Sys.time()
|
time <- Sys.time()
|
||||||
files <- s3fs::s3_dir_ls(
|
files <- s3fs::s3_dir_ls(
|
||||||
sprintf(
|
sprintf(
|
||||||
"devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive"
|
"devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive"
|
||||||
|
|
@ -54,8 +54,8 @@ curl::curl_download(
|
||||||
"https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db",
|
"https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db",
|
||||||
"PACKAGES.db"
|
"PACKAGES.db"
|
||||||
)
|
)
|
||||||
con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db")
|
con <- DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db")
|
||||||
df = DBI::dbReadTable(con, "packages")
|
df <- DBI::dbReadTable(con, "packages")
|
||||||
|
|
||||||
cranlike:::write_packages_files("/tmp", "PACKAGES.db")
|
cranlike:::write_packages_files("/tmp", "PACKAGES.db")
|
||||||
|
|
||||||
|
|
@ -69,11 +69,11 @@ system("cat /tmp/PACKAGES | grep '^Package: digest' | wc -l")
|
||||||
curl::curl_fetch_memory(
|
curl::curl_fetch_memory(
|
||||||
"https://cloud.r-project.org/src/contrib/Meta/archive.rds"
|
"https://cloud.r-project.org/src/contrib/Meta/archive.rds"
|
||||||
)
|
)
|
||||||
remotes = readRDS(url(
|
remotes <- readRDS(url(
|
||||||
"https://cloud.r-project.org/src/contrib/Meta/archive.rds",
|
"https://cloud.r-project.org/src/contrib/Meta/archive.rds",
|
||||||
"rb"
|
"rb"
|
||||||
))
|
))
|
||||||
packages = readRDS(url(
|
packages <- readRDS(url(
|
||||||
"https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds",
|
"https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds",
|
||||||
"rb"
|
"rb"
|
||||||
))
|
))
|
||||||
|
|
|
||||||
|
|
@ -42,14 +42,14 @@ files <- s3fs::s3_dir_ls(sprintf(
|
||||||
"devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"
|
"devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"
|
||||||
))
|
))
|
||||||
|
|
||||||
files_b = basename(files)
|
files_b <- basename(files)
|
||||||
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
files_b_split <- sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
||||||
|
|
||||||
cran_pkgs <- tools::CRAN_package_db() |>
|
cran_pkgs <- tools::CRAN_package_db() |>
|
||||||
filter(`Date/Publication` <= "2025-01-25") |>
|
filter(`Date/Publication` <= "2025-01-25") |>
|
||||||
pull(Package)
|
pull(Package)
|
||||||
|
|
||||||
pkgs = setdiff(cran_pkgs, files_b_split)
|
pkgs <- setdiff(cran_pkgs, files_b_split)
|
||||||
formatted_pkgs <- gsub(
|
formatted_pkgs <- gsub(
|
||||||
'"',
|
'"',
|
||||||
"'",
|
"'",
|
||||||
|
|
|
||||||
|
|
@ -39,14 +39,14 @@ files <- s3fs::s3_dir_ls(sprintf(
|
||||||
"devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"
|
"devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"
|
||||||
))
|
))
|
||||||
|
|
||||||
files_b = basename(files)
|
files_b <- basename(files)
|
||||||
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
files_b_split <- sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
||||||
files_b_split = setdiff(
|
files_b_split <- setdiff(
|
||||||
files_b_split,
|
files_b_split,
|
||||||
c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive")
|
c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive")
|
||||||
)
|
)
|
||||||
# windows-only
|
# windows-only
|
||||||
files_b_split = setdiff(
|
files_b_split <- setdiff(
|
||||||
files_b_split,
|
files_b_split,
|
||||||
c(
|
c(
|
||||||
"PACKAGES",
|
"PACKAGES",
|
||||||
|
|
@ -67,12 +67,12 @@ files_b_split = setdiff(
|
||||||
"R2wd"
|
"R2wd"
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
pkgs_index = available.packages(
|
pkgs_index <- available.packages(
|
||||||
"https://cran.devxy.io/amd64/jammy/latest/src/contrib"
|
"https://cran.devxy.io/amd64/jammy/latest/src/contrib"
|
||||||
)[, "Package"]
|
)[, "Package"]
|
||||||
pkgs_index_b = basename(pkgs_index)
|
pkgs_index_b <- basename(pkgs_index)
|
||||||
if (length(files_b_split) != length(pkgs_index_b)) {
|
if (length(files_b_split) != length(pkgs_index_b)) {
|
||||||
pkgs_missing = setdiff(files_b_split, pkgs_index_b)
|
pkgs_missing <- setdiff(files_b_split, pkgs_index_b)
|
||||||
message("Packages missing in index but present in S3:")
|
message("Packages missing in index but present in S3:")
|
||||||
pkgs_missing
|
pkgs_missing
|
||||||
formatted_pkgs <- gsub(
|
formatted_pkgs <- gsub(
|
||||||
|
|
|
||||||
1506
plans/2026-06-30-package-patching-implementation.md
Normal file
1506
plans/2026-06-30-package-patching-implementation.md
Normal file
File diff suppressed because it is too large
Load diff
166
plans/2026-08-07-per-minor-edge-routing.md
Normal file
166
plans/2026-08-07-per-minor-edge-routing.md
Normal file
|
|
@ -0,0 +1,166 @@
|
||||||
|
# Per-R-minor edge routing implementation plan
|
||||||
|
|
||||||
|
Spec: `specs/2026-08-07-per-minor-edge-routing-design.md`
|
||||||
|
|
||||||
|
**Goal:** let a stock `install.packages()` see the per-minor packages by routing `PACKAGES*` requests to `…/src/contrib/<x.y>/`, where `bincraft` publishes a union index.
|
||||||
|
|
||||||
|
**Architecture:** the union is built in `bincraft`; the edge script only redirects index requests, gated on a `UNION_SLOTS` script variable; the script lives in this repo and is applied by OpenTofu.
|
||||||
|
|
||||||
|
**Tech stack:** Deno / TypeScript (Bunny Edge Scripting, SDK 0.12), OpenTofu with `BunnyWay/bunnynet` 0.17, R (bincraft).
|
||||||
|
|
||||||
|
## Global constraints
|
||||||
|
|
||||||
|
- Redirect only `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds`; never a tarball, because the union index already carries the correct tarball URL for both classes of package.
|
||||||
|
- Every redirect carries `Cache-Control: no-store`; redirect targets stay UA-independent.
|
||||||
|
- `UNION_SLOTS` is empty by default, so deploying the script is a no-op until a slot is backfilled.
|
||||||
|
- A slot is `<arch>/<os>`, e.g. `amd64/alpine324`.
|
||||||
|
- Verified prerequisites: `PACKAGES*` is served `cdn-cache: BYPASS`, so the script sees every index request; `Deno.env.get()` reads script variables; the SDK local server listens on `127.0.0.1:8080`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 1: Edge script and its test matrix
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
|
||||||
|
- Create: `edge/rpkgs-router.ts`
|
||||||
|
- Create: `edge/rpkgs-router.test.ts`
|
||||||
|
- Modify: `justfile` (add `edge-test`)
|
||||||
|
|
||||||
|
**Produces:** a single-file script deployable as `bunnynet_compute_script.content`, reading `UNION_SLOTS` from the environment.
|
||||||
|
|
||||||
|
- [ ] **Step 1: write the test matrix first**
|
||||||
|
|
||||||
|
`edge/rpkgs-router.test.ts` spawns `deno run -A edge/rpkgs-router.ts` with `UNION_SLOTS=amd64/alpine324`, waits for `127.0.0.1:8080`, and issues requests with `redirect: "manual"`.
|
||||||
|
|
||||||
|
Cases, asserted on the `location` header (or its absence):
|
||||||
|
|
||||||
|
| # | path | User-Agent | expectation |
|
||||||
|
| --- | ------------------------------------------------------ | --------------------------------------------- | ------------------------------------------------------------ |
|
||||||
|
| 1 | `/amd64/alpine324/latest/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)` | 302 → `…/src/contrib/4.5/PACKAGES.gz` |
|
||||||
|
| 2 | same | `R (4.6.0 …)` | 302 → `…/src/contrib/4.6/PACKAGES.gz` |
|
||||||
|
| 3 | same, but slot `amd64/noble` | `R (4.5.3 …)` | no redirect (slot not in `UNION_SLOTS`) |
|
||||||
|
| 4 | `…/src/contrib/curl_7.1.0.tar.gz` | `R (4.5.3 …)` | no redirect |
|
||||||
|
| 5 | `…/src/contrib/4.5/PACKAGES.gz` | `R (4.5.3 …)` | no redirect (loop guard) |
|
||||||
|
| 6 | `…/src/contrib/PACKAGES.gz` | `curl/8.0` | no redirect (no R minor) |
|
||||||
|
| 7 | `/src/contrib/PACKAGES.gz` | alpine UA with `Alpine Linux … 3.24` | 302 → `/amd64/alpine324/latest/src/contrib/4.5/PACKAGES.gz` |
|
||||||
|
| 8 | `/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)`, no distro | 302 → `cran.r-project.org`, **not** a `linux-musl` slot |
|
||||||
|
| 9 | `/src/contrib/foo_1.0.tar.gz` | `R (4.5.1 aarch64-apple-darwin20 …)` | 302 → `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` |
|
||||||
|
| 10 | `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | any | 302 → `cran.r-project.org` |
|
||||||
|
| 11 | any redirect above | — | `cache-control: no-store` |
|
||||||
|
|
||||||
|
- [ ] **Step 2: run the tests and watch them fail**
|
||||||
|
|
||||||
|
`just edge-test` → every case fails, because `edge/rpkgs-router.ts` does not exist.
|
||||||
|
|
||||||
|
- [ ] **Step 3: write `edge/rpkgs-router.ts`**
|
||||||
|
|
||||||
|
Order of evaluation in `onOriginRequest`:
|
||||||
|
|
||||||
|
1. normalise `//` runs in the path
|
||||||
|
2. darwin `/src/contrib/*` → `/bin/macosx/<flavour>/contrib/<x.y>/`
|
||||||
|
3. `/bin/macosx/**` → CRAN
|
||||||
|
4. `/{arch}/{os}/latest/src/contrib/<rest>`: pass through if `rest` already starts with `<x.y>/`, or is not an index file, or the slot is not in `UNION_SLOTS`, or the UA has no R minor; otherwise redirect into `<x.y>/`
|
||||||
|
5. `/`, `/src/contrib`, `/src/contrib/**`: resolve arch+os from the UA, redirect to CRAN when the distro is unidentifiable, otherwise redirect to the qualified path, adding `<x.y>/` under the same index-file rule
|
||||||
|
6. anything else: pass through
|
||||||
|
|
||||||
|
The R minor comes from either `R/4.5.3` or `R (4.5.3 …)`, so a stock UA is enough. The `linux-gnu` / `linux-musl` fallback in `parseUserAgent` is deleted: those are not slot names.
|
||||||
|
|
||||||
|
- [ ] **Step 4: run the tests until they pass**
|
||||||
|
|
||||||
|
`just edge-test`
|
||||||
|
|
||||||
|
- [ ] **Step 5: commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add edge/rpkgs-router.ts edge/rpkgs-router.test.ts justfile
|
||||||
|
git commit -m "feat(edge): route PACKAGES requests to the per-R-minor slot"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 2: Manage the script from OpenTofu
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
|
||||||
|
- Modify: `cdn.tf`
|
||||||
|
|
||||||
|
**Consumes:** `edge/rpkgs-router.ts` from Task 1.
|
||||||
|
|
||||||
|
- [ ] **Step 1: add the resources**
|
||||||
|
|
||||||
|
```terraform
|
||||||
|
resource "bunnynet_compute_script" "rpkgs_router" {
|
||||||
|
type = "middleware"
|
||||||
|
name = "rpkgs-router"
|
||||||
|
content = file("${path.module}/edge/rpkgs-router.ts")
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" {
|
||||||
|
script = bunnynet_compute_script.rpkgs_router.id
|
||||||
|
name = "UNION_SLOTS"
|
||||||
|
default_value = ""
|
||||||
|
required = false
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
and replace `middleware_script = 29277` with `middleware_script = bunnynet_compute_script.rpkgs_router.id`.
|
||||||
|
|
||||||
|
- [ ] **Step 2: validate**
|
||||||
|
|
||||||
|
`tofu init -backend=false && tofu validate`
|
||||||
|
|
||||||
|
- [ ] **Step 3: import the existing script (needs `BUNNYNET_API_KEY`)**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
tofu import bunnynet_compute_script.rpkgs_router 29277
|
||||||
|
tofu plan
|
||||||
|
```
|
||||||
|
|
||||||
|
The plan must show an in-place `content` update and **no** replacement of the pull zone. A replacement means the import did not take.
|
||||||
|
|
||||||
|
- [ ] **Step 4: commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add cdn.tf
|
||||||
|
git commit -m "feat(cdn): manage the edge middleware script from this repo"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 3: Union index writer in bincraft
|
||||||
|
|
||||||
|
**Files (repo `codefloe.com/rpkgs/bincraft`):**
|
||||||
|
|
||||||
|
- Modify: `R/package_index.R`
|
||||||
|
- Test: `tests/testthat/test-package_index.R`
|
||||||
|
|
||||||
|
**Produces:** `write_union_index(flat_records, minor_records)` returning the merged records, called from `upload_package_index()` when `r_minor` is set.
|
||||||
|
|
||||||
|
- [ ] **Step 1: write the failing tests**
|
||||||
|
|
||||||
|
- a package present in both slots keeps the per-minor record, with `Path = "4.5"`
|
||||||
|
- a package only in the flat slot survives with no `Path`
|
||||||
|
- a package only in the per-minor slot survives with `Path = "4.5"`
|
||||||
|
- a union smaller than the flat input raises an error rather than returning
|
||||||
|
|
||||||
|
- [ ] **Step 2: run them and watch them fail**
|
||||||
|
|
||||||
|
`Rscript -e 'testthat::test_file("tests/testthat/test-package_index.R")'`
|
||||||
|
|
||||||
|
- [ ] **Step 3: implement `write_union_index()` and call it from `upload_package_index()`**
|
||||||
|
|
||||||
|
After `update_PACKAGES()` has written the per-minor index, read the flat slot's `PACKAGES.rds`, set `Path = <r_minor>` on the per-minor records, drop the flat records for packages the per-minor slot already has, and rewrite `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` in the per-minor slot.
|
||||||
|
|
||||||
|
- [ ] **Step 4: run the tests until they pass**
|
||||||
|
|
||||||
|
- [ ] **Step 5: commit and open the PR against bincraft**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 4: Roll out slot by slot
|
||||||
|
|
||||||
|
- [ ] Re-index one slot (`amd64/alpine324`, R 4.5) and confirm the union index lists both `curl` (per-minor, `Path: 4.5`) and `jsonlite` (flat, no `Path`).
|
||||||
|
- [ ] Set `UNION_SLOTS = "amd64/alpine324"` and confirm in `reg.devxy.io/r/r-alpine:4.5-3.24` that `available.packages()` returns the union count and `"curl" %in% rownames(...)`.
|
||||||
|
- [ ] Add `arm64/alpine324`, then the remaining slots.
|
||||||
|
|
||||||
|
`install.packages("curl")` will still fail to build on `alpine324` until that slot's source tarballs are replaced with real binaries. That is tracked separately.
|
||||||
157
specs/2026-06-30-package-patching-design.md
Normal file
157
specs/2026-06-30-package-patching-design.md
Normal file
|
|
@ -0,0 +1,157 @@
|
||||||
|
# Design: Dynamic per-package patching during binary builds
|
||||||
|
|
||||||
|
Date: 2026-06-30
|
||||||
|
Status: Approved (pending spec review)
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
Some CRAN packages fail to compile on specific build platforms due to compiler- or OS-specific issues that have nothing to do with the package being built.
|
||||||
|
The canonical example is `RcppParallel`: its bundled Intel TBB sources fail to build on musl (Alpine) and on newer OS/compiler combinations.
|
||||||
|
Observed failure on `ubuntu-2604` ("resolute") with `g++ 15.2.0`:
|
||||||
|
|
||||||
|
```
|
||||||
|
../build/common.inc:74: *** "" is not supported. Add build/.inc file with os-specific settings . Stop.
|
||||||
|
make: *** [Makevars:163: tbb] Error 2
|
||||||
|
ERROR: compilation failed for package 'RcppParallel'
|
||||||
|
```
|
||||||
|
|
||||||
|
Because `RcppParallel` is a dependency of many packages, a single such failure cascades: every dependent package (e.g. `rts2`) also fails, even though nothing is wrong with the dependent itself.
|
||||||
|
|
||||||
|
Today there is no way to intervene.
|
||||||
|
A package can only be **excluded** (`local/excluded-packages.json`), which is all-or-nothing and does not help dependents.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Allow a curated set of packages to be "patched" — via lightweight build-time overrides or, when necessary, real source diffs — **before** they are installed, whether the package is a direct build target or a transitive dependency pulled in by `pak`.
|
||||||
|
|
||||||
|
## Key constraint that drives the design
|
||||||
|
|
||||||
|
When `RcppParallel` fails here, it is being installed as a **transitive dependency** by `pak`, inside `bincraft::build_binary_package()`.
|
||||||
|
`pak` downloads, configures, and compiles it in one subprocess; this repo never touches that source.
|
||||||
|
For a fix to reach a dependency-of-a-dependency, the fixed package must be visible to `pak` itself, where `pak`'s repositories/sources are configured — which is inside `bincraft`.
|
||||||
|
|
||||||
|
Decisions taken during brainstorming:
|
||||||
|
|
||||||
|
- **Mechanism lives in `bincraft`** (the engine), because only there can transitive deps be influenced.
|
||||||
|
- **Patch tiers: both, env-overrides first.** Support cheap per-package env vars / configure args / Makevars (version-independent) _and_ true source diffs (version-pinned), preferring the lightweight override.
|
||||||
|
- **Registry data lives in this repo** (`build-cran-binaries`) and is passed into `bincraft`, keeping `bincraft` as pure mechanism and the frequently-changing policy data with operational config.
|
||||||
|
|
||||||
|
## Approaches considered
|
||||||
|
|
||||||
|
| Approach | How pak sees the fix | Verdict |
|
||||||
|
| ---------------------------------------------------------------------------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| A. Patched **source** repo — drop patched `.tar.gz` source into a local repo, prepend it | pak recompiles from your source | Simple, but env-tier overrides leak globally (one subprocess builds everything) and the dep recompiles on every dependent build |
|
||||||
|
| **B. Pre-built patched binary repo (chosen)** | pak installs a ready binary by repo priority | Per-package scoping is free; no recompile; the binary is a cacheable/uploadable artifact that fits the existing system |
|
||||||
|
| C. pkgdepends per-build hook | intercept each build | No clean per-package pre-compile hook exists; fragile |
|
||||||
|
|
||||||
|
Chosen: **B**.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
### Registry (this repo)
|
||||||
|
|
||||||
|
```
|
||||||
|
local/patches/
|
||||||
|
registry.json # the manifest
|
||||||
|
RcppParallel/
|
||||||
|
fix.patch # optional source diff, referenced by an entry
|
||||||
|
```
|
||||||
|
|
||||||
|
`registry.json` is an array of entries:
|
||||||
|
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"package": "RcppParallel",
|
||||||
|
"versions": "*",
|
||||||
|
"platforms": ["alpine", "ubuntu-2604"],
|
||||||
|
"env": { "RCPP_PARALLEL_USE_TBB": "0" },
|
||||||
|
"configure_args": [],
|
||||||
|
"makevars": {},
|
||||||
|
"patch": null,
|
||||||
|
"reason": "bundled TBB fails to build on musl / newer compilers"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
Field semantics:
|
||||||
|
|
||||||
|
- `package` (string, required): CRAN package name.
|
||||||
|
- `versions` (string, required): `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`.
|
||||||
|
Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against.
|
||||||
|
- `platforms` (array of strings, required): matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`).
|
||||||
|
An entry matches if any listed token matches any build token.
|
||||||
|
`["*"]` matches all platforms.
|
||||||
|
- `env` (object, optional): environment variables exported only for this package's isolated build.
|
||||||
|
- `configure_args` (array, optional): passed as `--configure-args` to the isolated build.
|
||||||
|
- `makevars` (object, optional): key/value pairs written into a package-local Makevars for the isolated build.
|
||||||
|
- `patch` (string or null, optional): path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building.
|
||||||
|
- `reason` (string, required): human explanation, surfaced in logs and metadata.
|
||||||
|
|
||||||
|
A fix is any combination of `env`, `configure_args`, `makevars`, and `patch`.
|
||||||
|
"Env-first" is an authoring guideline (prefer the lightweight override) and an ordering of effort, not a runtime branch — all present fields are applied together for the isolated build.
|
||||||
|
|
||||||
|
### Flow (inside bincraft, around existing pak resolution)
|
||||||
|
|
||||||
|
1. **Resolve** the dependency set (dry-run) to learn the concrete versions `pak` will install.
|
||||||
|
Reuse bincraft's existing resolution where possible (e.g. a `pkgdepends` proposal: `$resolve()` → inspect resolution → ... → `$solve()` / `$install()` after the local repo is prepended).
|
||||||
|
2. For each resolved package that matches a registry entry (name + `versions` + `platforms`): obtain a **patched binary** for the exact `version × platform × arch × R-minor`:
|
||||||
|
- **Cache hit** (local `/mnt/cache/patched-binaries/` or S3): fetch it into the local repo.
|
||||||
|
- **Cache miss**: download the CRAN **source** for that version, apply the source `patch` (if any) to the unpacked tree, build the binary in isolation with `env` / `configure_args` / `makevars` applied, then place the binary in the local repo and write it to the cache (and S3 if uploading is enabled).
|
||||||
|
3. **Prepend** the local binary repo (`file://…`) to `pak`'s repo list, and regenerate its `PACKAGES` index.
|
||||||
|
4. Run the **normal install**.
|
||||||
|
`pak` resolves the patched binary for the matched package — direct or transitive — because it wins on repo priority for an equal version, and installs it without recompiling.
|
||||||
|
|
||||||
|
### Caching (essential)
|
||||||
|
|
||||||
|
`RcppParallel` is a dependency of dozens of packages; without caching the fix would be rebuilt on every dependent build.
|
||||||
|
Patched binaries are keyed by:
|
||||||
|
|
||||||
|
```
|
||||||
|
<package>_<version>_<platform>_<arch>_<rminor>_<patchhash>
|
||||||
|
```
|
||||||
|
|
||||||
|
`patchhash` is a hash of the normalized registry entry plus the referenced diff file contents.
|
||||||
|
Editing a patch therefore changes the hash and auto-invalidates stale cached binaries.
|
||||||
|
|
||||||
|
- Local cache: `/mnt/cache/patched-binaries/`.
|
||||||
|
- Optional S3 cache for cross-build reuse: a dedicated `…/patched/` slot under the existing arch/codename structure, mirroring how normal binaries are stored.
|
||||||
|
|
||||||
|
### S3 upload
|
||||||
|
|
||||||
|
Patched binaries **are** uploaded to S3 (in addition to the local cache) so they are reused across CI jobs and machines, not just within one container.
|
||||||
|
They live in a separate `patched/` slot and are not published into the user-facing `src/contrib` index — they are an internal build accelerator, not a distributed artifact.
|
||||||
|
|
||||||
|
## Error handling
|
||||||
|
|
||||||
|
- **Source diff fails to apply** (CRAN moved past the pinned version): log a clear warning, skip that entry, and proceed.
|
||||||
|
The package builds unpatched (status quo) and may fail.
|
||||||
|
The skipped/failed-to-apply patch is surfaced in build metadata.
|
||||||
|
- **Pre-build of the patched binary fails**: log a warning, skip, proceed.
|
||||||
|
- **No version or platform match**: skip silently (the entry simply does not apply to this build).
|
||||||
|
- **Overlapping entries for one package**: the most specific entry wins (a concrete `platforms`/`versions` beats `"*"`).
|
||||||
|
Genuine ambiguity (two equally specific, conflicting entries) is a validation error reported before the build.
|
||||||
|
|
||||||
|
## Observability
|
||||||
|
|
||||||
|
- One log line per applied patch, e.g.: `Applying patch to RcppParallel 5.1.11-2 [env: RCPP_PARALLEL_USE_TBB=0]: bundled TBB fails on musl / newer compilers`
|
||||||
|
- The set of applied patches (package, version, `patchhash`) is recorded in the Postgres build-metadata row for the build, so it is queryable later.
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
- **Unit (registry):** parsing and matching — version constraints, platform token matching, precedence/specificity, and detection of ambiguous overlaps.
|
||||||
|
- **Unit (cache key):** `patchhash` changes when the entry or diff changes; is stable otherwise.
|
||||||
|
- **Integration:** `RcppParallel` on `resolute` (and/or Alpine) fails to build without a registry entry and succeeds with one; a dependent package such as `rts2` succeeds once the dependency is patched.
|
||||||
|
- **Failure path:** an entry pinned to an old version against a newer CRAN release → graceful skip with a warning, build continues.
|
||||||
|
|
||||||
|
## Out of scope
|
||||||
|
|
||||||
|
- Shipping a default registry inside `bincraft` (registry is repo-local for now; a baseline-in-engine + repo-override model can come later if needed).
|
||||||
|
- Publishing patched binaries into the public `src/contrib` index.
|
||||||
|
- Automatic detection of which packages need patches — entries are curated by hand.
|
||||||
|
|
||||||
|
## Split of work
|
||||||
|
|
||||||
|
- **bincraft:** the mechanism — registry ingestion, resolution hook, isolated patched-binary build, caching/upload, local-repo prepend, logging, metadata recording.
|
||||||
|
A new `patches` argument on `build_binary_package()`.
|
||||||
|
- **build-cran-binaries (this repo):** the `local/patches/` registry and diffs, passing `patches = "local/patches"` through `build-one.R` / `build-all.R`, and documentation.
|
||||||
174
specs/2026-08-07-per-minor-edge-routing-design.md
Normal file
174
specs/2026-08-07-per-minor-edge-routing-design.md
Normal file
|
|
@ -0,0 +1,174 @@
|
||||||
|
# Design: Routing clients to per-R-minor binary slots
|
||||||
|
|
||||||
|
Date: 2026-08-07
|
||||||
|
Status: Approved (pending spec review)
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
`bincraft` routes ABI-"risky" packages to a per-minor slot `…/latest/src/contrib/<x.y>/` and indexes every directory independently (`upload_package_index()` calls `cranlike::update_PACKAGES()` on one prefix at a time).
|
||||||
|
Nothing unions those indices, and `contrib.url()` only ever yields `<repos>/src/contrib`, so no value of `options(repos)` can address a per-minor slot.
|
||||||
|
Only `uvr` resolves per-minor URLs, which means the per-minor slots are invisible to `install.packages()` by construction.
|
||||||
|
|
||||||
|
Measured on 2026-08-07:
|
||||||
|
|
||||||
|
| slot | flat `src/contrib` | `src/contrib/4.5` | unique packages only in the per-minor slot |
|
||||||
|
| ----------------- | ------------------ | ----------------- | ------------------------------------------ |
|
||||||
|
| `amd64/alpine324` | 21 640 | 3 310 | 2 886 |
|
||||||
|
| `amd64/noble` | 24 495 | 398 | 23 |
|
||||||
|
|
||||||
|
This is what issue #63 records as "missing binaries" on `alpine324`.
|
||||||
|
The packages are not missing; they are in a directory base R cannot reach.
|
||||||
|
The user-visible symptom in `reg.devxy.io/r/r-alpine:4.5-3.24` is:
|
||||||
|
|
||||||
|
```
|
||||||
|
> install.packages("curl")
|
||||||
|
Warning message:
|
||||||
|
package 'curl' is not available for this version of R
|
||||||
|
```
|
||||||
|
|
||||||
|
A second, unrelated defect exists on the same slot and is **out of scope here**: many `alpine324` tarballs are byte-identical CRAN _source_ tarballs that the index nevertheless stamps `Built: R 4.5.3; …-linux-musl`.
|
||||||
|
Routing exposes `curl`; only a rebuild of that slot makes it install.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Let a stock `install.packages()` see one complete package list for its own R minor, without duplicating tarballs and without an R-version-varying cache key anywhere in the CDN.
|
||||||
|
|
||||||
|
## Key constraint that drives the design
|
||||||
|
|
||||||
|
R resolves a package's download URL from the index, not from the request path, and it keeps the `contriburl` it _asked for_ rather than the one it was redirected to.
|
||||||
|
Measured with `options(repos = …/latest)` against a middleware that redirects the index into `4.5/`:
|
||||||
|
|
||||||
|
```
|
||||||
|
curl available: TRUE
|
||||||
|
curl repo: …/latest/src/contrib # the flat URL, not the 4.5 one it was served from
|
||||||
|
```
|
||||||
|
|
||||||
|
So the union index is always addressed relative to the **flat** directory, whatever path it was fetched from.
|
||||||
|
`available.packages()` honours a `Path:` field and folds it into the `Repository` column, which gives the whole routing for free:
|
||||||
|
|
||||||
|
- a per-minor record carries `Path: <x.y>`, so its tarball is fetched from `…/src/contrib/<x.y>/`
|
||||||
|
- a flat record carries no `Path`, so its tarball is fetched from `…/src/contrib/`
|
||||||
|
|
||||||
|
Verified end to end against the live CDN with a locally built union index for `amd64/alpine324` (31 507 records):
|
||||||
|
|
||||||
|
```
|
||||||
|
curl: 7.1.0 -> …/latest/src/contrib/4.5 -> curl_7.1.0.tar.gz 717 725 B
|
||||||
|
jsonlite: 2.0.0 -> …/latest/src/contrib -> jsonlite_2.0.0.tar.gz 1 055 849 B
|
||||||
|
```
|
||||||
|
|
||||||
|
The corollary is that the edge script must **not** rewrite tarball requests: every tarball URL is already correct when it leaves the client, and redirecting one into `<x.y>/` would break exactly the flat packages the union is meant to preserve.
|
||||||
|
|
||||||
|
The complementary trick does not work: R's `gzcon()` reads only the first member of a concatenated gzip stream (10 291 of an expected 31 931 records), so an edge-side merge would have to fully decompress and recompress both indices and additionally 404 `PACKAGES.rds` to stop R preferring it.
|
||||||
|
That is why the union is produced in `bincraft`, not at the edge.
|
||||||
|
|
||||||
|
## Approaches considered
|
||||||
|
|
||||||
|
| Approach | Where the union lives | Verdict |
|
||||||
|
| ----------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- |
|
||||||
|
| **A. Union index written by `bincraft` (chosen)** | per-minor `PACKAGES*`, per-minor entries carry `Path: <x.y>` | Edge does one redirect; `PACKAGES.rds` stays correct; no duplication |
|
||||||
|
| B. Merge at the edge | middleware fetches both indices, recompresses | ~2 MB decompress/recompress per cache fill, cache key must include the R minor, breaks R's `.rds` fast path |
|
||||||
|
| C. Move the minor up the path (`latest/<x.y>/src/contrib/`) | addressable by `options(repos)` directly | No edge logic at all, but a full layout migration and breaks the published URL contract |
|
||||||
|
|
||||||
|
Chosen: **A**.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
### bincraft: union index (separate PR)
|
||||||
|
|
||||||
|
After writing a per-minor index, republish it as a union of that slot and the flat slot:
|
||||||
|
|
||||||
|
1. Read the flat slot's `PACKAGES.rds` and the per-minor slot's own records.
|
||||||
|
2. Set `Path: <x.y>` on every per-minor record, so its tarball resolves into the per-minor directory.
|
||||||
|
3. Drop every flat record whose package is already present in the per-minor slot, so the per-minor build always wins, and leave the survivors without a `Path`.
|
||||||
|
4. Write the merged `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib/<x.y>/`.
|
||||||
|
|
||||||
|
Guard: refuse to publish a union with fewer records than the flat index it was built from.
|
||||||
|
A truncated union is worse than no union, because it silently removes packages from every client on that minor.
|
||||||
|
|
||||||
|
### Edge script (this repo)
|
||||||
|
|
||||||
|
The script routes `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` requests, and nothing else.
|
||||||
|
|
||||||
|
```
|
||||||
|
normalize path
|
||||||
|
parseClient(UA) -> { rMinor, arch, os } # rMinor from "R (4.5.3 …)" or "R/4.5.3"
|
||||||
|
darwin branches # unchanged
|
||||||
|
if path is /{arch}/{os}/latest/src/contrib/PACKAGES*
|
||||||
|
already under /<x.y>/ ? pass through # loop guard
|
||||||
|
rMinor known && slot in UNION_SLOTS ? 302 -> …/src/contrib/<rMinor>/PACKAGES*
|
||||||
|
else pass through # flat slot, today's behaviour
|
||||||
|
if path is /src/contrib/… # bare root
|
||||||
|
resolve arch + os; unknown -> 302 to CRAN
|
||||||
|
then apply the same PACKAGES* rule
|
||||||
|
else pass through
|
||||||
|
```
|
||||||
|
|
||||||
|
Redirects carry `Cache-Control: no-store`.
|
||||||
|
Every cacheable URL is therefore UA-independent, and no cache key has to vary by R version.
|
||||||
|
|
||||||
|
### Repaired bare-root detection
|
||||||
|
|
||||||
|
The bare `https://cran.rpkgs.com` form is currently broken for every Linux client that uses a stock R user agent.
|
||||||
|
`ALPINE_REGEX`, `UBUNTU_REGEX` and `RHEL_REGEX` only match a Posit-style user agent the user has to set by hand; stock R never carries the distro, so the script falls through to `extractOs()` and redirects to a slot that does not exist:
|
||||||
|
|
||||||
|
```
|
||||||
|
UA: R (4.5.3 x86_64-pc-linux-musl …) -> 302 /amd64/linux-musl/latest/… (404)
|
||||||
|
UA: R (4.5.3 x86_64-pc-linux-gnu …) -> 302 /amd64/linux-gnu/latest/… (404)
|
||||||
|
```
|
||||||
|
|
||||||
|
The fallback to a phantom `linux-musl` / `linux-gnu` slot is removed.
|
||||||
|
An unidentifiable distro redirects to CRAN, which is the existing behaviour for an unparseable user agent.
|
||||||
|
The R _minor_ is always present in a stock user agent, so per-minor routing itself does not depend on distro detection.
|
||||||
|
|
||||||
|
### Rollout gate
|
||||||
|
|
||||||
|
`UNION_SLOTS` is a `bunnynet_compute_script_variable` listing the slots whose per-minor index is already a union.
|
||||||
|
It is empty by default, so deploying the script changes nothing until `bincraft` has backfilled a slot, and a rollback is a variable edit rather than a code deploy.
|
||||||
|
All slots currently carry `4.4`, `4.5` and `4.6`; a client on any other minor falls through to the flat slot.
|
||||||
|
|
||||||
|
### Deployment from this repo
|
||||||
|
|
||||||
|
The script is a file in the repo, applied by the existing OpenTofu configuration:
|
||||||
|
|
||||||
|
```
|
||||||
|
edge/rpkgs-router.ts # the script
|
||||||
|
edge/rpkgs-router.test.ts # UA x path -> expected Location matrix
|
||||||
|
cdn.tf # bunnynet_compute_script + _variable
|
||||||
|
```
|
||||||
|
|
||||||
|
Provider `BunnyWay/bunnynet` v0.17.0 (already pinned) ships `bunnynet_compute_script` with `content` loadable via `file()`, plus `bunnynet_compute_script_variable`.
|
||||||
|
`middleware_script = bunnynet_compute_script.rpkgs_router.id` replaces the hard-coded `29277`, after a one-time `tofu import` of the existing script.
|
||||||
|
|
||||||
|
## Error handling
|
||||||
|
|
||||||
|
- Unknown R minor, or a slot not listed in `UNION_SLOTS`: pass through to the flat slot.
|
||||||
|
The client sees exactly today's behaviour.
|
||||||
|
- Unparseable distro on the bare-root form: redirect to CRAN.
|
||||||
|
- A request already under `…/src/contrib/<x.y>/`: pass through, so a redirect can never loop.
|
||||||
|
- A per-minor slot that does not exist for a listed minor: the client gets the origin's 404.
|
||||||
|
`UNION_SLOTS` is the operator's assertion that the slot is ready, so this is a configuration error, not a runtime condition to paper over.
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
Local, before any apply: `deno run -A edge/rpkgs-router.ts` serves the middleware against the real origin, so `edge/rpkgs-router.test.ts` drives the whole matrix against that local server.
|
||||||
|
|
||||||
|
- User agent matrix: R 4.4 / 4.5 / 4.6 on musl and gnu, both arches, Posit-style and stock forms, plus a darwin UA and a non-R UA.
|
||||||
|
- Path matrix: `PACKAGES`, `PACKAGES.gz`, `PACKAGES.rds`, a tarball, a path already under `4.5/`, and `/src/contrib/…` on the bare root.
|
||||||
|
- Assertion is the `Location` header (or its absence), not the body.
|
||||||
|
|
||||||
|
After apply, a smoke test against `cran.rpkgs.com`:
|
||||||
|
|
||||||
|
- `available.packages()` inside `reg.devxy.io/r/r-alpine:4.5-3.24` returns the union count, and `"curl" %in% rownames(...)` is `TRUE`.
|
||||||
|
- A flat-slot package still downloads from `…/src/contrib/`, and a per-minor package downloads from `…/src/contrib/<x.y>/`.
|
||||||
|
|
||||||
|
## Out of scope
|
||||||
|
|
||||||
|
- `Meta/archive.rds` stays flat-only, so `remotes::install_version()` does not see per-minor archives.
|
||||||
|
- The `alpine324` source-tarball defect: that slot serves CRAN sources stamped as binaries, and needs a rebuild independent of this work.
|
||||||
|
- Any change to how `uvr` resolves per-minor URLs; it already addresses the slots directly.
|
||||||
|
|
||||||
|
## Split of work
|
||||||
|
|
||||||
|
1. `bincraft`: union index writer plus its guard, and a re-index of one slot to validate.
|
||||||
|
2. This repo: `edge/rpkgs-router.ts`, its test matrix, and the `cdn.tf` resources with `UNION_SLOTS` empty.
|
||||||
|
3. Enable `UNION_SLOTS` slot by slot as `bincraft` backfills them.
|
||||||
167
specs/2026-08-12-shard-weekly-rebuild-design.md
Normal file
167
specs/2026-08-12-shard-weekly-rebuild-design.md
Normal file
|
|
@ -0,0 +1,167 @@
|
||||||
|
# Design: Sharding and resuming the weekly rebuild
|
||||||
|
|
||||||
|
Date: 2026-08-12
|
||||||
|
Status: Approved (pending spec review)
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
`weekly-rebuild-missing` runs one job per `<os>-<arch>` and walks that slot's rebuild list serially in a single `R -q -e` invocation (`.crow/weekly-rebuild-missing.yaml:165`).
|
||||||
|
Until 2026-08-09 that was cheap, because every source fallback was skipped as "already built" and the list was effectively empty.
|
||||||
|
Since bincraft #105/#106/#107 and build-cran-binaries #159 the gate works, and the lists are now large.
|
||||||
|
|
||||||
|
Share of records whose object is byte-identical to CRAN's source, measured against `cran.r-project.org` MD5s on 2026-08-12:
|
||||||
|
|
||||||
|
| slot | records | source-served | share |
|
||||||
|
| ------------------ | ------: | ------------: | -----------: |
|
||||||
|
| `amd64/resolute` | 24 212 | 15 023 | 62.1% |
|
||||||
|
| `arm64/resolute` | 24 291 | 13 670 | 56.3% |
|
||||||
|
| `arm64/alpine324` | 24 328 | 9 514 | 39.2% |
|
||||||
|
| `amd64/alpine324` | 24 343 | 8 917 | 36.7% |
|
||||||
|
| `arm64/rhel10` | 24 695 | 5 384 | 21.9% |
|
||||||
|
| `amd64/rhel10` | 24 881 | 4 712 | 19.2% |
|
||||||
|
| 12 remaining slots | ~24 700 | 850 to 2 130 | 3.5% to 8.7% |
|
||||||
|
|
||||||
|
A single serial job cannot absorb that.
|
||||||
|
Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) started on 2026-08-09, ran for roughly two days, reached `[8692/23885] cholera`, and was killed there.
|
||||||
|
|
||||||
|
Two distinct failures follow from that shape.
|
||||||
|
|
||||||
|
**No parallelism.** The work is embarrassingly parallel across packages, but one job does all of it.
|
||||||
|
|
||||||
|
**No resumability, and no clean stopping point.** The loop has no terminating condition other than exhausting the list, so the only way to stop it is a kill.
|
||||||
|
A restarted run re-reads the same list and walks it from the first entry.
|
||||||
|
It skips completed packages via `check_s3_root_package()`, but that costs a CRAN version resolution and an S3 `HEAD` per package, thousands of times, before it reaches new work.
|
||||||
|
Worse, a kill is not a pipeline failure: the `Purge CDN cache` step is guarded by `when: status: [success, failure]` (`.crow/weekly-rebuild-missing.yaml:206-207`), and on 10910 it produced no output at all.
|
||||||
|
So the ~4 600 binaries that run did publish stayed hidden behind stale edge copies.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Turn each slot's rebuild into bounded, parallel, restartable units, without introducing state that can disagree with the bucket.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### 1. Shard the matrix three ways
|
||||||
|
|
||||||
|
Each of the 18 `OS`/`ARCH` rows in `.crow/weekly-rebuild-missing.yaml` gains `SPLIT_INTO: 3` and `SPLIT_INDEX: 1|2|3`, giving 54 rows.
|
||||||
|
This mirrors `.crow/build-all-versions.yaml:57-98`, which already shards its matrix four ways per arch.
|
||||||
|
|
||||||
|
Routing needs no change.
|
||||||
|
The cron filter `cron: weekly-rebuild-missing-${OS}-${ARCH}` and the manual `evaluate: weekly_rebuild_missing == "${OS}-${ARCH}"` both match all three shards of a slot.
|
||||||
|
Placement stays on the `rpkgs-${ARCH}` group label, so shards queue against available capacity rather than oversubscribing it.
|
||||||
|
|
||||||
|
### 2. Extract the loop into `local/rebuild-missing.R`
|
||||||
|
|
||||||
|
The build is currently a single ~1 500-character `R -q -e` argument.
|
||||||
|
Shard arithmetic and resume logic do not belong in a YAML string, and none of it is testable there.
|
||||||
|
The loop moves to `local/rebuild-missing.R`, invoked as `Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX`, mirroring `local/build-all.R`.
|
||||||
|
Its body is unchanged in substance: read `/tmp/rebuild_pkgs.txt`, subtract `local/excluded-packages.json`, loop with `tryCatch` around `bincraft::build_binary_package()`.
|
||||||
|
|
||||||
|
The slice is **interleaved**, not contiguous:
|
||||||
|
|
||||||
|
```r
|
||||||
|
# the list is alphabetical and build cost clusters by name (Rcpp*, Bioc*,
|
||||||
|
# rstan*), so contiguous thirds would be badly unbalanced
|
||||||
|
mine <- pkgs[seq(split_index, length(pkgs), by = split_into)]
|
||||||
|
```
|
||||||
|
|
||||||
|
`local/build-all.R:64` uses contiguous chunks via `cut()`.
|
||||||
|
That is fine there because its list is every CRAN package and version, so the chunks average out.
|
||||||
|
Here the list is a filtered backlog in which expensive families sit adjacent, so interleaving is the better default.
|
||||||
|
Interleaving also makes each shard's `[i/n]` progress representative of the slot as a whole.
|
||||||
|
|
||||||
|
### 3. Resume by re-deriving state from the bucket
|
||||||
|
|
||||||
|
Before the loop, the shard performs one `s3fs::s3_dir_info()` on `devxy-rpkgs-binaries/<arch>/<codename>/latest/src/contrib` and reads the `etag` column.
|
||||||
|
It fetches CRAN's `PACKAGES` once for the latest version and published `MD5sum` of every package.
|
||||||
|
A package is still outstanding if and only if the object at `<pkg>_<version>.tar.gz` has an ETag equal to CRAN's `MD5sum` for that version, which is the definition `check_s3_root_package()` already applies one package at a time.
|
||||||
|
|
||||||
|
```r
|
||||||
|
# one paginated listing instead of ~2900 sequential HEAD requests per shard
|
||||||
|
info <- s3fs::s3_dir_info(slot_dir)
|
||||||
|
etag <- setNames(gsub('^"|"$', "", info$etag), basename(info$uri))
|
||||||
|
|
||||||
|
key <- sprintf("%s_%s.tar.gz", mine, cran_version[mine])
|
||||||
|
# keep a package when no object exists yet, or when the object is still
|
||||||
|
# byte-identical to CRAN's source; drop it once a real binary is published
|
||||||
|
mine <- mine[is.na(etag[key]) | etag[key] == cran_md5[key]]
|
||||||
|
```
|
||||||
|
|
||||||
|
This is the whole resume mechanism.
|
||||||
|
There is no progress file, no volume, and no database cursor.
|
||||||
|
A restarted shard recomputes ground truth and continues where it stopped, and it is correct even when a sibling shard, a `process-updates` cron, or a manual `just rebuild` completed something in the meantime.
|
||||||
|
|
||||||
|
Three properties make this the right source of truth:
|
||||||
|
|
||||||
|
- **It is what the build itself checks.** Any other store can disagree with the bucket; this one cannot.
|
||||||
|
- **It is agent-independent.** `.crow/weekly-rebuild-missing.yaml` mounts no `volumes:`, unlike `.crow/build-all-versions.yaml:132-133`, so `/mnt/cache` is per-job and cannot carry progress anyway.
|
||||||
|
- **It costs one listing.** `cranlike`'s `s3` fork already does exactly this call against this bucket at ~24 000 objects, so the approach is proven at the required scale.
|
||||||
|
|
||||||
|
It must read ETags rather than the slot index's `Built` field, which is how `local/packages-to-build.R:104-130` answers the same question.
|
||||||
|
Under this design the index is not rewritten until the dependent re-index pipeline runs (section 5), so mid-run it cannot reflect the current run's progress.
|
||||||
|
|
||||||
|
Packages that genuinely fail to build re-publish their CRAN source, so they stay outstanding and would be retried on every restart.
|
||||||
|
That is already handled upstream: `bincraft::filter_packages_with_errors()` (`R/build_binaries.R:1018`, `:1143`) drops anything with `error_occurred = TRUE`, and `store_build_metadata = TRUE` is passed on every call.
|
||||||
|
No additional poison-pill filter is needed here.
|
||||||
|
|
||||||
|
Only the flat `src/contrib` path is considered.
|
||||||
|
The rebuild call passes no `is_r_minor_sensitive`, so it defaults to `FALSE` and only ever targets the flat path; the resume filter matches that scope deliberately.
|
||||||
|
|
||||||
|
### 4. Give each shard a wall-clock budget
|
||||||
|
|
||||||
|
`local/rebuild-missing.R` takes a budget, defaulting to 20 hours, and breaks out of the loop once it is exceeded:
|
||||||
|
|
||||||
|
```r
|
||||||
|
# exit cleanly rather than being killed, so the dependent re-index still runs
|
||||||
|
if (difftime(Sys.time(), started, units = "hours") > budget_hours) {
|
||||||
|
cat(sprintf("Budget of %sh reached after %d/%d packages; stopping cleanly\n", budget_hours, i, n))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
It exits 0 and reports how much of the slice it covered.
|
||||||
|
Every run then has a terminating condition, the re-index and purge always fire, and the remainder is picked up by the next run with no bookkeeping, because section 3 recomputes the outstanding set from scratch.
|
||||||
|
|
||||||
|
### 5. Move the re-index and purge into `.crow/weekly-rebuild-reindex.yaml`
|
||||||
|
|
||||||
|
Three shards per slot means three concurrent `upload_package_index()` calls on the same S3 prefix.
|
||||||
|
`cranlike::update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work.
|
||||||
|
The re-index steps (`.crow/weekly-rebuild-missing.yaml:171-176`) and the purge step (`:187-207`) therefore leave that file entirely.
|
||||||
|
|
||||||
|
The new file carries:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
depends_on:
|
||||||
|
- weekly-rebuild-missing
|
||||||
|
runs_on: [success, failure]
|
||||||
|
```
|
||||||
|
|
||||||
|
`runs_on: [success, failure]` validates as a workflow-level key under `crow lint`, so a failing shard no longer withholds the re-index.
|
||||||
|
The file uses the same 18-row matrix and the same `when:` gating as `weekly-rebuild-missing`, so it only re-indexes slots that actually ran.
|
||||||
|
Each row re-indexes the flat slot and every per-minor slot.
|
||||||
|
`scripts/purge_cdn_zone.sh` runs once on a single row, because all hostnames share pull zone `3857050` and 18 identical zone purges would be waste.
|
||||||
|
|
||||||
|
## Failure behaviour
|
||||||
|
|
||||||
|
| case | today | after |
|
||||||
|
| -------------------------- | ----------------------------------- | ----------------------------------------------------- |
|
||||||
|
| one package errors | `tryCatch` logs, loop continues | unchanged |
|
||||||
|
| a shard fails outright | purge runs, re-index does not | re-index and purge run via `runs_on` |
|
||||||
|
| a shard exceeds its budget | cannot happen, runs until killed | exits 0, re-index and purge run |
|
||||||
|
| a shard is killed | nothing runs | still nothing; trigger the re-index pipeline alone |
|
||||||
|
| a shard restarts | re-walks the list, HEAD per package | one listing, resumes at the first outstanding package |
|
||||||
|
|
||||||
|
The known cost of `depends_on` being file-level rather than row-level: on the weekly cron no slot is re-indexed until the slowest of all 54 jobs finishes.
|
||||||
|
The 20-hour budget bounds that at roughly one day.
|
||||||
|
|
||||||
|
## Out of scope
|
||||||
|
|
||||||
|
- `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row for the platform and arch, which is precisely the source-fallback set. That is a separate change.
|
||||||
|
- Bunny Perma-Cache eviction. `scripts/purge_cdn_zone.sh` purges the regular edge cache only; see the note in `CLAUDE.md` and issue history.
|
||||||
|
- The audit that produces the rebuild list is unchanged.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
- `crow lint .crow/` passes for both pipeline files.
|
||||||
|
- `local/rebuild-missing.R` gets unit coverage in `local/tests/` for the two pure pieces: the interleaved slice (disjoint, covering, deterministic) and the outstanding-set filter (source-served ETag kept, binary ETag dropped, absent object kept).
|
||||||
|
- A single-slot manual run of `alpine-324-amd64` shard 1 confirms the listing shortcut against the live bucket, and that the reported outstanding count is close to the 8 917 measured above divided by three.
|
||||||
|
- Restarting that shard mid-run confirms it resumes rather than replaying, by comparing the outstanding count it reports on the second start.
|
||||||
Loading…
Reference in a new issue