chore: ignore plans and specs directories

feat(stats): collect CDN access logs and publish download statistics
Add an hourly download-stats pipeline that reads the cran-rpkgs Bunny access logs into a download_stats schema in the build metadata database and publishes aggregated counts to /stats/downloads.json for the rpkgs.com /stats page. - Classify each request (download, index, redirect reason, probe, noise) mirroring the edge router, and split cloud/CI networks from the rest. - De-duplicate downloads per client, package version and day, and key clients on a salted HMAC so no IP is stored or published. - Exclude our own CI runners and record run windows to show gaps. - Declare the zone's logging settings and allow CORS for JSON.
16 changed files with 1283 additions and 2146 deletions
--- a/.crow/download-stats.yaml
+++ b/.crow/download-stats.yaml
@ -0,0 +1,79 @@
 # Collect CDN access logs and publish the public download statistics.
 #
 # Each run reads the last 70 hours of cran-rpkgs access logs from Bunny into
 # `download_stats.events` (build metadata Postgres), then rewrites
 # https://cran.rpkgs.com/stats/downloads.json, which the rpkgs.com /stats page
 # reads. Bunny keeps logs for only three days, so a run every hour leaves room
 # for two days of failed runs before anything is lost. Re-reading the window is
 # safe: events are keyed by Bunny's request id.
 #
 # The classification rules and what is excluded are documented in
 # `stats/README.md`.
 #
 # Register the `download-stats` cron (hourly, `0 * * * *`) in the crow UI, or
 # run manually:
 #   crow pipeline create --branch main --var download_stats=true devxy/build-cran-binaries
 #
 # Secrets:
 #   STATS_CLIENT_SALT   HMAC key for the client hash. Rotating it resets
 #                       distinct-client counts, so keep it stable.
 #   STATS_OWN_NETWORKS  Comma-separated CIDRs of our own CI runners.
 #   Both live in OpenBao at development/agent-dev/rpkgs-download-stats.
 variables:
  download_stats:
    description: 'Collect and publish download statistics. Also gates this pipeline.'
    options:
      - 'true'
      - 'false'
    default: 'false'
 when:
  - event: manual
    evaluate: 'download_stats == "true"'
  - event: cron
    cron: download-stats
 skip_clone: true
 labels:
  group: rpkgs-amd64
 steps:
  - name: 'Collect and publish download stats'
    image: oven/bun:1.4.4-alpine
    pull: true
    environment:
      BUNNYNET_API_KEY:
        from_secret: BUNNYNET_API_KEY
      PGPASS:
        from_secret: PGPASS
      B2_S3_ACCESS_KEY:
        from_secret: B2_S3_ACCESS_KEY
      B2_S3_SECRET_KEY:
        from_secret: B2_S3_SECRET_KEY
      STATS_CLIENT_SALT:
        from_secret: STATS_CLIENT_SALT
      STATS_OWN_NETWORKS:
        from_secret: STATS_OWN_NETWORKS
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
    commands:
      - apk add --no-cache -q git
      - git clone -q --depth 1 --branch "$CI_COMMIT_BRANCH" https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - bun test stats
      - bun stats/collect.ts
      - bun stats/publish.ts
    backend_options:
      kubernetes:
        resources:
          requests:
            memory: 256Mi
            cpu: 250m
          limits:
            memory: 1Gi
            cpu: 1000m
        tolerations:
          - key: 'CI'
            operator: 'Equal'
            value: 'true'
            effect: 'NoSchedule'
--- a/.gitignore
+++ b/.gitignore
@ -102,3 +102,7 @@ docs/
 # Superpowers SDD scratch (briefs, reports, ledger)
 .superpowers/
 # Internal design docs (plans and specs) are kept out of the repo.
 plans/
 specs/
--- a/.markdownlint-cli2.yaml
+++ b/.markdownlint-cli2.yaml
@ -2,6 +2,3 @@
 ignores:
  - LICENSE.md
  - docs/superpowers/**
  # Internal design docs (specs/plans) are not user-facing reference material.
  - specs/**
  - plans/**
--- a/README.md
+++ b/README.md
@ -73,8 +73,6 @@ Patching uses a two-tier approach:
 The system is implemented in `bincraft`: when a package needs patching, `bincraft` pre-builds it with the patch and serves the patched binary to `pak`, ensuring transitive dependents receive the fixed package.
 This way, the fix cascades to all packages that depend on it.
 For the design rationale and architecture, see `specs/2026-06-30-package-patching-design.md`.
 ## Build Environment
 Binaries are built on a mixed-architecture Kubernetes cluster using CI.
--- a/cdn.tf
+++ b/cdn.tf
@ -164,6 +164,23 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" {
  add_canonical_header = true
  # Access logs feed the public download statistics (`stats/`, published to
  # `/stats/downloads.json`). Both settings were already on; declaring them
  # keeps an apply from quietly switching off the only source of that data.
  # Bunny keeps the logs for three days, which is why `download-stats` runs on
  # a schedule. IPs stay anonymized to /24 (/64 for IPv6) at the edge.
  log_enabled          = true
  log_anonymized       = true
  log_anonymized_style = "OneDigit"
  # The rpkgs.com website reads `/stats/downloads.json` from the browser, so
  # JSON needs the CORS header too. The rest is the zone's existing list.
  cors_enabled = true
  cors_extensions = [
    "css", "eot", "gif", "jpeg", "jpg", "js", "json", "mp3", "mp4", "mpeg", "png", "svg", "ttf", "webm", "webp",
    "woff", "woff2",
  ]
  cache_stale = ["offline", "updating"]
  block_ips = var.cdn_block_ips
--- a/plans/2026-06-30-package-patching-implementation.md
+++ b/plans/2026-06-30-package-patching-implementation.md
--- a/plans/2026-08-07-per-minor-edge-routing.md
+++ b/plans/2026-08-07-per-minor-edge-routing.md
@ -1,166 +0,0 @@
 # Per-R-minor edge routing implementation plan
 Spec: `specs/2026-08-07-per-minor-edge-routing-design.md`
 **Goal:** let a stock `install.packages()` see the per-minor packages by routing `PACKAGES*` requests to `…/src/contrib/<x.y>/`, where `bincraft` publishes a union index.
 **Architecture:** the union is built in `bincraft`; the edge script only redirects index requests, gated on a `UNION_SLOTS` script variable; the script lives in this repo and is applied by OpenTofu.
 **Tech stack:** Deno / TypeScript (Bunny Edge Scripting, SDK 0.12), OpenTofu with `BunnyWay/bunnynet` 0.17, R (bincraft).
 ## Global constraints
 - Redirect only `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds`; never a tarball, because the union index already carries the correct tarball URL for both classes of package.
 - Every redirect carries `Cache-Control: no-store`; redirect targets stay UA-independent.
 - `UNION_SLOTS` is empty by default, so deploying the script is a no-op until a slot is backfilled.
 - A slot is `<arch>/<os>`, e.g. `amd64/alpine324`.
 - Verified prerequisites: `PACKAGES*` is served `cdn-cache: BYPASS`, so the script sees every index request; `Deno.env.get()` reads script variables; the SDK local server listens on `127.0.0.1:8080`.
 ---
 ## Task 1: Edge script and its test matrix
 **Files:**
 - Create: `edge/rpkgs-router.ts`
 - Create: `edge/rpkgs-router.test.ts`
 - Modify: `justfile` (add `edge-test`)
 **Produces:** a single-file script deployable as `bunnynet_compute_script.content`, reading `UNION_SLOTS` from the environment.
 - [ ] **Step 1: write the test matrix first**
 `edge/rpkgs-router.test.ts` spawns `deno run -A edge/rpkgs-router.ts` with `UNION_SLOTS=amd64/alpine324`, waits for `127.0.0.1:8080`, and issues requests with `redirect: "manual"`.
 Cases, asserted on the `location` header (or its absence):
 | #   | path                                                   | User-Agent                                    | expectation                                                  |
 | --- | ------------------------------------------------------ | --------------------------------------------- | ------------------------------------------------------------ |
 | 1   | `/amd64/alpine324/latest/src/contrib/PACKAGES.gz`      | `R (4.5.3 x86_64-pc-linux-musl …)`            | 302 → `…/src/contrib/4.5/PACKAGES.gz`                        |
 | 2   | same                                                   | `R (4.6.0 …)`                                 | 302 → `…/src/contrib/4.6/PACKAGES.gz`                        |
 | 3   | same, but slot `amd64/noble`                           | `R (4.5.3 …)`                                 | no redirect (slot not in `UNION_SLOTS`)                      |
 | 4   | `…/src/contrib/curl_7.1.0.tar.gz`                      | `R (4.5.3 …)`                                 | no redirect                                                  |
 | 5   | `…/src/contrib/4.5/PACKAGES.gz`                        | `R (4.5.3 …)`                                 | no redirect (loop guard)                                     |
 | 6   | `…/src/contrib/PACKAGES.gz`                            | `curl/8.0`                                    | no redirect (no R minor)                                     |
 | 7   | `/src/contrib/PACKAGES.gz`                             | alpine UA with `Alpine Linux … 3.24`          | 302 → `/amd64/alpine324/latest/src/contrib/4.5/PACKAGES.gz`  |
 | 8   | `/src/contrib/PACKAGES.gz`                             | `R (4.5.3 x86_64-pc-linux-musl …)`, no distro | 302 → `cran.r-project.org`, **not** a `linux-musl` slot      |
 | 9   | `/src/contrib/foo_1.0.tar.gz`                          | `R (4.5.1 aarch64-apple-darwin20 …)`          | 302 → `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` |
 | 10  | `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | any                                           | 302 → `cran.r-project.org`                                   |
 | 11  | any redirect above                                     | —                                             | `cache-control: no-store`                                    |
 - [ ] **Step 2: run the tests and watch them fail**
 `just edge-test` → every case fails, because `edge/rpkgs-router.ts` does not exist.
 - [ ] **Step 3: write `edge/rpkgs-router.ts`**
 Order of evaluation in `onOriginRequest`:
 1. normalise `//` runs in the path
 2. darwin `/src/contrib/*` → `/bin/macosx/<flavour>/contrib/<x.y>/`
 3. `/bin/macosx/**` → CRAN
 4. `/{arch}/{os}/latest/src/contrib/<rest>`: pass through if `rest` already starts with `<x.y>/`, or is not an index file, or the slot is not in `UNION_SLOTS`, or the UA has no R minor; otherwise redirect into `<x.y>/`
 5. `/`, `/src/contrib`, `/src/contrib/**`: resolve arch+os from the UA, redirect to CRAN when the distro is unidentifiable, otherwise redirect to the qualified path, adding `<x.y>/` under the same index-file rule
 6. anything else: pass through
 The R minor comes from either `R/4.5.3` or `R (4.5.3 …)`, so a stock UA is enough. The `linux-gnu` / `linux-musl` fallback in `parseUserAgent` is deleted: those are not slot names.
 - [ ] **Step 4: run the tests until they pass**
 `just edge-test`
 - [ ] **Step 5: commit**
 ```bash
 git add edge/rpkgs-router.ts edge/rpkgs-router.test.ts justfile
 git commit -m "feat(edge): route PACKAGES requests to the per-R-minor slot"
 ```
 ---
 ## Task 2: Manage the script from OpenTofu
 **Files:**
 - Modify: `cdn.tf`
 **Consumes:** `edge/rpkgs-router.ts` from Task 1.
 - [ ] **Step 1: add the resources**
 ```terraform
 resource "bunnynet_compute_script" "rpkgs_router" {
  type    = "middleware"
  name    = "rpkgs-router"
  content = file("${path.module}/edge/rpkgs-router.ts")
 }
 resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" {
  script        = bunnynet_compute_script.rpkgs_router.id
  name          = "UNION_SLOTS"
  default_value = ""
  required      = false
 }
 ```
 and replace `middleware_script = 29277` with `middleware_script = bunnynet_compute_script.rpkgs_router.id`.
 - [ ] **Step 2: validate**
 `tofu init -backend=false && tofu validate`
 - [ ] **Step 3: import the existing script (needs `BUNNYNET_API_KEY`)**
 ```bash
 tofu import bunnynet_compute_script.rpkgs_router 29277
 tofu plan
 ```
 The plan must show an in-place `content` update and **no** replacement of the pull zone. A replacement means the import did not take.
 - [ ] **Step 4: commit**
 ```bash
 git add cdn.tf
 git commit -m "feat(cdn): manage the edge middleware script from this repo"
 ```
 ---
 ## Task 3: Union index writer in bincraft
 **Files (repo `codefloe.com/rpkgs/bincraft`):**
 - Modify: `R/package_index.R`
 - Test: `tests/testthat/test-package_index.R`
 **Produces:** `write_union_index(flat_records, minor_records)` returning the merged records, called from `upload_package_index()` when `r_minor` is set.
 - [ ] **Step 1: write the failing tests**
 - a package present in both slots keeps the per-minor record, with `Path = "4.5"`
 - a package only in the flat slot survives with no `Path`
 - a package only in the per-minor slot survives with `Path = "4.5"`
 - a union smaller than the flat input raises an error rather than returning
 - [ ] **Step 2: run them and watch them fail**
 `Rscript -e 'testthat::test_file("tests/testthat/test-package_index.R")'`
 - [ ] **Step 3: implement `write_union_index()` and call it from `upload_package_index()`**
 After `update_PACKAGES()` has written the per-minor index, read the flat slot's `PACKAGES.rds`, set `Path = <r_minor>` on the per-minor records, drop the flat records for packages the per-minor slot already has, and rewrite `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` in the per-minor slot.
 - [ ] **Step 4: run the tests until they pass**
 - [ ] **Step 5: commit and open the PR against bincraft**
 ---
 ## Task 4: Roll out slot by slot
 - [ ] Re-index one slot (`amd64/alpine324`, R 4.5) and confirm the union index lists both `curl` (per-minor, `Path: 4.5`) and `jsonlite` (flat, no `Path`).
 - [ ] Set `UNION_SLOTS = "amd64/alpine324"` and confirm in `reg.devxy.io/r/r-alpine:4.5-3.24` that `available.packages()` returns the union count and `"curl" %in% rownames(...)`.
 - [ ] Add `arm64/alpine324`, then the remaining slots.
 `install.packages("curl")` will still fail to build on `alpine324` until that slot's source tarballs are replaced with real binaries. That is tracked separately.
--- a/specs/2026-06-30-package-patching-design.md
+++ b/specs/2026-06-30-package-patching-design.md
@ -1,157 +0,0 @@
 # Design: Dynamic per-package patching during binary builds
 Date: 2026-06-30
 Status: Approved (pending spec review)
 ## Problem
 Some CRAN packages fail to compile on specific build platforms due to compiler- or OS-specific issues that have nothing to do with the package being built.
 The canonical example is `RcppParallel`: its bundled Intel TBB sources fail to build on musl (Alpine) and on newer OS/compiler combinations.
 Observed failure on `ubuntu-2604` ("resolute") with `g++ 15.2.0`:
 ```
 ../build/common.inc:74: *** "" is not supported. Add build/.inc file with os-specific settings . Stop.
 make: *** [Makevars:163: tbb] Error 2
 ERROR: compilation failed for package 'RcppParallel'
 ```
 Because `RcppParallel` is a dependency of many packages, a single such failure cascades: every dependent package (e.g. `rts2`) also fails, even though nothing is wrong with the dependent itself.
 Today there is no way to intervene.
 A package can only be **excluded** (`local/excluded-packages.json`), which is all-or-nothing and does not help dependents.
 ## Goal
 Allow a curated set of packages to be "patched" — via lightweight build-time overrides or, when necessary, real source diffs — **before** they are installed, whether the package is a direct build target or a transitive dependency pulled in by `pak`.
 ## Key constraint that drives the design
 When `RcppParallel` fails here, it is being installed as a **transitive dependency** by `pak`, inside `bincraft::build_binary_package()`.
 `pak` downloads, configures, and compiles it in one subprocess; this repo never touches that source.
 For a fix to reach a dependency-of-a-dependency, the fixed package must be visible to `pak` itself, where `pak`'s repositories/sources are configured — which is inside `bincraft`.
 Decisions taken during brainstorming:
 - **Mechanism lives in `bincraft`** (the engine), because only there can transitive deps be influenced.
 - **Patch tiers: both, env-overrides first.** Support cheap per-package env vars / configure args / Makevars (version-independent) *and* true source diffs (version-pinned), preferring the lightweight override.
 - **Registry data lives in this repo** (`build-cran-binaries`) and is passed into `bincraft`, keeping `bincraft` as pure mechanism and the frequently-changing policy data with operational config.
 ## Approaches considered
 | Approach | How pak sees the fix | Verdict |
 |---|---|---|
 | A. Patched **source** repo — drop patched `.tar.gz` source into a local repo, prepend it | pak recompiles from your source | Simple, but env-tier overrides leak globally (one subprocess builds everything) and the dep recompiles on every dependent build |
 | **B. Pre-built patched binary repo (chosen)** | pak installs a ready binary by repo priority | Per-package scoping is free; no recompile; the binary is a cacheable/uploadable artifact that fits the existing system |
 | C. pkgdepends per-build hook | intercept each build | No clean per-package pre-compile hook exists; fragile |
 Chosen: **B**.
 ## Architecture
 ### Registry (this repo)
 ```
 local/patches/
  registry.json          # the manifest
  RcppParallel/
    fix.patch            # optional source diff, referenced by an entry
 ```
 `registry.json` is an array of entries:
 ```json
 [
  {
    "package": "RcppParallel",
    "versions": "*",
    "platforms": ["alpine", "ubuntu-2604"],
    "env": { "RCPP_PARALLEL_USE_TBB": "0" },
    "configure_args": [],
    "makevars": {},
    "patch": null,
    "reason": "bundled TBB fails to build on musl / newer compilers"
  }
 ]
 ```
 Field semantics:
 - `package` (string, required): CRAN package name.
 - `versions` (string, required): `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`.
  Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against.
 - `platforms` (array of strings, required): matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`).
  An entry matches if any listed token matches any build token.
  `["*"]` matches all platforms.
 - `env` (object, optional): environment variables exported only for this package's isolated build.
 - `configure_args` (array, optional): passed as `--configure-args` to the isolated build.
 - `makevars` (object, optional): key/value pairs written into a package-local Makevars for the isolated build.
 - `patch` (string or null, optional): path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building.
 - `reason` (string, required): human explanation, surfaced in logs and metadata.
 A fix is any combination of `env`, `configure_args`, `makevars`, and `patch`.
 "Env-first" is an authoring guideline (prefer the lightweight override) and an ordering of effort, not a runtime branch — all present fields are applied together for the isolated build.
 ### Flow (inside bincraft, around existing pak resolution)
 1. **Resolve** the dependency set (dry-run) to learn the concrete versions `pak` will install.
   Reuse bincraft's existing resolution where possible (e.g. a `pkgdepends` proposal: `$resolve()` → inspect resolution → ... → `$solve()` / `$install()` after the local repo is prepended).
 2. For each resolved package that matches a registry entry (name + `versions` + `platforms`): obtain a **patched binary** for the exact `version × platform × arch × R-minor`:
   - **Cache hit** (local `/mnt/cache/patched-binaries/` or S3): fetch it into the local repo.
   - **Cache miss**: download the CRAN **source** for that version, apply the source `patch` (if any) to the unpacked tree, build the binary in isolation with `env` / `configure_args` / `makevars` applied, then place the binary in the local repo and write it to the cache (and S3 if uploading is enabled).
 3. **Prepend** the local binary repo (`file://…`) to `pak`'s repo list, and regenerate its `PACKAGES` index.
 4. Run the **normal install**.
   `pak` resolves the patched binary for the matched package — direct or transitive — because it wins on repo priority for an equal version, and installs it without recompiling.
 ### Caching (essential)
 `RcppParallel` is a dependency of dozens of packages; without caching the fix would be rebuilt on every dependent build.
 Patched binaries are keyed by:
 ```
 <package>_<version>_<platform>_<arch>_<rminor>_<patchhash>
 ```
 `patchhash` is a hash of the normalized registry entry plus the referenced diff file contents.
 Editing a patch therefore changes the hash and auto-invalidates stale cached binaries.
 - Local cache: `/mnt/cache/patched-binaries/`.
 - Optional S3 cache for cross-build reuse: a dedicated `…/patched/` slot under the existing arch/codename structure, mirroring how normal binaries are stored.
 ### S3 upload
 Patched binaries **are** uploaded to S3 (in addition to the local cache) so they are reused across CI jobs and machines, not just within one container.
 They live in a separate `patched/` slot and are not published into the user-facing `src/contrib` index — they are an internal build accelerator, not a distributed artifact.
 ## Error handling
 - **Source diff fails to apply** (CRAN moved past the pinned version): log a clear warning, skip that entry, and proceed.
  The package builds unpatched (status quo) and may fail.
  The skipped/failed-to-apply patch is surfaced in build metadata.
 - **Pre-build of the patched binary fails**: log a warning, skip, proceed.
 - **No version or platform match**: skip silently (the entry simply does not apply to this build).
 - **Overlapping entries for one package**: the most specific entry wins (a concrete `platforms`/`versions` beats `"*"`).
  Genuine ambiguity (two equally specific, conflicting entries) is a validation error reported before the build.
 ## Observability
 - One log line per applied patch, e.g.: `Applying patch to RcppParallel 5.1.11-2 [env: RCPP_PARALLEL_USE_TBB=0]: bundled TBB fails on musl / newer compilers`
 - The set of applied patches (package, version, `patchhash`) is recorded in the Postgres build-metadata row for the build, so it is queryable later.
 ## Testing
 - **Unit (registry):** parsing and matching — version constraints, platform token matching, precedence/specificity, and detection of ambiguous overlaps.
 - **Unit (cache key):** `patchhash` changes when the entry or diff changes; is stable otherwise.
 - **Integration:** `RcppParallel` on `resolute` (and/or Alpine) fails to build without a registry entry and succeeds with one; a dependent package such as `rts2` succeeds once the dependency is patched.
 - **Failure path:** an entry pinned to an old version against a newer CRAN release → graceful skip with a warning, build continues.
 ## Out of scope
 - Shipping a default registry inside `bincraft` (registry is repo-local for now; a baseline-in-engine + repo-override model can come later if needed).
 - Publishing patched binaries into the public `src/contrib` index.
 - Automatic detection of which packages need patches — entries are curated by hand.
 ## Split of work
 - **bincraft:** the mechanism — registry ingestion, resolution hook, isolated patched-binary build, caching/upload, local-repo prepend, logging, metadata recording.
  A new `patches` argument on `build_binary_package()`.
 - **build-cran-binaries (this repo):** the `local/patches/` registry and diffs, passing `patches = "local/patches"` through `build-one.R` / `build-all.R`, and documentation.
--- a/specs/2026-08-07-per-minor-edge-routing-design.md
+++ b/specs/2026-08-07-per-minor-edge-routing-design.md
@ -1,174 +0,0 @@
 # Design: Routing clients to per-R-minor binary slots
 Date: 2026-08-07
 Status: Approved (pending spec review)
 ## Problem
 `bincraft` routes ABI-"risky" packages to a per-minor slot `…/latest/src/contrib/<x.y>/` and indexes every directory independently (`upload_package_index()` calls `cranlike::update_PACKAGES()` on one prefix at a time).
 Nothing unions those indices, and `contrib.url()` only ever yields `<repos>/src/contrib`, so no value of `options(repos)` can address a per-minor slot.
 Only `uvr` resolves per-minor URLs, which means the per-minor slots are invisible to `install.packages()` by construction.
 Measured on 2026-08-07:
 | slot              | flat `src/contrib` | `src/contrib/4.5` | unique packages only in the per-minor slot |
 | ----------------- | ------------------ | ----------------- | ------------------------------------------ |
 | `amd64/alpine324` | 21 640             | 3 310             | 2 886                                      |
 | `amd64/noble`     | 24 495             | 398               | 23                                         |
 This is what issue #63 records as "missing binaries" on `alpine324`.
 The packages are not missing; they are in a directory base R cannot reach.
 The user-visible symptom in `reg.devxy.io/r/r-alpine:4.5-3.24` is:
 ```
 > install.packages("curl")
 Warning message:
 package 'curl' is not available for this version of R
 ```
 A second, unrelated defect exists on the same slot and is **out of scope here**: many `alpine324` tarballs are byte-identical CRAN _source_ tarballs that the index nevertheless stamps `Built: R 4.5.3; …-linux-musl`.
 Routing exposes `curl`; only a rebuild of that slot makes it install.
 ## Goal
 Let a stock `install.packages()` see one complete package list for its own R minor, without duplicating tarballs and without an R-version-varying cache key anywhere in the CDN.
 ## Key constraint that drives the design
 R resolves a package's download URL from the index, not from the request path, and it keeps the `contriburl` it _asked for_ rather than the one it was redirected to.
 Measured with `options(repos = …/latest)` against a middleware that redirects the index into `4.5/`:
 ```
 curl available: TRUE
 curl repo: …/latest/src/contrib          # the flat URL, not the 4.5 one it was served from
 ```
 So the union index is always addressed relative to the **flat** directory, whatever path it was fetched from.
 `available.packages()` honours a `Path:` field and folds it into the `Repository` column, which gives the whole routing for free:
 - a per-minor record carries `Path: <x.y>`, so its tarball is fetched from `…/src/contrib/<x.y>/`
 - a flat record carries no `Path`, so its tarball is fetched from `…/src/contrib/`
 Verified end to end against the live CDN with a locally built union index for `amd64/alpine324` (31 507 records):
 ```
 curl:     7.1.0  -> …/latest/src/contrib/4.5   -> curl_7.1.0.tar.gz     717 725 B
 jsonlite: 2.0.0  -> …/latest/src/contrib       -> jsonlite_2.0.0.tar.gz 1 055 849 B
 ```
 The corollary is that the edge script must **not** rewrite tarball requests: every tarball URL is already correct when it leaves the client, and redirecting one into `<x.y>/` would break exactly the flat packages the union is meant to preserve.
 The complementary trick does not work: R's `gzcon()` reads only the first member of a concatenated gzip stream (10 291 of an expected 31 931 records), so an edge-side merge would have to fully decompress and recompress both indices and additionally 404 `PACKAGES.rds` to stop R preferring it.
 That is why the union is produced in `bincraft`, not at the edge.
 ## Approaches considered
 | Approach                                                    | Where the union lives                                        | Verdict                                                                                                     |
 | ----------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- |
 | **A. Union index written by `bincraft` (chosen)**           | per-minor `PACKAGES*`, per-minor entries carry `Path: <x.y>` | Edge does one redirect; `PACKAGES.rds` stays correct; no duplication                                        |
 | B. Merge at the edge                                        | middleware fetches both indices, recompresses                | ~2 MB decompress/recompress per cache fill, cache key must include the R minor, breaks R's `.rds` fast path |
 | C. Move the minor up the path (`latest/<x.y>/src/contrib/`) | addressable by `options(repos)` directly                     | No edge logic at all, but a full layout migration and breaks the published URL contract                     |
 Chosen: **A**.
 ## Architecture
 ### bincraft: union index (separate PR)
 After writing a per-minor index, republish it as a union of that slot and the flat slot:
 1. Read the flat slot's `PACKAGES.rds` and the per-minor slot's own records.
 2. Set `Path: <x.y>` on every per-minor record, so its tarball resolves into the per-minor directory.
 3. Drop every flat record whose package is already present in the per-minor slot, so the per-minor build always wins, and leave the survivors without a `Path`.
 4. Write the merged `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib/<x.y>/`.
 Guard: refuse to publish a union with fewer records than the flat index it was built from.
 A truncated union is worse than no union, because it silently removes packages from every client on that minor.
 ### Edge script (this repo)
 The script routes `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` requests, and nothing else.
 ```
 normalize path
 parseClient(UA) -> { rMinor, arch, os }      # rMinor from "R (4.5.3 …)" or "R/4.5.3"
 darwin branches                               # unchanged
 if path is /{arch}/{os}/latest/src/contrib/PACKAGES*
    already under /<x.y>/ ? pass through      # loop guard
    rMinor known && slot in UNION_SLOTS ? 302 -> …/src/contrib/<rMinor>/PACKAGES*
    else pass through                         # flat slot, today's behaviour
 if path is /src/contrib/…                     # bare root
    resolve arch + os; unknown -> 302 to CRAN
    then apply the same PACKAGES* rule
 else pass through
 ```
 Redirects carry `Cache-Control: no-store`.
 Every cacheable URL is therefore UA-independent, and no cache key has to vary by R version.
 ### Repaired bare-root detection
 The bare `https://cran.rpkgs.com` form is currently broken for every Linux client that uses a stock R user agent.
 `ALPINE_REGEX`, `UBUNTU_REGEX` and `RHEL_REGEX` only match a Posit-style user agent the user has to set by hand; stock R never carries the distro, so the script falls through to `extractOs()` and redirects to a slot that does not exist:
 ```
 UA: R (4.5.3 x86_64-pc-linux-musl …)  ->  302 /amd64/linux-musl/latest/…   (404)
 UA: R (4.5.3 x86_64-pc-linux-gnu  …)  ->  302 /amd64/linux-gnu/latest/…    (404)
 ```
 The fallback to a phantom `linux-musl` / `linux-gnu` slot is removed.
 An unidentifiable distro redirects to CRAN, which is the existing behaviour for an unparseable user agent.
 The R _minor_ is always present in a stock user agent, so per-minor routing itself does not depend on distro detection.
 ### Rollout gate
 `UNION_SLOTS` is a `bunnynet_compute_script_variable` listing the slots whose per-minor index is already a union.
 It is empty by default, so deploying the script changes nothing until `bincraft` has backfilled a slot, and a rollback is a variable edit rather than a code deploy.
 All slots currently carry `4.4`, `4.5` and `4.6`; a client on any other minor falls through to the flat slot.
 ### Deployment from this repo
 The script is a file in the repo, applied by the existing OpenTofu configuration:
 ```
 edge/rpkgs-router.ts          # the script
 edge/rpkgs-router.test.ts     # UA x path -> expected Location matrix
 cdn.tf                        # bunnynet_compute_script + _variable
 ```
 Provider `BunnyWay/bunnynet` v0.17.0 (already pinned) ships `bunnynet_compute_script` with `content` loadable via `file()`, plus `bunnynet_compute_script_variable`.
 `middleware_script = bunnynet_compute_script.rpkgs_router.id` replaces the hard-coded `29277`, after a one-time `tofu import` of the existing script.
 ## Error handling
 - Unknown R minor, or a slot not listed in `UNION_SLOTS`: pass through to the flat slot.
  The client sees exactly today's behaviour.
 - Unparseable distro on the bare-root form: redirect to CRAN.
 - A request already under `…/src/contrib/<x.y>/`: pass through, so a redirect can never loop.
 - A per-minor slot that does not exist for a listed minor: the client gets the origin's 404.
  `UNION_SLOTS` is the operator's assertion that the slot is ready, so this is a configuration error, not a runtime condition to paper over.
 ## Testing
 Local, before any apply: `deno run -A edge/rpkgs-router.ts` serves the middleware against the real origin, so `edge/rpkgs-router.test.ts` drives the whole matrix against that local server.
 - User agent matrix: R 4.4 / 4.5 / 4.6 on musl and gnu, both arches, Posit-style and stock forms, plus a darwin UA and a non-R UA.
 - Path matrix: `PACKAGES`, `PACKAGES.gz`, `PACKAGES.rds`, a tarball, a path already under `4.5/`, and `/src/contrib/…` on the bare root.
 - Assertion is the `Location` header (or its absence), not the body.
 After apply, a smoke test against `cran.rpkgs.com`:
 - `available.packages()` inside `reg.devxy.io/r/r-alpine:4.5-3.24` returns the union count, and `"curl" %in% rownames(...)` is `TRUE`.
 - A flat-slot package still downloads from `…/src/contrib/`, and a per-minor package downloads from `…/src/contrib/<x.y>/`.
 ## Out of scope
 - `Meta/archive.rds` stays flat-only, so `remotes::install_version()` does not see per-minor archives.
 - The `alpine324` source-tarball defect: that slot serves CRAN sources stamped as binaries, and needs a rebuild independent of this work.
 - Any change to how `uvr` resolves per-minor URLs; it already addresses the slots directly.
 ## Split of work
 1. `bincraft`: union index writer plus its guard, and a re-index of one slot to validate.
 2. This repo: `edge/rpkgs-router.ts`, its test matrix, and the `cdn.tf` resources with `UNION_SLOTS` empty.
 3. Enable `UNION_SLOTS` slot by slot as `bincraft` backfills them.
--- a/specs/2026-08-12-shard-weekly-rebuild-design.md
+++ b/specs/2026-08-12-shard-weekly-rebuild-design.md
@ -1,167 +0,0 @@
 # Design: Sharding and resuming the weekly rebuild
 Date: 2026-08-12
 Status: Approved (pending spec review)
 ## Problem
 `weekly-rebuild-missing` runs one job per `<os>-<arch>` and walks that slot's rebuild list serially in a single `R -q -e` invocation (`.crow/weekly-rebuild-missing.yaml:165`).
 Until 2026-08-09 that was cheap, because every source fallback was skipped as "already built" and the list was effectively empty.
 Since bincraft #105/#106/#107 and build-cran-binaries #159 the gate works, and the lists are now large.
 Share of records whose object is byte-identical to CRAN's source, measured against `cran.r-project.org` MD5s on 2026-08-12:
 | slot               | records | source-served |        share |
 | ------------------ | ------: | ------------: | -----------: |
 | `amd64/resolute`   |  24 212 |        15 023 |        62.1% |
 | `arm64/resolute`   |  24 291 |        13 670 |        56.3% |
 | `arm64/alpine324`  |  24 328 |         9 514 |        39.2% |
 | `amd64/alpine324`  |  24 343 |         8 917 |        36.7% |
 | `arm64/rhel10`     |  24 695 |         5 384 |        21.9% |
 | `amd64/rhel10`     |  24 881 |         4 712 |        19.2% |
 | 12 remaining slots | ~24 700 |  850 to 2 130 | 3.5% to 8.7% |
 A single serial job cannot absorb that.
 Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) started on 2026-08-09, ran for roughly two days, reached `[8692/23885] cholera`, and was killed there.
 Two distinct failures follow from that shape.
 **No parallelism.** The work is embarrassingly parallel across packages, but one job does all of it.
 **No resumability, and no clean stopping point.** The loop has no terminating condition other than exhausting the list, so the only way to stop it is a kill.
 A restarted run re-reads the same list and walks it from the first entry.
 It skips completed packages via `check_s3_root_package()`, but that costs a CRAN version resolution and an S3 `HEAD` per package, thousands of times, before it reaches new work.
 Worse, a kill is not a pipeline failure: the `Purge CDN cache` step is guarded by `when: status: [success, failure]` (`.crow/weekly-rebuild-missing.yaml:206-207`), and on 10910 it produced no output at all.
 So the ~4 600 binaries that run did publish stayed hidden behind stale edge copies.
 ## Goal
 Turn each slot's rebuild into bounded, parallel, restartable units, without introducing state that can disagree with the bucket.
 ## Design
 ### 1. Shard the matrix three ways
 Each of the 18 `OS`/`ARCH` rows in `.crow/weekly-rebuild-missing.yaml` gains `SPLIT_INTO: 3` and `SPLIT_INDEX: 1|2|3`, giving 54 rows.
 This mirrors `.crow/build-all-versions.yaml:57-98`, which already shards its matrix four ways per arch.
 Routing needs no change.
 The cron filter `cron: weekly-rebuild-missing-${OS}-${ARCH}` and the manual `evaluate: weekly_rebuild_missing == "${OS}-${ARCH}"` both match all three shards of a slot.
 Placement stays on the `rpkgs-${ARCH}` group label, so shards queue against available capacity rather than oversubscribing it.
 ### 2. Extract the loop into `local/rebuild-missing.R`
 The build is currently a single ~1 500-character `R -q -e` argument.
 Shard arithmetic and resume logic do not belong in a YAML string, and none of it is testable there.
 The loop moves to `local/rebuild-missing.R`, invoked as `Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX`, mirroring `local/build-all.R`.
 Its body is unchanged in substance: read `/tmp/rebuild_pkgs.txt`, subtract `local/excluded-packages.json`, loop with `tryCatch` around `bincraft::build_binary_package()`.
 The slice is **interleaved**, not contiguous:
 ```r
 # the list is alphabetical and build cost clusters by name (Rcpp*, Bioc*,
 # rstan*), so contiguous thirds would be badly unbalanced
 mine <- pkgs[seq(split_index, length(pkgs), by = split_into)]
 ```
 `local/build-all.R:64` uses contiguous chunks via `cut()`.
 That is fine there because its list is every CRAN package and version, so the chunks average out.
 Here the list is a filtered backlog in which expensive families sit adjacent, so interleaving is the better default.
 Interleaving also makes each shard's `[i/n]` progress representative of the slot as a whole.
 ### 3. Resume by re-deriving state from the bucket
 Before the loop, the shard performs one `s3fs::s3_dir_info()` on `devxy-rpkgs-binaries/<arch>/<codename>/latest/src/contrib` and reads the `etag` column.
 It fetches CRAN's `PACKAGES` once for the latest version and published `MD5sum` of every package.
 A package is still outstanding if and only if the object at `<pkg>_<version>.tar.gz` has an ETag equal to CRAN's `MD5sum` for that version, which is the definition `check_s3_root_package()` already applies one package at a time.
 ```r
 # one paginated listing instead of ~2900 sequential HEAD requests per shard
 info <- s3fs::s3_dir_info(slot_dir)
 etag <- setNames(gsub('^"|"$', "", info$etag), basename(info$uri))
 key <- sprintf("%s_%s.tar.gz", mine, cran_version[mine])
 # keep a package when no object exists yet, or when the object is still
 # byte-identical to CRAN's source; drop it once a real binary is published
 mine <- mine[is.na(etag[key]) | etag[key] == cran_md5[key]]
 ```
 This is the whole resume mechanism.
 There is no progress file, no volume, and no database cursor.
 A restarted shard recomputes ground truth and continues where it stopped, and it is correct even when a sibling shard, a `process-updates` cron, or a manual `just rebuild` completed something in the meantime.
 Three properties make this the right source of truth:
 - **It is what the build itself checks.** Any other store can disagree with the bucket; this one cannot.
 - **It is agent-independent.** `.crow/weekly-rebuild-missing.yaml` mounts no `volumes:`, unlike `.crow/build-all-versions.yaml:132-133`, so `/mnt/cache` is per-job and cannot carry progress anyway.
 - **It costs one listing.** `cranlike`'s `s3` fork already does exactly this call against this bucket at ~24 000 objects, so the approach is proven at the required scale.
 It must read ETags rather than the slot index's `Built` field, which is how `local/packages-to-build.R:104-130` answers the same question.
 Under this design the index is not rewritten until the dependent re-index pipeline runs (section 5), so mid-run it cannot reflect the current run's progress.
 Packages that genuinely fail to build re-publish their CRAN source, so they stay outstanding and would be retried on every restart.
 That is already handled upstream: `bincraft::filter_packages_with_errors()` (`R/build_binaries.R:1018`, `:1143`) drops anything with `error_occurred = TRUE`, and `store_build_metadata = TRUE` is passed on every call.
 No additional poison-pill filter is needed here.
 Only the flat `src/contrib` path is considered.
 The rebuild call passes no `is_r_minor_sensitive`, so it defaults to `FALSE` and only ever targets the flat path; the resume filter matches that scope deliberately.
 ### 4. Give each shard a wall-clock budget
 `local/rebuild-missing.R` takes a budget, defaulting to 20 hours, and breaks out of the loop once it is exceeded:
 ```r
 # exit cleanly rather than being killed, so the dependent re-index still runs
 if (difftime(Sys.time(), started, units = "hours") > budget_hours) {
  cat(sprintf("Budget of %sh reached after %d/%d packages; stopping cleanly\n", budget_hours, i, n))
  break
 }
 ```
 It exits 0 and reports how much of the slice it covered.
 Every run then has a terminating condition, the re-index and purge always fire, and the remainder is picked up by the next run with no bookkeeping, because section 3 recomputes the outstanding set from scratch.
 ### 5. Move the re-index and purge into `.crow/weekly-rebuild-reindex.yaml`
 Three shards per slot means three concurrent `upload_package_index()` calls on the same S3 prefix.
 `cranlike::update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work.
 The re-index steps (`.crow/weekly-rebuild-missing.yaml:171-176`) and the purge step (`:187-207`) therefore leave that file entirely.
 The new file carries:
 ```yaml
 depends_on:
  - weekly-rebuild-missing
 runs_on: [success, failure]
 ```
 `runs_on: [success, failure]` validates as a workflow-level key under `crow lint`, so a failing shard no longer withholds the re-index.
 The file uses the same 18-row matrix and the same `when:` gating as `weekly-rebuild-missing`, so it only re-indexes slots that actually ran.
 Each row re-indexes the flat slot and every per-minor slot.
 `scripts/purge_cdn_zone.sh` runs once on a single row, because all hostnames share pull zone `3857050` and 18 identical zone purges would be waste.
 ## Failure behaviour
 | case                       | today                               | after                                                 |
 | -------------------------- | ----------------------------------- | ----------------------------------------------------- |
 | one package errors         | `tryCatch` logs, loop continues     | unchanged                                             |
 | a shard fails outright     | purge runs, re-index does not       | re-index and purge run via `runs_on`                  |
 | a shard exceeds its budget | cannot happen, runs until killed    | exits 0, re-index and purge run                       |
 | a shard is killed          | nothing runs                        | still nothing; trigger the re-index pipeline alone    |
 | a shard restarts           | re-walks the list, HEAD per package | one listing, resumes at the first outstanding package |
 The known cost of `depends_on` being file-level rather than row-level: on the weekly cron no slot is re-indexed until the slowest of all 54 jobs finishes.
 The 20-hour budget bounds that at roughly one day.
 ## Out of scope
 - `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row for the platform and arch, which is precisely the source-fallback set. That is a separate change.
 - Bunny Perma-Cache eviction. `scripts/purge_cdn_zone.sh` purges the regular edge cache only; see the note in `CLAUDE.md` and issue history.
 - The audit that produces the rebuild list is unchanged.
 ## Verification
 - `crow lint .crow/` passes for both pipeline files.
 - `local/rebuild-missing.R` gets unit coverage in `local/tests/` for the two pure pieces: the interleaved slice (disjoint, covering, deterministic) and the outstanding-set filter (source-served ETag kept, binary ETag dropped, absent object kept).
 - A single-slot manual run of `alpine-324-amd64` shard 1 confirms the listing shortcut against the live bucket, and that the reported outstanding count is close to the 8 917 measured above divided by three.
 - Restarting that shard mid-run confirms it resumes rather than replaying, by comparing the outstanding count it reports on the second start.
--- a/stats/README.md
+++ b/stats/README.md
@ -0,0 +1,63 @@
 # Download statistics
 This directory turns the cran.rpkgs.com CDN access logs into the public download numbers on [rpkgs.com/stats](https://rpkgs.com/stats).
 The `download-stats` pipeline runs it every hour.
 ## Data flow
 1. `collect.ts` reads the last 70 hours of access logs from the Bunny Logging API v2 and inserts each request into `download_stats.events` in the build metadata database.
 2. `classify.ts` derives what each request was (package download, index fetch, redirect, probe, noise) from its path, status and User-Agent.
 3. `publish.ts` aggregates the events into `stats/downloads.json` in the binaries bucket and purges its CDN copy, so it is served at <https://cran.rpkgs.com/stats/downloads.json>.
 Bunny keeps access logs for three days.
 Each hourly run re-reads almost all of that window, and events are keyed by Bunny's request id, so re-reading never double counts and up to two days of failed runs lose nothing.
 Every run is recorded in `download_stats.runs`, and the dashboard marks any hour that no run covered.
 ## Privacy
 Bunny anonymizes IPs at the edge to /24 (IPv4) or /64 (IPv6) before they reach the logs.
 Not even that prefix is stored: each event keeps only `client`, an HMAC (keyed with `STATS_CLIENT_SALT`) of the prefix, the network (ASN) and the client platform from the User-Agent.
 The public JSON contains counts only, never a client hash, IP or User-Agent.
 Countries seen from fewer than three clients are folded into "other".
 ## What counts
 | Metric     | Definition                                                                                                                                                                       |
 | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | Download   | A distinct (client, package, version) served on one UTC day. Raw tarball requests are reported separately, because one CI job can re-download the same package many times a day. |
 | Client     | A distinct `client` hash that fetched an index or a package. Several machines behind one /24 count once, and one laptop on two networks counts twice, so this is an estimate.    |
 | Client-day | A client active on a given day, roughly one install session.                                                                                                                     |
 The same R session seen with and without renv's User-Agent prefix counts as one client, since `clientPlatform()` keys on the R part of the User-Agent.
 ## What is separated or excluded
 - **Cloud and CI networks** are kept but reported separately: Microsoft/GitHub, AWS, Google, Hetzner, OVH and other hosting ASNs, plus any network whose name says hosting, cloud or data centre.
 - **Our own CI runners** (`STATS_OWN_NETWORKS`) are excluded from every public number and only counted as excluded requests.
 - **Internal redirects** (`minor_routing`, `slot_resolution`, `https`) are not counted, only the request they lead to.
 - **Redirects away from us** are counted as their own metrics: `unsupported_r_minor` (demand for R versions we do not build), `archive_fallback` (old versions we have no binary for), `unknown_distro` and `macos`.
 - **Probes**: renv and pak request `__api__/status` to check for Posit Package Manager.
 - **Noise**: vulnerability scanners, crawlers and anything outside the repository layout.
 ## Changing the rules
 Change `classify.ts`, add a test case to `classify.test.ts` and bump `RULESET`.
 Then recompute every stored event:
 ```sh
 bun stats/collect.ts --reclassify
 ```
 The raw path, status, User-Agent and network are stored for exactly this reason.
 Whether a request came from our own runners is decided at ingest, so changing `STATS_OWN_NETWORKS` only affects new events.
 ## Running locally
 The secrets are in OpenBao: the Bunny key at `development/agent-dev/bunny`, the database password at `development/agent-dev/rpkgs-metadata-db` and the salt and own networks at `development/agent-dev/rpkgs-download-stats`.
 ```sh
 bun test stats
 bun stats/collect.ts
 # write the JSON locally instead of uploading it
 bun stats/publish.ts --out downloads.json
 ```
--- a/stats/classify.test.ts
+++ b/stats/classify.test.ts
@ -0,0 +1,139 @@
 import { describe, expect, test } from 'bun:test';
 import { classify, clientPlatform, parseNetworks, toEvent, tool, type LogEntry } from './classify';
 const R_RHEL = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)';
 const RENV_RHEL = 'renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)';
 const R_ALPINE = 'R/4.6.1 (Alpine Linux 3.24.1) (aarch64-unknown-linux-musl aarch64 linux-musl)';
 const R_43 = 'R (4.3.2 x86_64-pc-linux-gnu x86_64 linux-gnu)';
 const base = { status: 200, user_agent: R_RHEL, asn: 3303, asn_org: 'Swisscom', own: false };
 describe('classify', () => {
  test('flat tarball is a download with package, version and slot', () => {
    const d = classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/data.table_1.17.8.tar.gz' });
    expect(d).toMatchObject({
      kind: 'download',
      outcome: 'served',
      slot: 'amd64/rhel9',
      arch: 'amd64',
      os: 'rhel9',
      package: 'data.table',
      version: '1.17.8',
      r_minor: '4.5',
      r_version: '4.5.3',
      tool: 'R',
      network: 'other',
    });
  });
  test('per-minor tarball and index keep the slot', () => {
    expect(classify({ ...base, path: '/arm64/alpine324/latest/src/contrib/4.6/Rcpp_1.1.0.tar.gz' })).toMatchObject({
      kind: 'download',
      slot: 'arm64/alpine324',
      package: 'Rcpp',
    });
    expect(classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/4.5/PACKAGES.rds' }).kind).toBe('index');
  });
  test('index redirect with a known minor is internal minor routing', () => {
    const d = classify({ ...base, status: 302, path: '/amd64/rhel9/latest/src/contrib/PACKAGES.rds' });
    expect(d).toMatchObject({ kind: 'index', outcome: 'redirect', redirect_reason: 'minor_routing' });
  });
  test('redirect for an R minor we do not build is unsupported_r_minor', () => {
    const d = classify({ ...base, user_agent: R_43, status: 302, path: '/amd64/noble/latest/src/contrib/PACKAGES' });
    expect(d).toMatchObject({ redirect_reason: 'unsupported_r_minor', r_minor: '4.3' });
  });
  test('archived version without a binary falls back to CRAN', () => {
    const d = classify({
      ...base,
      status: 302,
      path: '/amd64/rhel9/latest/src/contrib/Archive/renv/renv_1.2.3.tar.gz',
    });
    expect(d).toMatchObject({ kind: 'download', redirect_reason: 'archive_fallback', package: 'renv' });
  });
  test('bare URL: identified distro resolves to a slot, stock R UA goes to CRAN', () => {
    expect(
      classify({ ...base, user_agent: R_ALPINE, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason,
    ).toBe('slot_resolution');
    expect(classify({ ...base, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason).toBe(
      'unknown_distro',
    );
  });
  test('renv and pak probing for Posit Package Manager is a probe', () => {
    expect(classify({ ...base, status: 404, path: '/amd64/__api__/status' }).kind).toBe('probe');
  });
  test('scanner paths are noise', () => {
    for (const path of ['/.env', '/wp-content/plugins/x.php', '/.git/config', '/xwpg.php']) {
      expect(classify({ ...base, status: 404, user_agent: 'Mozilla/5.0', path }).kind).toBe('noise');
    }
  });
  test('the stats file is not counted as repository use', () => {
    expect(classify({ ...base, path: '/stats/downloads.json' }).kind).toBe('stats');
  });
  test('cloud networks are labelled, own networks win over cloud', () => {
    expect(classify({ ...base, asn: 8075, asn_org: 'Microsoft Corporation', path: '/' }).network).toBe('cloud');
    expect(classify({ ...base, asn: null, asn_org: 'Some Hosting Ltd', path: '/' }).network).toBe('cloud');
    expect(classify({ ...base, asn: 24940, own: true, path: '/' }).network).toBe('own');
  });
  test('query strings and doubled slashes do not change the result', () => {
    expect(classify({ ...base, path: '/amd64//rhel9/latest/src/contrib/PACKAGES.rds?x=1' }).kind).toBe('index');
  });
 });
 describe('client identity', () => {
  test('renv and plain R from the same machine share one platform key', () => {
    expect(clientPlatform(RENV_RHEL)).toBe(clientPlatform(R_RHEL));
    expect(clientPlatform(R_ALPINE)).not.toBe(clientPlatform(R_RHEL));
  });
  test('tool detection', () => {
    expect(tool(RENV_RHEL)).toBe('renv');
    expect(tool(R_RHEL)).toBe('R');
    expect(tool('pak/0.9.0 R/4.5.1 (...)')).toBe('pak');
    expect(tool('curl/8.5.0')).toBe('curl');
    expect(tool('-')).toBe('none');
    expect(tool('Mozilla/5.0 (Windows NT 10.0)')).toBe('browser');
  });
  test('own networks match by prefix for IPv4 and IPv6', () => {
    const own = parseNetworks('203.0.113.0/24, 2001:db8:1:a100::/64');
    const matches = (ip: string) => own.some((m) => m(ip));
    expect(matches('203.0.113.0')).toBe(true);
    expect(matches('203.0.114.0')).toBe(false);
    expect(matches('2001:db8:1:a100::')).toBe(true);
    expect(matches('2001:db8:1:a101::')).toBe(false);
    expect(parseNetworks('')).toHaveLength(0);
  });
  test('toEvent hashes the client and never keeps the IP', () => {
    const entry: LogEntry = {
      timestamp: '2026-11-14T11:45:33.689+00:00',
      pullZoneId: 3857050,
      requestId: 'abc',
      cacheStatus: 'HIT',
      statusCode: 200,
      bytesSent: 1000,
      remoteIp: '203.0.113.0',
      countryCode: 'CH',
      host: 'cran.rpkgs.com',
      path: '/amd64/rhel9/latest/src/contrib/rlang_1.1.6.tar.gz',
      userAgent: RENV_RHEL,
      asn: 3303,
      asnOrganization: 'Swisscom',
    };
    const event = toEvent(entry, (key) => `h(${key.length})`, parseNetworks('203.0.113.0/24'));
    expect(event).not.toBeNull();
    expect(JSON.stringify(event)).not.toContain('203.0.113');
    expect(event!.own).toBe(true);
    expect(event!.network).toBe('own');
    expect(toEvent({ ...entry, requestId: null }, (k) => k, [])).toBeNull();
  });
 });
--- a/stats/classify.ts
+++ b/stats/classify.ts
@ -0,0 +1,410 @@
 /**
 * Turns one Bunny access-log entry into a classified download event.
 *
 * Everything here is pure so the rules can be unit-tested and re-run over the
 * stored events (`collect.ts --reclassify`) whenever they change. Bump RULESET
 * on any change that alters a derived column, so the public JSON can say which
 * rules produced it.
 *
 * The rules mirror `edge/rpkgs-router.ts`: a redirect is classified by what the
 * router would have done with that path and User-Agent, because the Location
 * header is not in the logs.
 */
 export const RULESET = 1;
 /** R minors with a per-minor slot; keep in sync with KNOWN_MINORS in cdn.tf. */
 export const KNOWN_MINORS = new Set(['4.4', '4.5', '4.6']);
 export type Kind =
  /** A package tarball. */
  | 'download'
  /** `PACKAGES*`: one per repository per `install.packages()` call. */
  | 'index'
  /** Other repository metadata, e.g. `Meta/archive.rds`. */
  | 'metadata'
  /** renv and pak asking whether this is a Posit Package Manager (`__api__/…`). */
  | 'probe'
  /** The repository root or `/src/contrib` without a file. */
  | 'root'
  /** Our own `stats/` files, so the dashboard does not count itself. */
  | 'stats'
  /** Anything else: vulnerability scanners, crawlers, typos. */
  | 'noise';
 export type Outcome = 'served' | 'redirect' | 'error';
 export type RedirectReason =
  /** Internal hop: index sent to its per-R-minor slot. Not a user-visible event. */
  | 'minor_routing'
  /** Internal hop: bare `cran.rpkgs.com` URL resolved to a slot. */
  | 'slot_resolution'
  /** Client runs an R minor we do not build for; sent to CRAN for source. */
  | 'unsupported_r_minor'
  /** Bare URL from a distro we could not identify; sent to CRAN. */
  | 'unknown_distro'
  /** Archived version we have no binary for; sent to the CRAN source tarball. */
  | 'archive_fallback'
  /** macOS client; sent to CRAN's macOS binaries. */
  | 'macos'
  /** `http://` to `https://`. */
  | 'https'
  | 'other';
 export type Tool = 'R' | 'renv' | 'pak' | 'rig' | 'curl' | 'wget' | 'python' | 'browser' | 'none' | 'other';
 export type Network = 'cloud' | 'other' | 'own';
 export interface LogEntry {
  timestamp: string;
  pullZoneId: number;
  requestId: string | null;
  cacheStatus: string | null;
  statusCode: number;
  bytesSent: number;
  remoteIp: string | null;
  countryCode: string | null;
  host: string | null;
  path: string | null;
  userAgent: string | null;
  asn: number | null;
  asnOrganization: string | null;
 }
 export interface Derived {
  ruleset: number;
  kind: Kind;
  outcome: Outcome;
  redirect_reason: RedirectReason | null;
  slot: string | null;
  arch: string | null;
  os: string | null;
  r_minor: string | null;
  r_version: string | null;
  package: string | null;
  version: string | null;
  tool: Tool;
  network: Network;
 }
 /** The raw columns kept per event; everything Derived is recomputed from these. */
 export interface RawEvent {
  request_id: string;
  ts: string;
  zone_id: number;
  host: string | null;
  status: number;
  cache_status: string | null;
  bytes: number;
  path: string;
  user_agent: string | null;
  country: string | null;
  asn: number | null;
  asn_org: string | null;
  /** HMAC of the (already anonymized) IP prefix, network and client platform. */
  client: string;
  /** Matched one of our own networks at ingest. The IP itself is not kept. */
  own: boolean;
 }
 export type Event = RawEvent & Derived;
 const SLOT_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib(?:\/(.*))?$/;
 const BARE_REGEX = /^\/src\/contrib(?:\/(.*))?$/;
 const MACOS_REGEX = /^\/bin\/macosx\/[^/]+\/contrib\/(\d+\.\d+)\/(.+)$/;
 const MINOR_DIR_REGEX = /^(\d+\.\d+)\/(.*)$/;
 const INDEX_REGEX = /^PACKAGES(\.gz|\.rds|\.json)?$/;
 const TARBALL_REGEX = /(?:^|\/)([A-Za-z][A-Za-z0-9.]*)_([0-9][^/_]*)\.(?:tar\.gz|tgz)$/;
 const R_VERSION_REGEXES = [/\bR\/(\d+\.\d+\.\d+)/, /\bR \((\d+\.\d+\.\d+)/];
 const ARCH_REGEX = /\b(x86_64|aarch64|arm64)\b/;
 /** The distro forms `parseSlot()` in the router recognises. */
 const DISTRO_REGEX = /(almalinux|rocky)[^\d]*\d+|Ubuntu \d+\.\d+|Alpine Linux|alpine-\d/i;
 /**
 * Networks whose clients are almost always machines: public clouds, CI
 * providers and hosting companies. A request from one of these is reported as
 * "cloud / CI" rather than dropped, since a CI pipeline is a real user too,
 * just not a person at a laptop.
 */
 const CLOUD_ASNS = new Map<number, string>([
  [8075, 'Microsoft (incl. GitHub Actions runners)'],
  [8068, 'Microsoft'],
  [8069, 'Microsoft'],
  [36459, 'GitHub'],
  [16509, 'Amazon'],
  [14618, 'Amazon'],
  [15169, 'Google'],
  [396982, 'Google Cloud'],
  [24940, 'Hetzner'],
  [213230, 'Hetzner Cloud'],
  [212317, 'Hetzner'],
  [16276, 'OVH'],
  [14061, 'DigitalOcean'],
  [51167, 'Contabo'],
  [40021, 'Contabo US'],
  [63949, 'Akamai / Linode'],
  [31898, 'Oracle'],
  [45102, 'Alibaba'],
  [37963, 'Alibaba'],
  [132203, 'Tencent'],
  [45090, 'Tencent'],
  [13335, 'Cloudflare'],
  [62240, 'Clouvider'],
  [12876, 'Scaleway'],
  [20473, 'Vultr'],
  [197540, 'netcup'],
  [9009, 'M247'],
  [60068, 'Datacamp / CDN77'],
  [36352, 'ColoCrossing'],
  [47583, 'Hostinger'],
  [207990, 'HostRoyale'],
 ]);
 const CLOUD_ORG_REGEX = /\b(hosting|cloud|server(s)?|data ?cent(er|re)|colo(cation)?|vps|dedicated)\b/i;
 export function isCloudNetwork(asn: number | null, org: string | null): boolean {
  if (asn !== null && CLOUD_ASNS.has(asn)) return true;
  return org !== null && CLOUD_ORG_REGEX.test(org);
 }
 export function normalizePath(path: string): string {
  const withoutQuery = path.split('?')[0] ?? '';
  return withoutQuery.replace(/\/{2,}/g, '/');
 }
 export function rVersion(userAgent: string): string | null {
  for (const regex of R_VERSION_REGEXES) {
    const match = userAgent.match(regex);
    if (match) return match[1];
  }
  return null;
 }
 export function rMinor(version: string | null): string | null {
  if (!version) return null;
  const [major, minor] = version.split('.');
  return `${major}.${minor}`;
 }
 export function tool(userAgent: string): Tool {
  const ua = userAgent.trim();
  if (ua === '' || ua === '-') return 'none';
  if (/^renv\b/i.test(ua)) return 'renv';
  if (/\bpak\b/i.test(ua)) return 'pak';
  if (/\brig\b/i.test(ua)) return 'rig';
  if (/^R[ /(]/.test(ua)) return 'R';
  if (/^curl\//i.test(ua)) return 'curl';
  if (/^wget\//i.test(ua)) return 'wget';
  if (/python|aiohttp|httpx/i.test(ua)) return 'python';
  if (/^Mozilla\//.test(ua)) return 'browser';
  return 'other';
 }
 /**
 * The part of the User-Agent that identifies the machine rather than the tool.
 *
 * `renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` and
 * `R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` are the same R session, one going
 * through renv and one not. Keying distinct clients on the full User-Agent
 * would count that machine twice.
 */
 export function clientPlatform(userAgent: string): string {
  const version = rVersion(userAgent);
  if (!version) return userAgent;
  const rPart = userAgent.slice(userAgent.search(/\bR[/ ]\(?\d/));
  return rPart.replace(/\bR\/\d+\.\d+\.\d+\s*/, '').trim() || version;
 }
 function slotParts(slot: string | null): { arch: string | null; os: string | null } {
  if (!slot) return { arch: null, os: null };
  const [arch, os] = slot.split('/');
  return { arch: arch ?? null, os: os ?? null };
 }
 function archFromUserAgent(userAgent: string): string | null {
  const match = userAgent.match(ARCH_REGEX);
  if (!match) return null;
  return match[1] === 'x86_64' ? 'amd64' : 'arm64';
 }
 function fileKind(rest: string): Kind {
  if (rest === '') return 'root';
  if (INDEX_REGEX.test(rest)) return 'index';
  if (TARBALL_REGEX.test(rest)) return 'download';
  if (/^Meta\//.test(rest) || /^PACKAGES/.test(rest)) return 'metadata';
  return 'noise';
 }
 function outcome(status: number): Outcome {
  if (status === 301 || status === 302 || status === 303 || status === 307 || status === 308) return 'redirect';
  if (status >= 200 && status < 400) return 'served';
  return 'error';
 }
 export function classify(
  raw: Pick<RawEvent, 'path' | 'status' | 'user_agent' | 'asn' | 'asn_org' | 'own'>,
  scheme = 'https',
 ): Derived {
  const path = normalizePath(raw.path);
  const userAgent = raw.user_agent ?? '';
  const version = rVersion(userAgent);
  const minor = rMinor(version);
  const result: Derived = {
    ruleset: RULESET,
    kind: 'noise',
    outcome: outcome(raw.status),
    redirect_reason: null,
    slot: null,
    arch: null,
    os: null,
    r_minor: minor,
    r_version: version,
    package: null,
    version: null,
    tool: tool(userAgent),
    network: raw.own ? 'own' : isCloudNetwork(raw.asn, raw.asn_org) ? 'cloud' : 'other',
  };
  let rest: string | null = null;
  let area: 'slot' | 'bare' | 'macos' | null = null;
  const slot = path.match(SLOT_REGEX);
  const bare = path.match(BARE_REGEX);
  const macos = path.match(MACOS_REGEX);
  if (path.includes('/__api__/')) {
    result.kind = 'probe';
  } else if (path.startsWith('/stats/')) {
    result.kind = 'stats';
  } else if (path === '/') {
    result.kind = 'root';
    area = 'bare';
  } else if (slot) {
    area = 'slot';
    result.slot = `${slot[1]}/${slot[2]}`;
    rest = slot[3] ?? '';
  } else if (bare) {
    area = 'bare';
    rest = bare[1] ?? '';
  } else if (macos) {
    area = 'macos';
    result.os = 'macos';
    rest = macos[2];
  }
  if (rest !== null) {
    const minorDir = rest.match(MINOR_DIR_REGEX);
    const file = minorDir ? minorDir[2] : rest;
    result.kind = fileKind(file);
    const tarball = file.match(TARBALL_REGEX);
    if (tarball) {
      result.package = tarball[1];
      result.version = tarball[2];
    }
  }
  const parts = slotParts(result.slot);
  result.arch = parts.arch ?? archFromUserAgent(userAgent);
  result.os = parts.os ?? result.os;
  if (result.outcome === 'redirect') {
    result.redirect_reason = redirectReason(raw.status, area, rest, minor, scheme, userAgent);
  }
  return result;
 }
 function redirectReason(
  status: number,
  area: 'slot' | 'bare' | 'macos' | null,
  rest: string | null,
  minor: string | null,
  scheme: string,
  userAgent: string,
 ): RedirectReason {
  if (status === 301 || scheme === 'http') return 'https';
  if (area === 'macos' || (area === 'bare' && /darwin/.test(userAgent))) return 'macos';
  if (minor !== null && !KNOWN_MINORS.has(minor) && (area === 'slot' || area === 'bare')) {
    return 'unsupported_r_minor';
  }
  if (area === 'slot') {
    if (rest !== null && rest.startsWith('Archive/')) return 'archive_fallback';
    if (rest !== null && INDEX_REGEX.test(rest)) return 'minor_routing';
    return 'other';
  }
  if (area === 'bare') {
    // The router resolves a bare URL only when it can identify the distro from
    // the User-Agent; otherwise it sends the client to CRAN. A stock R
    // User-Agent (`R (4.5.3 x86_64-pc-linux-gnu …)`) names no distro.
    return DISTRO_REGEX.test(userAgent) && ARCH_REGEX.test(userAgent) ? 'slot_resolution' : 'unknown_distro';
  }
  return 'other';
 }
 /** Parse `"a.b.c.0/24,2001:db8::/64"` into prefix matchers for our own networks. */
 export function parseNetworks(spec: string): ((ip: string) => boolean)[] {
  return spec
    .split(',')
    .map((cidr) => cidr.trim())
    .filter((cidr) => cidr.length > 0)
    .map((cidr) => {
      const [base, bitsText] = cidr.split('/');
      const v6 = base.includes(':');
      const bits = Number(bitsText ?? (v6 ? 128 : 32));
      const baseBits = ipBits(base);
      return (ip: string) => {
        if (ip.includes(':') !== v6) return false;
        const candidate = ipBits(ip);
        return baseBits !== null && candidate !== null && candidate.slice(0, bits) === baseBits.slice(0, bits);
      };
    });
 }
 function ipBits(ip: string): string | null {
  if (!ip.includes(':')) {
    const octets = ip.split('.').map(Number);
    if (octets.length !== 4 || octets.some((o) => !Number.isInteger(o) || o < 0 || o > 255)) return null;
    return octets.map((o) => o.toString(2).padStart(8, '0')).join('');
  }
  const [head, tail] = ip.split('::');
  const headGroups = head ? head.split(':') : [];
  const tailGroups = tail ? tail.split(':') : [];
  const missing = 8 - headGroups.length - tailGroups.length;
  if (ip.includes('::') ? missing < 0 : missing !== 0) return null;
  const groups = [...headGroups, ...Array(Math.max(missing, 0)).fill('0'), ...tailGroups];
  return groups
    .map((g) =>
      parseInt(g || '0', 16)
        .toString(2)
        .padStart(16, '0'),
    )
    .join('');
 }
 /** Build the stored event. Returns null for entries without a request id. */
 export function toEvent(
  entry: LogEntry,
  hashClient: (key: string) => string,
  ownNetworks: ((ip: string) => boolean)[],
 ): Event | null {
  if (!entry.requestId) return null;
  const ip = entry.remoteIp ?? '';
  const userAgent = entry.userAgent ?? '';
  const raw: RawEvent = {
    request_id: entry.requestId,
    ts: entry.timestamp,
    zone_id: entry.pullZoneId,
    host: entry.host,
    status: entry.statusCode,
    cache_status: entry.cacheStatus,
    bytes: entry.bytesSent,
    path: entry.path ?? '',
    user_agent: entry.userAgent,
    country: entry.countryCode,
    asn: entry.asn,
    asn_org: entry.asnOrganization,
    client: hashClient(`${ip}|${entry.asn ?? ''}|${clientPlatform(userAgent)}`),
    own: ip !== '' && ownNetworks.some((matches) => matches(ip)),
  };
  return { ...raw, ...classify(raw) };
 }
--- a/stats/collect.ts
+++ b/stats/collect.ts
@ -0,0 +1,157 @@
 /**
 * Pull the CDN access logs into `download_stats.events`.
 *
 *   bun stats/collect.ts               # fetch the last 70 hours, insert what is new
 *   bun stats/collect.ts --reclassify  # recompute derived columns after a rule change
 *
 * Bunny keeps logs for three days, so every run re-reads almost all of that
 * window. Events are keyed by Bunny's request id, which makes re-reading free
 * of double counts, and a run that fails is simply covered by the next one.
 *
 * Environment: BUNNYNET_API_KEY, PGPASS, STATS_CLIENT_SALT, and optionally
 * STATS_PULL_ZONE (default: cran-rpkgs) and STATS_OWN_NETWORKS (comma-separated
 * CIDRs of our own CI runners, kept out of the public numbers).
 */
 import { classify, parseNetworks, toEvent, type Event, type LogEntry } from './classify';
 import { connect, migrate } from './db';
 const PULL_ZONE = Number(process.env.STATS_PULL_ZONE ?? 3857050);
 const WINDOW_HOURS = 70;
 const PAGE_SIZE = 10000;
 const COLUMNS = [
  'request_id',
  'ts',
  'zone_id',
  'host',
  'status',
  'cache_status',
  'bytes',
  'path',
  'user_agent',
  'country',
  'asn',
  'asn_org',
  'client',
  'own',
  'ruleset',
  'kind',
  'outcome',
  'redirect_reason',
  'slot',
  'arch',
  'os',
  'r_minor',
  'r_version',
  'package',
  'version',
  'tool',
  'network',
 ] as const;
 function requireEnv(name: string): string {
  const value = process.env[name];
  if (!value) throw new Error(`${name} is not set`);
  return value;
 }
 async function fetchPage(
  apiKey: string,
  from: Date,
  to: Date,
  offset: number,
 ): Promise<{ data: LogEntry[]; hasMore: boolean }> {
  const url = new URL(`https://logging.bunnycdn.com/v2/pullzones/${PULL_ZONE}/logs`);
  url.searchParams.set('from', from.toISOString());
  url.searchParams.set('to', to.toISOString());
  url.searchParams.set('order', 'asc');
  url.searchParams.set('limit', String(PAGE_SIZE));
  url.searchParams.set('offset', String(offset));
  for (let attempt = 1; ; attempt++) {
    const response = await fetch(url, { headers: { AccessKey: apiKey, Accept: 'application/json' } });
    if (response.ok) {
      const body = (await response.json()) as { data: LogEntry[]; pagination: { hasMore: boolean } };
      return { data: body.data, hasMore: body.pagination.hasMore };
    }
    if ((response.status === 429 || response.status >= 500) && attempt < 6) {
      await Bun.sleep(2000 * attempt);
      continue;
    }
    throw new Error(`Bunny logging API returned ${response.status}: ${await response.text()}`);
  }
 }
 async function collect(): Promise<void> {
  const apiKey = requireEnv('BUNNYNET_API_KEY');
  const salt = requireEnv('STATS_CLIENT_SALT');
  const ownNetworks = parseNetworks(process.env.STATS_OWN_NETWORKS ?? '');
  const hashClient = (key: string) => new Bun.CryptoHasher('sha256', salt).update(key).digest('hex').slice(0, 20);
  // Fixing `to` up front keeps offset pagination stable while new requests
  // keep arriving.
  const to = new Date();
  const from = new Date(to.getTime() - WINDOW_HOURS * 3600 * 1000);
  const events: Event[] = [];
  let fetched = 0;
  for (let offset = 0; ;) {
    const page = await fetchPage(apiKey, from, to, offset);
    fetched += page.data.length;
    for (const entry of page.data) {
      const event = toEvent(entry, hashClient, ownNetworks);
      if (event) events.push(event);
    }
    if (!page.hasMore || page.data.length === 0) break;
    offset += page.data.length;
  }
  const sql = connect();
  await migrate(sql);
  let inserted = 0;
  for (let i = 0; i < events.length; i += 1000) {
    const batch = events.slice(i, i + 1000);
    const rows = await sql`
      insert into download_stats.events ${sql(batch, ...COLUMNS)}
      on conflict (request_id) do nothing
      returning 1`;
    inserted += rows.length;
  }
  await sql`
    insert into download_stats.runs (zone_id, window_from, window_to, fetched, inserted)
    values (${PULL_ZONE}, ${from}, ${to}, ${fetched}, ${inserted})`;
  await sql.close();
  console.log(
    `fetched ${fetched} log entries from ${from.toISOString()} to ${to.toISOString()}, inserted ${inserted} new`,
  );
 }
 async function reclassify(): Promise<void> {
  const sql = connect();
  await migrate(sql);
  const rows: Event[] =
    await sql`select request_id, path, status, user_agent, asn, asn_org, own from download_stats.events`;
  let updated = 0;
  for (let i = 0; i < rows.length; i += 1000) {
    const batch = rows.slice(i, i + 1000).map((row) => ({ request_id: row.request_id, ...classify(row) }));
    await sql.begin(async (tx) => {
      for (const row of batch) {
        await tx`
          update download_stats.events set
            ruleset = ${row.ruleset}, kind = ${row.kind}, outcome = ${row.outcome},
            redirect_reason = ${row.redirect_reason}, slot = ${row.slot}, arch = ${row.arch}, os = ${row.os},
            r_minor = ${row.r_minor}, r_version = ${row.r_version}, package = ${row.package},
            version = ${row.version}, tool = ${row.tool}, network = ${row.network}
          where request_id = ${row.request_id}`;
      }
    });
    updated += batch.length;
  }
  await sql.close();
  console.log(`reclassified ${updated} events`);
 }
 if (import.meta.main) {
  await (process.argv.includes('--reclassify') ? reclassify() : collect());
 }
--- a/stats/db.ts
+++ b/stats/db.ts
@ -0,0 +1,75 @@
 import { SQL } from 'bun';
 /**
 * Download events live in their own `download_stats` schema in the build
 * metadata database, next to (but never joined with) `single_builds`.
 *
 * No IP address is stored, not even the /24 prefix Bunny logs: only a salted
 * HMAC of it (`client`) and whether it matched one of our own networks.
 */
 const SCHEMA = `
 create schema if not exists download_stats;
 create table if not exists download_stats.events (
  request_id      text primary key,
  ts              timestamptz not null,
  zone_id         integer not null,
  host            text,
  status          smallint not null,
  cache_status    text,
  bytes           bigint not null,
  path            text not null,
  user_agent      text,
  country         text,
  asn             integer,
  asn_org         text,
  client          text not null,
  own             boolean not null,
  ruleset         smallint not null,
  kind            text not null,
  outcome         text not null,
  redirect_reason text,
  slot            text,
  arch            text,
  os              text,
  r_minor         text,
  r_version       text,
  package         text,
  version         text,
  tool            text not null,
  network         text not null
 );
 create index if not exists events_ts_idx on download_stats.events (ts);
 create index if not exists events_package_idx on download_stats.events (package) where kind = 'download';
 -- One row per collection run. The union of successful windows is what the
 -- dashboard reports as covered; anything outside it is shown as a gap.
 create table if not exists download_stats.runs (
  id          bigserial primary key,
  zone_id     integer not null,
  started_at  timestamptz not null default now(),
  window_from timestamptz not null,
  window_to   timestamptz not null,
  fetched     integer not null,
  inserted    integer not null
 );
 `;
 export function connect(): SQL {
  const password = process.env.PGPASS;
  if (!password) throw new Error('PGPASS is not set');
  return new SQL({
    hostname: process.env.PGHOST ?? 'r-binaries.devxy.io',
    port: Number(process.env.PGPORT ?? 15432),
    database: process.env.PGDATABASE ?? 'build_metadata',
    username: process.env.PGUSER ?? 'rpkgs',
    password,
    tls: true,
    max: 2,
  });
 }
 export async function migrate(sql: SQL): Promise<void> {
  await sql.unsafe(SCHEMA);
 }
--- a/stats/publish.ts
+++ b/stats/publish.ts
@ -0,0 +1,339 @@
 /**
 * Aggregate `download_stats.events` into the public `stats/downloads.json`.
 *
 *   bun stats/publish.ts --out downloads.json   # write locally only
 *   bun stats/publish.ts                        # upload to the bucket and purge the CDN copy
 *
 * The output holds counts only. Distinct clients are counted from the stored
 * HMAC, which never leaves the database, and countries seen from fewer than
 * MIN_CLIENTS clients are folded into "other" so no single client can be
 * picked out by where it is.
 *
 * Every number excludes our own CI runners. Cloud and CI networks are kept but
 * reported separately, since one pipeline can outweigh every person combined.
 */
 import { S3Client, type SQL } from 'bun';
 import { KNOWN_MINORS, RULESET } from './classify';
 import { connect, migrate } from './db';
 const MIN_CLIENTS = 3;
 const PUBLIC_URL = 'https://cran.rpkgs.com/stats/downloads.json';
 const OBJECT_KEY = 'stats/downloads.json';
 /** Split of a count by network class. */
 export interface Split {
  total: number;
  cloud: number;
  other: number;
 }
 export interface DayStats {
  date: string;
  /** Hours of this UTC day for which logs were collected (24 = complete). */
  coveredHours: number;
  /** Tarball requests served, before any de-duplication. */
  requests: number;
  /** Distinct (client, package, version) served on the day. */
  downloads: Split;
  /** Distinct clients that fetched an index or a package. */
  clients: Split;
  /** Distinct clients sent to CRAN because we do not build for their R minor. */
  unsupportedMinorClients: number;
  bytes: number;
 }
 export interface WindowStats {
  days: number;
  downloads: Split;
  /** Distinct clients across the whole window. */
  clients: Split;
  /** Sum of daily distinct clients: roughly, install sessions. */
  clientDays: Split;
 }
 export interface Breakdown {
  key: string;
  clients: number;
  downloads: number;
 }
 export interface PackageStats {
  name: string;
  downloads30d: number;
  clients30d: number;
  downloadsAll: number;
 }
 export interface PublicStats {
  schema: 1;
  ruleset: number;
  generatedAt: string;
  firstDay: string | null;
  lastDay: string | null;
  supportedMinors: string[];
  daily: DayStats[];
  windows: { last7: WindowStats; last30: WindowStats; all: WindowStats };
  /** Distribution of distinct packages fetched per client per day (30 days). */
  packagesPerClientDay: { median: number; p90: number };
  /** Last 30 days. */
  breakdowns: {
    network: Breakdown[];
    os: Breakdown[];
    arch: Breakdown[];
    rMinor: Breakdown[];
    tool: Breakdown[];
    country: Breakdown[];
  };
  /** Last 30 days: clients we sent elsewhere, and why. */
  redirects: { reason: string; clients: number; requests: number }[];
  unsupportedMinors: Breakdown[];
  /** Last 30 days: requests that were not repository use at all. */
  excluded: { ownRunners: number; probes: number; noise: number; errors: number };
  packages: PackageStats[];
 }
 const split = (row: Record<string, unknown>, prefix: string): Split => ({
  total: Number(row[`${prefix}_total`] ?? 0),
  cloud: Number(row[`${prefix}_cloud`] ?? 0),
  other: Number(row[`${prefix}_other`] ?? 0),
 });
 async function coverage(sql: SQL): Promise<Map<string, number>> {
  const runs: { window_from: Date; window_to: Date }[] =
    await sql`select window_from, window_to from download_stats.runs order by window_from`;
  const merged: [number, number][] = [];
  for (const run of runs) {
    const from = run.window_from.getTime();
    const to = run.window_to.getTime();
    const last = merged.at(-1);
    if (last && from <= last[1]) last[1] = Math.max(last[1], to);
    else merged.push([from, to]);
  }
  const hours = new Map<string, number>();
  const DAY = 86400000;
  for (const [from, to] of merged) {
    for (let day = Math.floor(from / DAY) * DAY; day < to; day += DAY) {
      const overlap = Math.min(to, day + DAY) - Math.max(from, day);
      const key = new Date(day).toISOString().slice(0, 10);
      hours.set(key, (hours.get(key) ?? 0) + overlap / 3600000);
    }
  }
  return hours;
 }
 async function windowStats(sql: SQL, days: number | null): Promise<WindowStats> {
  const since = days === null ? new Date(0) : new Date(Date.now() - days * 86400000);
  const [row] = await sql`
    with ev as (
      select client, network, kind, package, version, (ts at time zone 'UTC')::date as day
      from download_stats.events
      where network <> 'own' and outcome = 'served' and kind in ('download', 'index') and ts >= ${since}
    ),
    dl as (select distinct client, network, package, version, day from ev where kind = 'download'),
    cd as (select distinct client, network, day from ev)
    select
      (select count(*) from dl) as downloads_total,
      (select count(*) from dl where network = 'cloud') as downloads_cloud,
      (select count(*) from dl where network = 'other') as downloads_other,
      (select count(distinct client) from ev) as clients_total,
      (select count(distinct client) from ev where network = 'cloud') as clients_cloud,
      (select count(distinct client) from ev where network = 'other') as clients_other,
      (select count(*) from cd) as client_days_total,
      (select count(*) from cd where network = 'cloud') as client_days_cloud,
      (select count(*) from cd where network = 'other') as client_days_other,
      (select min(day) from ev) as first_day`;
  const firstDay = row.first_day ? new Date(row.first_day) : null;
  const span = firstDay ? Math.max(1, Math.ceil((Date.now() - firstDay.getTime()) / 86400000)) : 0;
  return {
    days: days === null ? span : Math.min(days, span),
    downloads: split(row, 'downloads'),
    clients: split(row, 'clients'),
    clientDays: split(row, 'client_days'),
  };
 }
 async function breakdown(sql: SQL, column: string, since: Date): Promise<Breakdown[]> {
  const rows: { key: string | null; clients: number; downloads: number }[] = await sql`
    with ev as (
      select ${sql(column)}::text as key, client, kind, package, version, (ts at time zone 'UTC')::date as day
      from download_stats.events
      where network <> 'own' and outcome = 'served' and kind in ('download', 'index') and ts >= ${since}
    )
    select key, count(distinct client)::int as clients,
      (select count(*) from (select distinct client, package, version, day from ev e2
        where e2.kind = 'download' and e2.key is not distinct from ev.key) d)::int as downloads
    from ev group by key order by clients desc`;
  return rows.map((row) => ({ key: row.key ?? 'unknown', clients: row.clients, downloads: row.downloads }));
 }
 /** Fold rows below the client threshold into one "other" row. */
 function suppress(rows: Breakdown[]): Breakdown[] {
  const kept = rows.filter((row) => row.clients >= MIN_CLIENTS);
  const folded = rows.filter((row) => row.clients < MIN_CLIENTS);
  if (folded.length === 0) return kept;
  return [
    ...kept,
    {
      key: 'other',
      clients: folded.reduce((sum, row) => sum + row.clients, 0),
      downloads: folded.reduce((sum, row) => sum + row.downloads, 0),
    },
  ];
 }
 export async function buildStats(sql: SQL): Promise<PublicStats> {
  const since30 = new Date(Date.now() - 30 * 86400000);
  const covered = await coverage(sql);
  const dailyRows: Record<string, unknown>[] = await sql`
    with ev as (
      select *, (ts at time zone 'UTC')::date as day from download_stats.events where network <> 'own'
    ),
    served as (select * from ev where outcome = 'served' and kind in ('download', 'index')),
    dl as (select distinct client, network, package, version, day from served where kind = 'download'),
    cl as (select distinct client, network, day from served)
    select d.day::text as date,
      (select count(*) from served s where s.day = d.day and s.kind = 'download')::int as requests,
      (select count(*) from dl where dl.day = d.day)::int as downloads_total,
      (select count(*) from dl where dl.day = d.day and network = 'cloud')::int as downloads_cloud,
      (select count(*) from dl where dl.day = d.day and network = 'other')::int as downloads_other,
      (select count(*) from cl where cl.day = d.day)::int as clients_total,
      (select count(*) from cl where cl.day = d.day and network = 'cloud')::int as clients_cloud,
      (select count(*) from cl where cl.day = d.day and network = 'other')::int as clients_other,
      (select count(distinct client) from ev e where e.day = d.day and e.redirect_reason = 'unsupported_r_minor')::int
        as unsupported,
      (select coalesce(sum(bytes), 0) from served s where s.day = d.day)::bigint as bytes
    from (select distinct day from ev) d order by d.day`;
  const daily: DayStats[] = dailyRows.map((row) => ({
    date: String(row.date),
    coveredHours: Math.round(Math.min(24, covered.get(String(row.date)) ?? 0) * 10) / 10,
    requests: Number(row.requests),
    downloads: split(row, 'downloads'),
    clients: split(row, 'clients'),
    unsupportedMinorClients: Number(row.unsupported),
    bytes: Number(row.bytes),
  }));
  const [perSession] = await sql`
    with per as (
      select client, (ts at time zone 'UTC')::date as day, count(distinct (package, version)) as n
      from download_stats.events
      where network <> 'own' and outcome = 'served' and kind = 'download' and ts >= ${since30}
      group by 1, 2
    )
    select coalesce(percentile_cont(0.5) within group (order by n), 0) as median,
      coalesce(percentile_cont(0.9) within group (order by n), 0) as p90
    from per`;
  const redirects: { reason: string; clients: number; requests: number }[] = await sql`
    select redirect_reason as reason, count(distinct client)::int as clients, count(*)::int as requests
    from download_stats.events
    where network <> 'own' and outcome = 'redirect' and ts >= ${since30}
      and redirect_reason not in ('minor_routing', 'slot_resolution', 'https')
    group by 1 order by clients desc`;
  const unsupportedMinors: Breakdown[] = await sql`
    select r_minor as key, count(distinct client)::int as clients, count(*)::int as downloads
    from download_stats.events
    where network <> 'own' and redirect_reason = 'unsupported_r_minor' and ts >= ${since30}
    group by 1 order by clients desc`;
  const [excluded] = await sql`
    select
      count(*) filter (where network = 'own')::int as own,
      count(*) filter (where network <> 'own' and kind = 'probe')::int as probes,
      count(*) filter (where network <> 'own' and kind = 'noise')::int as noise,
      count(*) filter (where network <> 'own' and kind in ('download', 'index') and outcome = 'error')::int as errors
    from download_stats.events where ts >= ${since30}`;
  const packages: PackageStats[] = (
    await sql`
      with dl as (
        select distinct client, package, version, (ts at time zone 'UTC')::date as day, ts >= ${since30} as recent
        from download_stats.events
        where network <> 'own' and outcome = 'served' and kind = 'download' and package is not null
      )
      select package as name,
        count(*) filter (where recent)::int as downloads30d,
        count(distinct client) filter (where recent)::int as clients30d,
        count(*)::int as downloadsall
      from dl group by package order by downloads30d desc, downloadsall desc, name`
  ).map((row: Record<string, unknown>) => ({
    name: String(row.name),
    downloads30d: Number(row.downloads30d),
    clients30d: Number(row.clients30d),
    downloadsAll: Number(row.downloadsall),
  }));
  return {
    schema: 1,
    ruleset: RULESET,
    generatedAt: new Date().toISOString(),
    firstDay: daily[0]?.date ?? null,
    lastDay: daily.at(-1)?.date ?? null,
    supportedMinors: [...KNOWN_MINORS],
    daily,
    windows: {
      last7: await windowStats(sql, 7),
      last30: await windowStats(sql, 30),
      all: await windowStats(sql, null),
    },
    packagesPerClientDay: { median: Number(perSession.median), p90: Number(perSession.p90) },
    breakdowns: {
      network: await breakdown(sql, 'network', since30),
      os: await breakdown(sql, 'os', since30),
      arch: await breakdown(sql, 'arch', since30),
      rMinor: await breakdown(sql, 'r_minor', since30),
      tool: await breakdown(sql, 'tool', since30),
      country: suppress(await breakdown(sql, 'country', since30)),
    },
    redirects,
    unsupportedMinors,
    excluded: {
      ownRunners: Number(excluded.own),
      probes: Number(excluded.probes),
      noise: Number(excluded.noise),
      errors: Number(excluded.errors),
    },
    packages,
  };
 }
 async function upload(body: string): Promise<void> {
  const s3 = new S3Client({
    accessKeyId: process.env.B2_S3_ACCESS_KEY,
    secretAccessKey: process.env.B2_S3_SECRET_KEY,
    bucket: process.env.STATS_BUCKET ?? 'devxy-rpkgs-binaries',
    endpoint: 'https://s3.eu-central-003.backblazeb2.com',
    region: 'eu-central-003',
  });
  await s3.write(OBJECT_KEY, body, { type: 'application/json' });
  // The pull zone overrides origin caching with a one-year TTL, so the edge
  // copy has to be purged for the new file to show up.
  const purge = new URL('https://api.bunny.net/purge');
  purge.searchParams.set('url', PUBLIC_URL);
  purge.searchParams.set('async', 'false');
  const response = await fetch(purge, { method: 'POST', headers: { AccessKey: process.env.BUNNYNET_API_KEY ?? '' } });
  if (!response.ok) throw new Error(`purging ${PUBLIC_URL} failed with ${response.status}: ${await response.text()}`);
 }
 if (import.meta.main) {
  const sql = connect();
  await migrate(sql);
  const stats = await buildStats(sql);
  await sql.close();
  const body = JSON.stringify(stats);
  const outIndex = process.argv.indexOf('--out');
  if (outIndex !== -1) {
    await Bun.write(process.argv[outIndex + 1], JSON.stringify(stats, null, 2));
    console.log(`wrote ${process.argv[outIndex + 1]} (${body.length} bytes)`);
  } else {
    await upload(body);
    console.log(
      `published ${PUBLIC_URL} (${body.length} bytes, ${stats.daily.length} days, ${stats.packages.length} packages)`,
    );
  }
 }