devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
0

feat(stats): collect CDN access logs and publish download statistics (#251)
All checks were successful
ci/crow/manual/download-stats Pipeline was successful
ci/crow/cron/download-stats Pipeline was successful

Adds visibility into how cran.rpkgs.com is actually used, from the Bunny access logs, without sending anything to a third party.

## What it does

- `.crow/download-stats.yaml` runs hourly: `stats/collect.ts` reads the last 70 hours of cran-rpkgs logs (Bunny keeps three days) into a new `download_stats` schema in the build metadata database, then `stats/publish.ts` writes aggregated counts to `stats/downloads.json` in the binaries bucket and purges the CDN copy.
- `stats/classify.ts` labels each request (download, index, redirect reason, renv/pak probe, scanner noise) by mirroring `edge/rpkgs-router.ts`, and splits cloud/CI networks (by ASN) from everything else.
- Downloads are de-duplicated per client, package version and UTC day; clients are a salted HMAC of the anonymized /24 prefix, ASN and R platform, so no IP is stored or published.
- Our own CI runners (`STATS_OWN_NETWORKS`) are excluded; collection windows are recorded so the dashboard can show gaps.
- `cdn.tf` declares the zone's existing logging settings and adds `json` to the CORS extensions so rpkgs.com can read the file.
- `stats/README.md` documents every rule.

The rpkgs.com `/stats` page that renders this is in rpkgs/website.

## Before merging

- Add Crow secrets `STATS_CLIENT_SALT` and `STATS_OWN_NETWORKS` (values in OpenBao at `development/agent-dev/rpkgs-download-stats`).
- `tofu apply` for the CORS change.
- After merge, register the `download-stats` cron (`0 * * * *`).

## Verification

- `bun test stats`: 15 tests pass.
- Ran `collect.ts` locally against the live API and database: 14,273 events inserted, a second run inserted 0.
- `publish.ts --out` produced the JSON the website was previewed with; the upload and purge path has only run in CI so far (B2 credentials are CI-only).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Reviewed-on: #251
This commit is contained in:
Patrick Schratz 2026-11-14 12:10:59 +00:00 • committed by Patrick Schratz
commit 72440f9cf5

139
stats/classify.test.ts Normal file
View file

@ -0,0 +1,139 @@
import { describe, expect, test } from 'bun:test';
import { classify, clientPlatform, parseNetworks, toEvent, tool, type LogEntry } from './classify';
const R_RHEL = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)';
const RENV_RHEL = 'renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)';
const R_ALPINE = 'R/4.6.1 (Alpine Linux 3.24.1) (aarch64-unknown-linux-musl aarch64 linux-musl)';
const R_43 = 'R (4.3.2 x86_64-pc-linux-gnu x86_64 linux-gnu)';
const base = { status: 200, user_agent: R_RHEL, asn: 3303, asn_org: 'Swisscom', own: false };
describe('classify', () => {
test('flat tarball is a download with package, version and slot', () => {
const d = classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/data.table_1.17.8.tar.gz' });
expect(d).toMatchObject({
kind: 'download',
outcome: 'served',
slot: 'amd64/rhel9',
arch: 'amd64',
os: 'rhel9',
package: 'data.table',
version: '1.17.8',
r_minor: '4.5',
r_version: '4.5.3',
tool: 'R',
network: 'other',
});
});
test('per-minor tarball and index keep the slot', () => {
expect(classify({ ...base, path: '/arm64/alpine324/latest/src/contrib/4.6/Rcpp_1.1.0.tar.gz' })).toMatchObject({
kind: 'download',
slot: 'arm64/alpine324',
package: 'Rcpp',
});
expect(classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/4.5/PACKAGES.rds' }).kind).toBe('index');
});
test('index redirect with a known minor is internal minor routing', () => {
const d = classify({ ...base, status: 302, path: '/amd64/rhel9/latest/src/contrib/PACKAGES.rds' });
expect(d).toMatchObject({ kind: 'index', outcome: 'redirect', redirect_reason: 'minor_routing' });
});
test('redirect for an R minor we do not build is unsupported_r_minor', () => {
const d = classify({ ...base, user_agent: R_43, status: 302, path: '/amd64/noble/latest/src/contrib/PACKAGES' });
expect(d).toMatchObject({ redirect_reason: 'unsupported_r_minor', r_minor: '4.3' });
});
test('archived version without a binary falls back to CRAN', () => {
const d = classify({
...base,
status: 302,
path: '/amd64/rhel9/latest/src/contrib/Archive/renv/renv_1.2.3.tar.gz',
});
expect(d).toMatchObject({ kind: 'download', redirect_reason: 'archive_fallback', package: 'renv' });
});
test('bare URL: identified distro resolves to a slot, stock R UA goes to CRAN', () => {
expect(
classify({ ...base, user_agent: R_ALPINE, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason,
).toBe('slot_resolution');
expect(classify({ ...base, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason).toBe(
'unknown_distro',
);
});
test('renv and pak probing for Posit Package Manager is a probe', () => {
expect(classify({ ...base, status: 404, path: '/amd64/__api__/status' }).kind).toBe('probe');
});
test('scanner paths are noise', () => {
for (const path of ['/.env', '/wp-content/plugins/x.php', '/.git/config', '/xwpg.php']) {
expect(classify({ ...base, status: 404, user_agent: 'Mozilla/5.0', path }).kind).toBe('noise');
}
});
test('the stats file is not counted as repository use', () => {
expect(classify({ ...base, path: '/stats/downloads.json' }).kind).toBe('stats');
});
test('cloud networks are labelled, own networks win over cloud', () => {
expect(classify({ ...base, asn: 8075, asn_org: 'Microsoft Corporation', path: '/' }).network).toBe('cloud');
expect(classify({ ...base, asn: null, asn_org: 'Some Hosting Ltd', path: '/' }).network).toBe('cloud');
expect(classify({ ...base, asn: 24940, own: true, path: '/' }).network).toBe('own');
});
test('query strings and doubled slashes do not change the result', () => {
expect(classify({ ...base, path: '/amd64//rhel9/latest/src/contrib/PACKAGES.rds?x=1' }).kind).toBe('index');
});
});
describe('client identity', () => {
test('renv and plain R from the same machine share one platform key', () => {
expect(clientPlatform(RENV_RHEL)).toBe(clientPlatform(R_RHEL));
expect(clientPlatform(R_ALPINE)).not.toBe(clientPlatform(R_RHEL));
});
test('tool detection', () => {
expect(tool(RENV_RHEL)).toBe('renv');
expect(tool(R_RHEL)).toBe('R');
expect(tool('pak/0.9.0 R/4.5.1 (...)')).toBe('pak');
expect(tool('curl/8.5.0')).toBe('curl');
expect(tool('-')).toBe('none');
expect(tool('Mozilla/5.0 (Windows NT 10.0)')).toBe('browser');
});
test('own networks match by prefix for IPv4 and IPv6', () => {
const own = parseNetworks('203.0.113.0/24, 2001:db8:1:a100::/64');
const matches = (ip: string) => own.some((m) => m(ip));
expect(matches('203.0.113.0')).toBe(true);
expect(matches('203.0.114.0')).toBe(false);
expect(matches('2001:db8:1:a100::')).toBe(true);
expect(matches('2001:db8:1:a101::')).toBe(false);
expect(parseNetworks('')).toHaveLength(0);
});
test('toEvent hashes the client and never keeps the IP', () => {
const entry: LogEntry = {
timestamp: '2026-11-14T11:45:33.689+00:00',
pullZoneId: 3857050,
requestId: 'abc',
cacheStatus: 'HIT',
statusCode: 200,
bytesSent: 1000,
remoteIp: '203.0.113.0',
countryCode: 'CH',
host: 'cran.rpkgs.com',
path: '/amd64/rhel9/latest/src/contrib/rlang_1.1.6.tar.gz',
userAgent: RENV_RHEL,
asn: 3303,
asnOrganization: 'Swisscom',
};
const event = toEvent(entry, (key) => `h(${key.length})`, parseNetworks('203.0.113.0/24'));
expect(event).not.toBeNull();
expect(JSON.stringify(event)).not.toContain('203.0.113');
expect(event!.own).toBe(true);
expect(event!.network).toBe('own');
expect(toEvent({ ...entry, requestId: null }, (k) => k, [])).toBeNull();
});
});