devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
0

feat(stats): collect CDN access logs and publish download statistics

Add an hourly download-stats pipeline that reads the cran-rpkgs Bunny
access logs into a download_stats schema in the build metadata database
and publishes aggregated counts to /stats/downloads.json for the
rpkgs.com /stats page.

- Classify each request (download, index, redirect reason, probe, noise)
  mirroring the edge router, and split cloud/CI networks from the rest.
- De-duplicate downloads per client, package version and day, and key
  clients on a salted HMAC so no IP is stored or published.
- Exclude our own CI runners and record run windows to show gaps.
- Declare the zone's logging settings and allow CORS for JSON.
This commit is contained in:
Patrick Schratz 2026-11-14 13:02:53 +01:00
commit 658254afc5

139
stats/classify.test.ts Normal file
View file

@ -0,0 +1,139 @@
import { describe, expect, test } from 'bun:test';
import { classify, clientPlatform, parseNetworks, toEvent, tool, type LogEntry } from './classify';
const R_RHEL = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)';
const RENV_RHEL = 'renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)';
const R_ALPINE = 'R/4.6.1 (Alpine Linux 3.24.1) (aarch64-unknown-linux-musl aarch64 linux-musl)';
const R_43 = 'R (4.3.2 x86_64-pc-linux-gnu x86_64 linux-gnu)';
const base = { status: 200, user_agent: R_RHEL, asn: 3303, asn_org: 'Swisscom', own: false };
describe('classify', () => {
test('flat tarball is a download with package, version and slot', () => {
const d = classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/data.table_1.17.8.tar.gz' });
expect(d).toMatchObject({
kind: 'download',
outcome: 'served',
slot: 'amd64/rhel9',
arch: 'amd64',
os: 'rhel9',
package: 'data.table',
version: '1.17.8',
r_minor: '4.5',
r_version: '4.5.3',
tool: 'R',
network: 'other',
});
});
test('per-minor tarball and index keep the slot', () => {
expect(classify({ ...base, path: '/arm64/alpine324/latest/src/contrib/4.6/Rcpp_1.1.0.tar.gz' })).toMatchObject({
kind: 'download',
slot: 'arm64/alpine324',
package: 'Rcpp',
});
expect(classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/4.5/PACKAGES.rds' }).kind).toBe('index');
});
test('index redirect with a known minor is internal minor routing', () => {
const d = classify({ ...base, status: 302, path: '/amd64/rhel9/latest/src/contrib/PACKAGES.rds' });
expect(d).toMatchObject({ kind: 'index', outcome: 'redirect', redirect_reason: 'minor_routing' });
});
test('redirect for an R minor we do not build is unsupported_r_minor', () => {
const d = classify({ ...base, user_agent: R_43, status: 302, path: '/amd64/noble/latest/src/contrib/PACKAGES' });
expect(d).toMatchObject({ redirect_reason: 'unsupported_r_minor', r_minor: '4.3' });
});
test('archived version without a binary falls back to CRAN', () => {
const d = classify({
...base,
status: 302,
path: '/amd64/rhel9/latest/src/contrib/Archive/renv/renv_1.2.3.tar.gz',
});
expect(d).toMatchObject({ kind: 'download', redirect_reason: 'archive_fallback', package: 'renv' });
});
test('bare URL: identified distro resolves to a slot, stock R UA goes to CRAN', () => {
expect(
classify({ ...base, user_agent: R_ALPINE, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason,
).toBe('slot_resolution');
expect(classify({ ...base, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason).toBe(
'unknown_distro',
);
});
test('renv and pak probing for Posit Package Manager is a probe', () => {
expect(classify({ ...base, status: 404, path: '/amd64/__api__/status' }).kind).toBe('probe');
});
test('scanner paths are noise', () => {
for (const path of ['/.env', '/wp-content/plugins/x.php', '/.git/config', '/xwpg.php']) {
expect(classify({ ...base, status: 404, user_agent: 'Mozilla/5.0', path }).kind).toBe('noise');
}
});
test('the stats file is not counted as repository use', () => {
expect(classify({ ...base, path: '/stats/downloads.json' }).kind).toBe('stats');
});
test('cloud networks are labelled, own networks win over cloud', () => {
expect(classify({ ...base, asn: 8075, asn_org: 'Microsoft Corporation', path: '/' }).network).toBe('cloud');
expect(classify({ ...base, asn: null, asn_org: 'Some Hosting Ltd', path: '/' }).network).toBe('cloud');
expect(classify({ ...base, asn: 24940, own: true, path: '/' }).network).toBe('own');
});
test('query strings and doubled slashes do not change the result', () => {
expect(classify({ ...base, path: '/amd64//rhel9/latest/src/contrib/PACKAGES.rds?x=1' }).kind).toBe('index');
});
});
describe('client identity', () => {
test('renv and plain R from the same machine share one platform key', () => {
expect(clientPlatform(RENV_RHEL)).toBe(clientPlatform(R_RHEL));
expect(clientPlatform(R_ALPINE)).not.toBe(clientPlatform(R_RHEL));
});
test('tool detection', () => {
expect(tool(RENV_RHEL)).toBe('renv');
expect(tool(R_RHEL)).toBe('R');
expect(tool('pak/0.9.0 R/4.5.1 (...)')).toBe('pak');
expect(tool('curl/8.5.0')).toBe('curl');
expect(tool('-')).toBe('none');
expect(tool('Mozilla/5.0 (Windows NT 10.0)')).toBe('browser');
});
test('own networks match by prefix for IPv4 and IPv6', () => {
const own = parseNetworks('203.0.113.0/24, 2001:db8:1:a100::/64');
const matches = (ip: string) => own.some((m) => m(ip));
expect(matches('203.0.113.0')).toBe(true);
expect(matches('203.0.114.0')).toBe(false);
expect(matches('2001:db8:1:a100::')).toBe(true);
expect(matches('2001:db8:1:a101::')).toBe(false);
expect(parseNetworks('')).toHaveLength(0);
});
test('toEvent hashes the client and never keeps the IP', () => {
const entry: LogEntry = {
timestamp: '2026-11-14T11:45:33.689+00:00',
pullZoneId: 3857050,
requestId: 'abc',
cacheStatus: 'HIT',
statusCode: 200,
bytesSent: 1000,
remoteIp: '203.0.113.0',
countryCode: 'CH',
host: 'cran.rpkgs.com',
path: '/amd64/rhel9/latest/src/contrib/rlang_1.1.6.tar.gz',
userAgent: RENV_RHEL,
asn: 3303,
asnOrganization: 'Swisscom',
};
const event = toEvent(entry, (key) => `h(${key.length})`, parseNetworks('203.0.113.0/24'));
expect(event).not.toBeNull();
expect(JSON.stringify(event)).not.toContain('203.0.113');
expect(event!.own).toBe(true);
expect(event!.network).toBe('own');
expect(toEvent({ ...entry, requestId: null }, (k) => k, [])).toBeNull();
});
});