feat(stats): collect CDN access logs and publish download statistics
Add an hourly download-stats pipeline that reads the cran-rpkgs Bunny access logs into a download_stats schema in the build metadata database and publishes aggregated counts to /stats/downloads.json for the rpkgs.com /stats page. - Classify each request (download, index, redirect reason, probe, noise) mirroring the edge router, and split cloud/CI networks from the rest. - De-duplicate downloads per client, package version and day, and key clients on a salted HMAC so no IP is stored or published. - Exclude our own CI runners and record run windows to show gaps. - Declare the zone's logging settings and allow CORS for JSON.
This commit is contained in:
parent
56dc06734c
commit
658254afc5
1 changed files with 1279 additions and 0 deletions
139
stats/classify.test.ts
Normal file
139
stats/classify.test.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import { describe, expect, test } from 'bun:test';
|
||||
import { classify, clientPlatform, parseNetworks, toEvent, tool, type LogEntry } from './classify';
|
||||
|
||||
const R_RHEL = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)';
|
||||
const RENV_RHEL = 'renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)';
|
||||
const R_ALPINE = 'R/4.6.1 (Alpine Linux 3.24.1) (aarch64-unknown-linux-musl aarch64 linux-musl)';
|
||||
const R_43 = 'R (4.3.2 x86_64-pc-linux-gnu x86_64 linux-gnu)';
|
||||
|
||||
const base = { status: 200, user_agent: R_RHEL, asn: 3303, asn_org: 'Swisscom', own: false };
|
||||
|
||||
describe('classify', () => {
|
||||
test('flat tarball is a download with package, version and slot', () => {
|
||||
const d = classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/data.table_1.17.8.tar.gz' });
|
||||
expect(d).toMatchObject({
|
||||
kind: 'download',
|
||||
outcome: 'served',
|
||||
slot: 'amd64/rhel9',
|
||||
arch: 'amd64',
|
||||
os: 'rhel9',
|
||||
package: 'data.table',
|
||||
version: '1.17.8',
|
||||
r_minor: '4.5',
|
||||
r_version: '4.5.3',
|
||||
tool: 'R',
|
||||
network: 'other',
|
||||
});
|
||||
});
|
||||
|
||||
test('per-minor tarball and index keep the slot', () => {
|
||||
expect(classify({ ...base, path: '/arm64/alpine324/latest/src/contrib/4.6/Rcpp_1.1.0.tar.gz' })).toMatchObject({
|
||||
kind: 'download',
|
||||
slot: 'arm64/alpine324',
|
||||
package: 'Rcpp',
|
||||
});
|
||||
expect(classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/4.5/PACKAGES.rds' }).kind).toBe('index');
|
||||
});
|
||||
|
||||
test('index redirect with a known minor is internal minor routing', () => {
|
||||
const d = classify({ ...base, status: 302, path: '/amd64/rhel9/latest/src/contrib/PACKAGES.rds' });
|
||||
expect(d).toMatchObject({ kind: 'index', outcome: 'redirect', redirect_reason: 'minor_routing' });
|
||||
});
|
||||
|
||||
test('redirect for an R minor we do not build is unsupported_r_minor', () => {
|
||||
const d = classify({ ...base, user_agent: R_43, status: 302, path: '/amd64/noble/latest/src/contrib/PACKAGES' });
|
||||
expect(d).toMatchObject({ redirect_reason: 'unsupported_r_minor', r_minor: '4.3' });
|
||||
});
|
||||
|
||||
test('archived version without a binary falls back to CRAN', () => {
|
||||
const d = classify({
|
||||
...base,
|
||||
status: 302,
|
||||
path: '/amd64/rhel9/latest/src/contrib/Archive/renv/renv_1.2.3.tar.gz',
|
||||
});
|
||||
expect(d).toMatchObject({ kind: 'download', redirect_reason: 'archive_fallback', package: 'renv' });
|
||||
});
|
||||
|
||||
test('bare URL: identified distro resolves to a slot, stock R UA goes to CRAN', () => {
|
||||
expect(
|
||||
classify({ ...base, user_agent: R_ALPINE, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason,
|
||||
).toBe('slot_resolution');
|
||||
expect(classify({ ...base, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason).toBe(
|
||||
'unknown_distro',
|
||||
);
|
||||
});
|
||||
|
||||
test('renv and pak probing for Posit Package Manager is a probe', () => {
|
||||
expect(classify({ ...base, status: 404, path: '/amd64/__api__/status' }).kind).toBe('probe');
|
||||
});
|
||||
|
||||
test('scanner paths are noise', () => {
|
||||
for (const path of ['/.env', '/wp-content/plugins/x.php', '/.git/config', '/xwpg.php']) {
|
||||
expect(classify({ ...base, status: 404, user_agent: 'Mozilla/5.0', path }).kind).toBe('noise');
|
||||
}
|
||||
});
|
||||
|
||||
test('the stats file is not counted as repository use', () => {
|
||||
expect(classify({ ...base, path: '/stats/downloads.json' }).kind).toBe('stats');
|
||||
});
|
||||
|
||||
test('cloud networks are labelled, own networks win over cloud', () => {
|
||||
expect(classify({ ...base, asn: 8075, asn_org: 'Microsoft Corporation', path: '/' }).network).toBe('cloud');
|
||||
expect(classify({ ...base, asn: null, asn_org: 'Some Hosting Ltd', path: '/' }).network).toBe('cloud');
|
||||
expect(classify({ ...base, asn: 24940, own: true, path: '/' }).network).toBe('own');
|
||||
});
|
||||
|
||||
test('query strings and doubled slashes do not change the result', () => {
|
||||
expect(classify({ ...base, path: '/amd64//rhel9/latest/src/contrib/PACKAGES.rds?x=1' }).kind).toBe('index');
|
||||
});
|
||||
});
|
||||
|
||||
describe('client identity', () => {
|
||||
test('renv and plain R from the same machine share one platform key', () => {
|
||||
expect(clientPlatform(RENV_RHEL)).toBe(clientPlatform(R_RHEL));
|
||||
expect(clientPlatform(R_ALPINE)).not.toBe(clientPlatform(R_RHEL));
|
||||
});
|
||||
|
||||
test('tool detection', () => {
|
||||
expect(tool(RENV_RHEL)).toBe('renv');
|
||||
expect(tool(R_RHEL)).toBe('R');
|
||||
expect(tool('pak/0.9.0 R/4.5.1 (...)')).toBe('pak');
|
||||
expect(tool('curl/8.5.0')).toBe('curl');
|
||||
expect(tool('-')).toBe('none');
|
||||
expect(tool('Mozilla/5.0 (Windows NT 10.0)')).toBe('browser');
|
||||
});
|
||||
|
||||
test('own networks match by prefix for IPv4 and IPv6', () => {
|
||||
const own = parseNetworks('203.0.113.0/24, 2001:db8:1:a100::/64');
|
||||
const matches = (ip: string) => own.some((m) => m(ip));
|
||||
expect(matches('203.0.113.0')).toBe(true);
|
||||
expect(matches('203.0.114.0')).toBe(false);
|
||||
expect(matches('2001:db8:1:a100::')).toBe(true);
|
||||
expect(matches('2001:db8:1:a101::')).toBe(false);
|
||||
expect(parseNetworks('')).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('toEvent hashes the client and never keeps the IP', () => {
|
||||
const entry: LogEntry = {
|
||||
timestamp: '2026-11-14T11:45:33.689+00:00',
|
||||
pullZoneId: 3857050,
|
||||
requestId: 'abc',
|
||||
cacheStatus: 'HIT',
|
||||
statusCode: 200,
|
||||
bytesSent: 1000,
|
||||
remoteIp: '203.0.113.0',
|
||||
countryCode: 'CH',
|
||||
host: 'cran.rpkgs.com',
|
||||
path: '/amd64/rhel9/latest/src/contrib/rlang_1.1.6.tar.gz',
|
||||
userAgent: RENV_RHEL,
|
||||
asn: 3303,
|
||||
asnOrganization: 'Swisscom',
|
||||
};
|
||||
const event = toEvent(entry, (key) => `h(${key.length})`, parseNetworks('203.0.113.0/24'));
|
||||
expect(event).not.toBeNull();
|
||||
expect(JSON.stringify(event)).not.toContain('203.0.113');
|
||||
expect(event!.own).toBe(true);
|
||||
expect(event!.network).toBe('own');
|
||||
expect(toEvent({ ...entry, requestId: null }, (k) => k, [])).toBeNull();
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue