feat(stats): collect CDN access logs and publish download statistics (#251)
Adds visibility into how cran.rpkgs.com is actually used, from the Bunny access logs, without sending anything to a third party. ## What it does - `.crow/download-stats.yaml` runs hourly: `stats/collect.ts` reads the last 70 hours of cran-rpkgs logs (Bunny keeps three days) into a new `download_stats` schema in the build metadata database, then `stats/publish.ts` writes aggregated counts to `stats/downloads.json` in the binaries bucket and purges the CDN copy. - `stats/classify.ts` labels each request (download, index, redirect reason, renv/pak probe, scanner noise) by mirroring `edge/rpkgs-router.ts`, and splits cloud/CI networks (by ASN) from everything else. - Downloads are de-duplicated per client, package version and UTC day; clients are a salted HMAC of the anonymized /24 prefix, ASN and R platform, so no IP is stored or published. - Our own CI runners (`STATS_OWN_NETWORKS`) are excluded; collection windows are recorded so the dashboard can show gaps. - `cdn.tf` declares the zone's existing logging settings and adds `json` to the CORS extensions so rpkgs.com can read the file. - `stats/README.md` documents every rule. The rpkgs.com `/stats` page that renders this is in rpkgs/website. ## Before merging - Add Crow secrets `STATS_CLIENT_SALT` and `STATS_OWN_NETWORKS` (values in OpenBao at `development/agent-dev/rpkgs-download-stats`). - `tofu apply` for the CORS change. - After merge, register the `download-stats` cron (`0 * * * *`). ## Verification - `bun test stats`: 15 tests pass. - Ran `collect.ts` locally against the live API and database: 14,273 events inserted, a second run inserted 0. - `publish.ts --out` produced the JSON the website was previewed with; the upload and purge path has only run in CI so far (B2 credentials are CI-only). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Reviewed-on: #251
This commit is contained in:
Patrick Schratz
parent
56dc06734c
commit
72440f9cf5
1 changed files with 1283 additions and 0 deletions
410
stats/classify.ts
Normal file
410
stats/classify.ts
Normal file
|
|
@ -0,0 +1,410 @@
|
|||
/**
|
||||
* Turns one Bunny access-log entry into a classified download event.
|
||||
*
|
||||
* Everything here is pure so the rules can be unit-tested and re-run over the
|
||||
* stored events (`collect.ts --reclassify`) whenever they change. Bump RULESET
|
||||
* on any change that alters a derived column, so the public JSON can say which
|
||||
* rules produced it.
|
||||
*
|
||||
* The rules mirror `edge/rpkgs-router.ts`: a redirect is classified by what the
|
||||
* router would have done with that path and User-Agent, because the Location
|
||||
* header is not in the logs.
|
||||
*/
|
||||
|
||||
export const RULESET = 1;
|
||||
|
||||
/** R minors with a per-minor slot; keep in sync with KNOWN_MINORS in cdn.tf. */
|
||||
export const KNOWN_MINORS = new Set(['4.4', '4.5', '4.6']);
|
||||
|
||||
export type Kind =
|
||||
/** A package tarball. */
|
||||
| 'download'
|
||||
/** `PACKAGES*`: one per repository per `install.packages()` call. */
|
||||
| 'index'
|
||||
/** Other repository metadata, e.g. `Meta/archive.rds`. */
|
||||
| 'metadata'
|
||||
/** renv and pak asking whether this is a Posit Package Manager (`__api__/…`). */
|
||||
| 'probe'
|
||||
/** The repository root or `/src/contrib` without a file. */
|
||||
| 'root'
|
||||
/** Our own `stats/` files, so the dashboard does not count itself. */
|
||||
| 'stats'
|
||||
/** Anything else: vulnerability scanners, crawlers, typos. */
|
||||
| 'noise';
|
||||
|
||||
export type Outcome = 'served' | 'redirect' | 'error';
|
||||
|
||||
export type RedirectReason =
|
||||
/** Internal hop: index sent to its per-R-minor slot. Not a user-visible event. */
|
||||
| 'minor_routing'
|
||||
/** Internal hop: bare `cran.rpkgs.com` URL resolved to a slot. */
|
||||
| 'slot_resolution'
|
||||
/** Client runs an R minor we do not build for; sent to CRAN for source. */
|
||||
| 'unsupported_r_minor'
|
||||
/** Bare URL from a distro we could not identify; sent to CRAN. */
|
||||
| 'unknown_distro'
|
||||
/** Archived version we have no binary for; sent to the CRAN source tarball. */
|
||||
| 'archive_fallback'
|
||||
/** macOS client; sent to CRAN's macOS binaries. */
|
||||
| 'macos'
|
||||
/** `http://` to `https://`. */
|
||||
| 'https'
|
||||
| 'other';
|
||||
|
||||
export type Tool = 'R' | 'renv' | 'pak' | 'rig' | 'curl' | 'wget' | 'python' | 'browser' | 'none' | 'other';
|
||||
|
||||
export type Network = 'cloud' | 'other' | 'own';
|
||||
|
||||
export interface LogEntry {
|
||||
timestamp: string;
|
||||
pullZoneId: number;
|
||||
requestId: string | null;
|
||||
cacheStatus: string | null;
|
||||
statusCode: number;
|
||||
bytesSent: number;
|
||||
remoteIp: string | null;
|
||||
countryCode: string | null;
|
||||
host: string | null;
|
||||
path: string | null;
|
||||
userAgent: string | null;
|
||||
asn: number | null;
|
||||
asnOrganization: string | null;
|
||||
}
|
||||
|
||||
export interface Derived {
|
||||
ruleset: number;
|
||||
kind: Kind;
|
||||
outcome: Outcome;
|
||||
redirect_reason: RedirectReason | null;
|
||||
slot: string | null;
|
||||
arch: string | null;
|
||||
os: string | null;
|
||||
r_minor: string | null;
|
||||
r_version: string | null;
|
||||
package: string | null;
|
||||
version: string | null;
|
||||
tool: Tool;
|
||||
network: Network;
|
||||
}
|
||||
|
||||
/** The raw columns kept per event; everything Derived is recomputed from these. */
|
||||
export interface RawEvent {
|
||||
request_id: string;
|
||||
ts: string;
|
||||
zone_id: number;
|
||||
host: string | null;
|
||||
status: number;
|
||||
cache_status: string | null;
|
||||
bytes: number;
|
||||
path: string;
|
||||
user_agent: string | null;
|
||||
country: string | null;
|
||||
asn: number | null;
|
||||
asn_org: string | null;
|
||||
/** HMAC of the (already anonymized) IP prefix, network and client platform. */
|
||||
client: string;
|
||||
/** Matched one of our own networks at ingest. The IP itself is not kept. */
|
||||
own: boolean;
|
||||
}
|
||||
|
||||
export type Event = RawEvent & Derived;
|
||||
|
||||
const SLOT_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib(?:\/(.*))?$/;
|
||||
const BARE_REGEX = /^\/src\/contrib(?:\/(.*))?$/;
|
||||
const MACOS_REGEX = /^\/bin\/macosx\/[^/]+\/contrib\/(\d+\.\d+)\/(.+)$/;
|
||||
const MINOR_DIR_REGEX = /^(\d+\.\d+)\/(.*)$/;
|
||||
const INDEX_REGEX = /^PACKAGES(\.gz|\.rds|\.json)?$/;
|
||||
const TARBALL_REGEX = /(?:^|\/)([A-Za-z][A-Za-z0-9.]*)_([0-9][^/_]*)\.(?:tar\.gz|tgz)$/;
|
||||
const R_VERSION_REGEXES = [/\bR\/(\d+\.\d+\.\d+)/, /\bR \((\d+\.\d+\.\d+)/];
|
||||
const ARCH_REGEX = /\b(x86_64|aarch64|arm64)\b/;
|
||||
/** The distro forms `parseSlot()` in the router recognises. */
|
||||
const DISTRO_REGEX = /(almalinux|rocky)[^\d]*\d+|Ubuntu \d+\.\d+|Alpine Linux|alpine-\d/i;
|
||||
|
||||
/**
|
||||
* Networks whose clients are almost always machines: public clouds, CI
|
||||
* providers and hosting companies. A request from one of these is reported as
|
||||
* "cloud / CI" rather than dropped, since a CI pipeline is a real user too,
|
||||
* just not a person at a laptop.
|
||||
*/
|
||||
const CLOUD_ASNS = new Map<number, string>([
|
||||
[8075, 'Microsoft (incl. GitHub Actions runners)'],
|
||||
[8068, 'Microsoft'],
|
||||
[8069, 'Microsoft'],
|
||||
[36459, 'GitHub'],
|
||||
[16509, 'Amazon'],
|
||||
[14618, 'Amazon'],
|
||||
[15169, 'Google'],
|
||||
[396982, 'Google Cloud'],
|
||||
[24940, 'Hetzner'],
|
||||
[213230, 'Hetzner Cloud'],
|
||||
[212317, 'Hetzner'],
|
||||
[16276, 'OVH'],
|
||||
[14061, 'DigitalOcean'],
|
||||
[51167, 'Contabo'],
|
||||
[40021, 'Contabo US'],
|
||||
[63949, 'Akamai / Linode'],
|
||||
[31898, 'Oracle'],
|
||||
[45102, 'Alibaba'],
|
||||
[37963, 'Alibaba'],
|
||||
[132203, 'Tencent'],
|
||||
[45090, 'Tencent'],
|
||||
[13335, 'Cloudflare'],
|
||||
[62240, 'Clouvider'],
|
||||
[12876, 'Scaleway'],
|
||||
[20473, 'Vultr'],
|
||||
[197540, 'netcup'],
|
||||
[9009, 'M247'],
|
||||
[60068, 'Datacamp / CDN77'],
|
||||
[36352, 'ColoCrossing'],
|
||||
[47583, 'Hostinger'],
|
||||
[207990, 'HostRoyale'],
|
||||
]);
|
||||
|
||||
const CLOUD_ORG_REGEX = /\b(hosting|cloud|server(s)?|data ?cent(er|re)|colo(cation)?|vps|dedicated)\b/i;
|
||||
|
||||
export function isCloudNetwork(asn: number | null, org: string | null): boolean {
|
||||
if (asn !== null && CLOUD_ASNS.has(asn)) return true;
|
||||
return org !== null && CLOUD_ORG_REGEX.test(org);
|
||||
}
|
||||
|
||||
export function normalizePath(path: string): string {
|
||||
const withoutQuery = path.split('?')[0] ?? '';
|
||||
return withoutQuery.replace(/\/{2,}/g, '/');
|
||||
}
|
||||
|
||||
export function rVersion(userAgent: string): string | null {
|
||||
for (const regex of R_VERSION_REGEXES) {
|
||||
const match = userAgent.match(regex);
|
||||
if (match) return match[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function rMinor(version: string | null): string | null {
|
||||
if (!version) return null;
|
||||
const [major, minor] = version.split('.');
|
||||
return `${major}.${minor}`;
|
||||
}
|
||||
|
||||
export function tool(userAgent: string): Tool {
|
||||
const ua = userAgent.trim();
|
||||
if (ua === '' || ua === '-') return 'none';
|
||||
if (/^renv\b/i.test(ua)) return 'renv';
|
||||
if (/\bpak\b/i.test(ua)) return 'pak';
|
||||
if (/\brig\b/i.test(ua)) return 'rig';
|
||||
if (/^R[ /(]/.test(ua)) return 'R';
|
||||
if (/^curl\//i.test(ua)) return 'curl';
|
||||
if (/^wget\//i.test(ua)) return 'wget';
|
||||
if (/python|aiohttp|httpx/i.test(ua)) return 'python';
|
||||
if (/^Mozilla\//.test(ua)) return 'browser';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
/**
|
||||
* The part of the User-Agent that identifies the machine rather than the tool.
|
||||
*
|
||||
* `renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` and
|
||||
* `R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` are the same R session, one going
|
||||
* through renv and one not. Keying distinct clients on the full User-Agent
|
||||
* would count that machine twice.
|
||||
*/
|
||||
export function clientPlatform(userAgent: string): string {
|
||||
const version = rVersion(userAgent);
|
||||
if (!version) return userAgent;
|
||||
const rPart = userAgent.slice(userAgent.search(/\bR[/ ]\(?\d/));
|
||||
return rPart.replace(/\bR\/\d+\.\d+\.\d+\s*/, '').trim() || version;
|
||||
}
|
||||
|
||||
function slotParts(slot: string | null): { arch: string | null; os: string | null } {
|
||||
if (!slot) return { arch: null, os: null };
|
||||
const [arch, os] = slot.split('/');
|
||||
return { arch: arch ?? null, os: os ?? null };
|
||||
}
|
||||
|
||||
function archFromUserAgent(userAgent: string): string | null {
|
||||
const match = userAgent.match(ARCH_REGEX);
|
||||
if (!match) return null;
|
||||
return match[1] === 'x86_64' ? 'amd64' : 'arm64';
|
||||
}
|
||||
|
||||
function fileKind(rest: string): Kind {
|
||||
if (rest === '') return 'root';
|
||||
if (INDEX_REGEX.test(rest)) return 'index';
|
||||
if (TARBALL_REGEX.test(rest)) return 'download';
|
||||
if (/^Meta\//.test(rest) || /^PACKAGES/.test(rest)) return 'metadata';
|
||||
return 'noise';
|
||||
}
|
||||
|
||||
function outcome(status: number): Outcome {
|
||||
if (status === 301 || status === 302 || status === 303 || status === 307 || status === 308) return 'redirect';
|
||||
if (status >= 200 && status < 400) return 'served';
|
||||
return 'error';
|
||||
}
|
||||
|
||||
export function classify(
|
||||
raw: Pick<RawEvent, 'path' | 'status' | 'user_agent' | 'asn' | 'asn_org' | 'own'>,
|
||||
scheme = 'https',
|
||||
): Derived {
|
||||
const path = normalizePath(raw.path);
|
||||
const userAgent = raw.user_agent ?? '';
|
||||
const version = rVersion(userAgent);
|
||||
const minor = rMinor(version);
|
||||
const result: Derived = {
|
||||
ruleset: RULESET,
|
||||
kind: 'noise',
|
||||
outcome: outcome(raw.status),
|
||||
redirect_reason: null,
|
||||
slot: null,
|
||||
arch: null,
|
||||
os: null,
|
||||
r_minor: minor,
|
||||
r_version: version,
|
||||
package: null,
|
||||
version: null,
|
||||
tool: tool(userAgent),
|
||||
network: raw.own ? 'own' : isCloudNetwork(raw.asn, raw.asn_org) ? 'cloud' : 'other',
|
||||
};
|
||||
|
||||
let rest: string | null = null;
|
||||
let area: 'slot' | 'bare' | 'macos' | null = null;
|
||||
|
||||
const slot = path.match(SLOT_REGEX);
|
||||
const bare = path.match(BARE_REGEX);
|
||||
const macos = path.match(MACOS_REGEX);
|
||||
|
||||
if (path.includes('/__api__/')) {
|
||||
result.kind = 'probe';
|
||||
} else if (path.startsWith('/stats/')) {
|
||||
result.kind = 'stats';
|
||||
} else if (path === '/') {
|
||||
result.kind = 'root';
|
||||
area = 'bare';
|
||||
} else if (slot) {
|
||||
area = 'slot';
|
||||
result.slot = `${slot[1]}/${slot[2]}`;
|
||||
rest = slot[3] ?? '';
|
||||
} else if (bare) {
|
||||
area = 'bare';
|
||||
rest = bare[1] ?? '';
|
||||
} else if (macos) {
|
||||
area = 'macos';
|
||||
result.os = 'macos';
|
||||
rest = macos[2];
|
||||
}
|
||||
|
||||
if (rest !== null) {
|
||||
const minorDir = rest.match(MINOR_DIR_REGEX);
|
||||
const file = minorDir ? minorDir[2] : rest;
|
||||
result.kind = fileKind(file);
|
||||
const tarball = file.match(TARBALL_REGEX);
|
||||
if (tarball) {
|
||||
result.package = tarball[1];
|
||||
result.version = tarball[2];
|
||||
}
|
||||
}
|
||||
|
||||
const parts = slotParts(result.slot);
|
||||
result.arch = parts.arch ?? archFromUserAgent(userAgent);
|
||||
result.os = parts.os ?? result.os;
|
||||
|
||||
if (result.outcome === 'redirect') {
|
||||
result.redirect_reason = redirectReason(raw.status, area, rest, minor, scheme, userAgent);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function redirectReason(
|
||||
status: number,
|
||||
area: 'slot' | 'bare' | 'macos' | null,
|
||||
rest: string | null,
|
||||
minor: string | null,
|
||||
scheme: string,
|
||||
userAgent: string,
|
||||
): RedirectReason {
|
||||
if (status === 301 || scheme === 'http') return 'https';
|
||||
if (area === 'macos' || (area === 'bare' && /darwin/.test(userAgent))) return 'macos';
|
||||
if (minor !== null && !KNOWN_MINORS.has(minor) && (area === 'slot' || area === 'bare')) {
|
||||
return 'unsupported_r_minor';
|
||||
}
|
||||
if (area === 'slot') {
|
||||
if (rest !== null && rest.startsWith('Archive/')) return 'archive_fallback';
|
||||
if (rest !== null && INDEX_REGEX.test(rest)) return 'minor_routing';
|
||||
return 'other';
|
||||
}
|
||||
if (area === 'bare') {
|
||||
// The router resolves a bare URL only when it can identify the distro from
|
||||
// the User-Agent; otherwise it sends the client to CRAN. A stock R
|
||||
// User-Agent (`R (4.5.3 x86_64-pc-linux-gnu …)`) names no distro.
|
||||
return DISTRO_REGEX.test(userAgent) && ARCH_REGEX.test(userAgent) ? 'slot_resolution' : 'unknown_distro';
|
||||
}
|
||||
return 'other';
|
||||
}
|
||||
|
||||
/** Parse `"a.b.c.0/24,2001:db8::/64"` into prefix matchers for our own networks. */
|
||||
export function parseNetworks(spec: string): ((ip: string) => boolean)[] {
|
||||
return spec
|
||||
.split(',')
|
||||
.map((cidr) => cidr.trim())
|
||||
.filter((cidr) => cidr.length > 0)
|
||||
.map((cidr) => {
|
||||
const [base, bitsText] = cidr.split('/');
|
||||
const v6 = base.includes(':');
|
||||
const bits = Number(bitsText ?? (v6 ? 128 : 32));
|
||||
const baseBits = ipBits(base);
|
||||
return (ip: string) => {
|
||||
if (ip.includes(':') !== v6) return false;
|
||||
const candidate = ipBits(ip);
|
||||
return baseBits !== null && candidate !== null && candidate.slice(0, bits) === baseBits.slice(0, bits);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function ipBits(ip: string): string | null {
|
||||
if (!ip.includes(':')) {
|
||||
const octets = ip.split('.').map(Number);
|
||||
if (octets.length !== 4 || octets.some((o) => !Number.isInteger(o) || o < 0 || o > 255)) return null;
|
||||
return octets.map((o) => o.toString(2).padStart(8, '0')).join('');
|
||||
}
|
||||
const [head, tail] = ip.split('::');
|
||||
const headGroups = head ? head.split(':') : [];
|
||||
const tailGroups = tail ? tail.split(':') : [];
|
||||
const missing = 8 - headGroups.length - tailGroups.length;
|
||||
if (ip.includes('::') ? missing < 0 : missing !== 0) return null;
|
||||
const groups = [...headGroups, ...Array(Math.max(missing, 0)).fill('0'), ...tailGroups];
|
||||
return groups
|
||||
.map((g) =>
|
||||
parseInt(g || '0', 16)
|
||||
.toString(2)
|
||||
.padStart(16, '0'),
|
||||
)
|
||||
.join('');
|
||||
}
|
||||
|
||||
/** Build the stored event. Returns null for entries without a request id. */
|
||||
export function toEvent(
|
||||
entry: LogEntry,
|
||||
hashClient: (key: string) => string,
|
||||
ownNetworks: ((ip: string) => boolean)[],
|
||||
): Event | null {
|
||||
if (!entry.requestId) return null;
|
||||
const ip = entry.remoteIp ?? '';
|
||||
const userAgent = entry.userAgent ?? '';
|
||||
const raw: RawEvent = {
|
||||
request_id: entry.requestId,
|
||||
ts: entry.timestamp,
|
||||
zone_id: entry.pullZoneId,
|
||||
host: entry.host,
|
||||
status: entry.statusCode,
|
||||
cache_status: entry.cacheStatus,
|
||||
bytes: entry.bytesSent,
|
||||
path: entry.path ?? '',
|
||||
user_agent: entry.userAgent,
|
||||
country: entry.countryCode,
|
||||
asn: entry.asn,
|
||||
asn_org: entry.asnOrganization,
|
||||
client: hashClient(`${ip}|${entry.asn ?? ''}|${clientPlatform(userAgent)}`),
|
||||
own: ip !== '' && ownNetworks.some((matches) => matches(ip)),
|
||||
};
|
||||
return { ...raw, ...classify(raw) };
|
||||
}
|
||||
Loading…
Reference in a new issue