pat-s
658254afc5
Add an hourly download-stats pipeline that reads the cran-rpkgs Bunny access logs into a download_stats schema in the build metadata database and publishes aggregated counts to /stats/downloads.json for the rpkgs.com /stats page. - Classify each request (download, index, redirect reason, probe, noise) mirroring the edge router, and split cloud/CI networks from the rest. - De-duplicate downloads per client, package version and day, and key clients on a salted HMAC so no IP is stored or published. - Exclude our own CI runners and record run windows to show gaps. - Declare the zone's logging settings and allow CORS for JSON.
410 lines
13 KiB
TypeScript
410 lines
13 KiB
TypeScript
/**
|
|
* Turns one Bunny access-log entry into a classified download event.
|
|
*
|
|
* Everything here is pure so the rules can be unit-tested and re-run over the
|
|
* stored events (`collect.ts --reclassify`) whenever they change. Bump RULESET
|
|
* on any change that alters a derived column, so the public JSON can say which
|
|
* rules produced it.
|
|
*
|
|
* The rules mirror `edge/rpkgs-router.ts`: a redirect is classified by what the
|
|
* router would have done with that path and User-Agent, because the Location
|
|
* header is not in the logs.
|
|
*/
|
|
|
|
export const RULESET = 1;
|
|
|
|
/** R minors with a per-minor slot; keep in sync with KNOWN_MINORS in cdn.tf. */
|
|
export const KNOWN_MINORS = new Set(['4.4', '4.5', '4.6']);
|
|
|
|
export type Kind =
|
|
/** A package tarball. */
|
|
| 'download'
|
|
/** `PACKAGES*`: one per repository per `install.packages()` call. */
|
|
| 'index'
|
|
/** Other repository metadata, e.g. `Meta/archive.rds`. */
|
|
| 'metadata'
|
|
/** renv and pak asking whether this is a Posit Package Manager (`__api__/…`). */
|
|
| 'probe'
|
|
/** The repository root or `/src/contrib` without a file. */
|
|
| 'root'
|
|
/** Our own `stats/` files, so the dashboard does not count itself. */
|
|
| 'stats'
|
|
/** Anything else: vulnerability scanners, crawlers, typos. */
|
|
| 'noise';
|
|
|
|
export type Outcome = 'served' | 'redirect' | 'error';
|
|
|
|
export type RedirectReason =
|
|
/** Internal hop: index sent to its per-R-minor slot. Not a user-visible event. */
|
|
| 'minor_routing'
|
|
/** Internal hop: bare `cran.rpkgs.com` URL resolved to a slot. */
|
|
| 'slot_resolution'
|
|
/** Client runs an R minor we do not build for; sent to CRAN for source. */
|
|
| 'unsupported_r_minor'
|
|
/** Bare URL from a distro we could not identify; sent to CRAN. */
|
|
| 'unknown_distro'
|
|
/** Archived version we have no binary for; sent to the CRAN source tarball. */
|
|
| 'archive_fallback'
|
|
/** macOS client; sent to CRAN's macOS binaries. */
|
|
| 'macos'
|
|
/** `http://` to `https://`. */
|
|
| 'https'
|
|
| 'other';
|
|
|
|
export type Tool = 'R' | 'renv' | 'pak' | 'rig' | 'curl' | 'wget' | 'python' | 'browser' | 'none' | 'other';
|
|
|
|
export type Network = 'cloud' | 'other' | 'own';
|
|
|
|
export interface LogEntry {
|
|
timestamp: string;
|
|
pullZoneId: number;
|
|
requestId: string | null;
|
|
cacheStatus: string | null;
|
|
statusCode: number;
|
|
bytesSent: number;
|
|
remoteIp: string | null;
|
|
countryCode: string | null;
|
|
host: string | null;
|
|
path: string | null;
|
|
userAgent: string | null;
|
|
asn: number | null;
|
|
asnOrganization: string | null;
|
|
}
|
|
|
|
export interface Derived {
|
|
ruleset: number;
|
|
kind: Kind;
|
|
outcome: Outcome;
|
|
redirect_reason: RedirectReason | null;
|
|
slot: string | null;
|
|
arch: string | null;
|
|
os: string | null;
|
|
r_minor: string | null;
|
|
r_version: string | null;
|
|
package: string | null;
|
|
version: string | null;
|
|
tool: Tool;
|
|
network: Network;
|
|
}
|
|
|
|
/** The raw columns kept per event; everything Derived is recomputed from these. */
|
|
export interface RawEvent {
|
|
request_id: string;
|
|
ts: string;
|
|
zone_id: number;
|
|
host: string | null;
|
|
status: number;
|
|
cache_status: string | null;
|
|
bytes: number;
|
|
path: string;
|
|
user_agent: string | null;
|
|
country: string | null;
|
|
asn: number | null;
|
|
asn_org: string | null;
|
|
/** HMAC of the (already anonymized) IP prefix, network and client platform. */
|
|
client: string;
|
|
/** Matched one of our own networks at ingest. The IP itself is not kept. */
|
|
own: boolean;
|
|
}
|
|
|
|
export type Event = RawEvent & Derived;
|
|
|
|
const SLOT_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib(?:\/(.*))?$/;
|
|
const BARE_REGEX = /^\/src\/contrib(?:\/(.*))?$/;
|
|
const MACOS_REGEX = /^\/bin\/macosx\/[^/]+\/contrib\/(\d+\.\d+)\/(.+)$/;
|
|
const MINOR_DIR_REGEX = /^(\d+\.\d+)\/(.*)$/;
|
|
const INDEX_REGEX = /^PACKAGES(\.gz|\.rds|\.json)?$/;
|
|
const TARBALL_REGEX = /(?:^|\/)([A-Za-z][A-Za-z0-9.]*)_([0-9][^/_]*)\.(?:tar\.gz|tgz)$/;
|
|
const R_VERSION_REGEXES = [/\bR\/(\d+\.\d+\.\d+)/, /\bR \((\d+\.\d+\.\d+)/];
|
|
const ARCH_REGEX = /\b(x86_64|aarch64|arm64)\b/;
|
|
/** The distro forms `parseSlot()` in the router recognises. */
|
|
const DISTRO_REGEX = /(almalinux|rocky)[^\d]*\d+|Ubuntu \d+\.\d+|Alpine Linux|alpine-\d/i;
|
|
|
|
/**
|
|
* Networks whose clients are almost always machines: public clouds, CI
|
|
* providers and hosting companies. A request from one of these is reported as
|
|
* "cloud / CI" rather than dropped, since a CI pipeline is a real user too,
|
|
* just not a person at a laptop.
|
|
*/
|
|
const CLOUD_ASNS = new Map<number, string>([
|
|
[8075, 'Microsoft (incl. GitHub Actions runners)'],
|
|
[8068, 'Microsoft'],
|
|
[8069, 'Microsoft'],
|
|
[36459, 'GitHub'],
|
|
[16509, 'Amazon'],
|
|
[14618, 'Amazon'],
|
|
[15169, 'Google'],
|
|
[396982, 'Google Cloud'],
|
|
[24940, 'Hetzner'],
|
|
[213230, 'Hetzner Cloud'],
|
|
[212317, 'Hetzner'],
|
|
[16276, 'OVH'],
|
|
[14061, 'DigitalOcean'],
|
|
[51167, 'Contabo'],
|
|
[40021, 'Contabo US'],
|
|
[63949, 'Akamai / Linode'],
|
|
[31898, 'Oracle'],
|
|
[45102, 'Alibaba'],
|
|
[37963, 'Alibaba'],
|
|
[132203, 'Tencent'],
|
|
[45090, 'Tencent'],
|
|
[13335, 'Cloudflare'],
|
|
[62240, 'Clouvider'],
|
|
[12876, 'Scaleway'],
|
|
[20473, 'Vultr'],
|
|
[197540, 'netcup'],
|
|
[9009, 'M247'],
|
|
[60068, 'Datacamp / CDN77'],
|
|
[36352, 'ColoCrossing'],
|
|
[47583, 'Hostinger'],
|
|
[207990, 'HostRoyale'],
|
|
]);
|
|
|
|
const CLOUD_ORG_REGEX = /\b(hosting|cloud|server(s)?|data ?cent(er|re)|colo(cation)?|vps|dedicated)\b/i;
|
|
|
|
export function isCloudNetwork(asn: number | null, org: string | null): boolean {
|
|
if (asn !== null && CLOUD_ASNS.has(asn)) return true;
|
|
return org !== null && CLOUD_ORG_REGEX.test(org);
|
|
}
|
|
|
|
export function normalizePath(path: string): string {
|
|
const withoutQuery = path.split('?')[0] ?? '';
|
|
return withoutQuery.replace(/\/{2,}/g, '/');
|
|
}
|
|
|
|
export function rVersion(userAgent: string): string | null {
|
|
for (const regex of R_VERSION_REGEXES) {
|
|
const match = userAgent.match(regex);
|
|
if (match) return match[1];
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function rMinor(version: string | null): string | null {
|
|
if (!version) return null;
|
|
const [major, minor] = version.split('.');
|
|
return `${major}.${minor}`;
|
|
}
|
|
|
|
export function tool(userAgent: string): Tool {
|
|
const ua = userAgent.trim();
|
|
if (ua === '' || ua === '-') return 'none';
|
|
if (/^renv\b/i.test(ua)) return 'renv';
|
|
if (/\bpak\b/i.test(ua)) return 'pak';
|
|
if (/\brig\b/i.test(ua)) return 'rig';
|
|
if (/^R[ /(]/.test(ua)) return 'R';
|
|
if (/^curl\//i.test(ua)) return 'curl';
|
|
if (/^wget\//i.test(ua)) return 'wget';
|
|
if (/python|aiohttp|httpx/i.test(ua)) return 'python';
|
|
if (/^Mozilla\//.test(ua)) return 'browser';
|
|
return 'other';
|
|
}
|
|
|
|
/**
|
|
* The part of the User-Agent that identifies the machine rather than the tool.
|
|
*
|
|
* `renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` and
|
|
* `R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` are the same R session, one going
|
|
* through renv and one not. Keying distinct clients on the full User-Agent
|
|
* would count that machine twice.
|
|
*/
|
|
export function clientPlatform(userAgent: string): string {
|
|
const version = rVersion(userAgent);
|
|
if (!version) return userAgent;
|
|
const rPart = userAgent.slice(userAgent.search(/\bR[/ ]\(?\d/));
|
|
return rPart.replace(/\bR\/\d+\.\d+\.\d+\s*/, '').trim() || version;
|
|
}
|
|
|
|
function slotParts(slot: string | null): { arch: string | null; os: string | null } {
|
|
if (!slot) return { arch: null, os: null };
|
|
const [arch, os] = slot.split('/');
|
|
return { arch: arch ?? null, os: os ?? null };
|
|
}
|
|
|
|
function archFromUserAgent(userAgent: string): string | null {
|
|
const match = userAgent.match(ARCH_REGEX);
|
|
if (!match) return null;
|
|
return match[1] === 'x86_64' ? 'amd64' : 'arm64';
|
|
}
|
|
|
|
function fileKind(rest: string): Kind {
|
|
if (rest === '') return 'root';
|
|
if (INDEX_REGEX.test(rest)) return 'index';
|
|
if (TARBALL_REGEX.test(rest)) return 'download';
|
|
if (/^Meta\//.test(rest) || /^PACKAGES/.test(rest)) return 'metadata';
|
|
return 'noise';
|
|
}
|
|
|
|
function outcome(status: number): Outcome {
|
|
if (status === 301 || status === 302 || status === 303 || status === 307 || status === 308) return 'redirect';
|
|
if (status >= 200 && status < 400) return 'served';
|
|
return 'error';
|
|
}
|
|
|
|
export function classify(
|
|
raw: Pick<RawEvent, 'path' | 'status' | 'user_agent' | 'asn' | 'asn_org' | 'own'>,
|
|
scheme = 'https',
|
|
): Derived {
|
|
const path = normalizePath(raw.path);
|
|
const userAgent = raw.user_agent ?? '';
|
|
const version = rVersion(userAgent);
|
|
const minor = rMinor(version);
|
|
const result: Derived = {
|
|
ruleset: RULESET,
|
|
kind: 'noise',
|
|
outcome: outcome(raw.status),
|
|
redirect_reason: null,
|
|
slot: null,
|
|
arch: null,
|
|
os: null,
|
|
r_minor: minor,
|
|
r_version: version,
|
|
package: null,
|
|
version: null,
|
|
tool: tool(userAgent),
|
|
network: raw.own ? 'own' : isCloudNetwork(raw.asn, raw.asn_org) ? 'cloud' : 'other',
|
|
};
|
|
|
|
let rest: string | null = null;
|
|
let area: 'slot' | 'bare' | 'macos' | null = null;
|
|
|
|
const slot = path.match(SLOT_REGEX);
|
|
const bare = path.match(BARE_REGEX);
|
|
const macos = path.match(MACOS_REGEX);
|
|
|
|
if (path.includes('/__api__/')) {
|
|
result.kind = 'probe';
|
|
} else if (path.startsWith('/stats/')) {
|
|
result.kind = 'stats';
|
|
} else if (path === '/') {
|
|
result.kind = 'root';
|
|
area = 'bare';
|
|
} else if (slot) {
|
|
area = 'slot';
|
|
result.slot = `${slot[1]}/${slot[2]}`;
|
|
rest = slot[3] ?? '';
|
|
} else if (bare) {
|
|
area = 'bare';
|
|
rest = bare[1] ?? '';
|
|
} else if (macos) {
|
|
area = 'macos';
|
|
result.os = 'macos';
|
|
rest = macos[2];
|
|
}
|
|
|
|
if (rest !== null) {
|
|
const minorDir = rest.match(MINOR_DIR_REGEX);
|
|
const file = minorDir ? minorDir[2] : rest;
|
|
result.kind = fileKind(file);
|
|
const tarball = file.match(TARBALL_REGEX);
|
|
if (tarball) {
|
|
result.package = tarball[1];
|
|
result.version = tarball[2];
|
|
}
|
|
}
|
|
|
|
const parts = slotParts(result.slot);
|
|
result.arch = parts.arch ?? archFromUserAgent(userAgent);
|
|
result.os = parts.os ?? result.os;
|
|
|
|
if (result.outcome === 'redirect') {
|
|
result.redirect_reason = redirectReason(raw.status, area, rest, minor, scheme, userAgent);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
function redirectReason(
|
|
status: number,
|
|
area: 'slot' | 'bare' | 'macos' | null,
|
|
rest: string | null,
|
|
minor: string | null,
|
|
scheme: string,
|
|
userAgent: string,
|
|
): RedirectReason {
|
|
if (status === 301 || scheme === 'http') return 'https';
|
|
if (area === 'macos' || (area === 'bare' && /darwin/.test(userAgent))) return 'macos';
|
|
if (minor !== null && !KNOWN_MINORS.has(minor) && (area === 'slot' || area === 'bare')) {
|
|
return 'unsupported_r_minor';
|
|
}
|
|
if (area === 'slot') {
|
|
if (rest !== null && rest.startsWith('Archive/')) return 'archive_fallback';
|
|
if (rest !== null && INDEX_REGEX.test(rest)) return 'minor_routing';
|
|
return 'other';
|
|
}
|
|
if (area === 'bare') {
|
|
// The router resolves a bare URL only when it can identify the distro from
|
|
// the User-Agent; otherwise it sends the client to CRAN. A stock R
|
|
// User-Agent (`R (4.5.3 x86_64-pc-linux-gnu …)`) names no distro.
|
|
return DISTRO_REGEX.test(userAgent) && ARCH_REGEX.test(userAgent) ? 'slot_resolution' : 'unknown_distro';
|
|
}
|
|
return 'other';
|
|
}
|
|
|
|
/** Parse `"a.b.c.0/24,2001:db8::/64"` into prefix matchers for our own networks. */
|
|
export function parseNetworks(spec: string): ((ip: string) => boolean)[] {
|
|
return spec
|
|
.split(',')
|
|
.map((cidr) => cidr.trim())
|
|
.filter((cidr) => cidr.length > 0)
|
|
.map((cidr) => {
|
|
const [base, bitsText] = cidr.split('/');
|
|
const v6 = base.includes(':');
|
|
const bits = Number(bitsText ?? (v6 ? 128 : 32));
|
|
const baseBits = ipBits(base);
|
|
return (ip: string) => {
|
|
if (ip.includes(':') !== v6) return false;
|
|
const candidate = ipBits(ip);
|
|
return baseBits !== null && candidate !== null && candidate.slice(0, bits) === baseBits.slice(0, bits);
|
|
};
|
|
});
|
|
}
|
|
|
|
function ipBits(ip: string): string | null {
|
|
if (!ip.includes(':')) {
|
|
const octets = ip.split('.').map(Number);
|
|
if (octets.length !== 4 || octets.some((o) => !Number.isInteger(o) || o < 0 || o > 255)) return null;
|
|
return octets.map((o) => o.toString(2).padStart(8, '0')).join('');
|
|
}
|
|
const [head, tail] = ip.split('::');
|
|
const headGroups = head ? head.split(':') : [];
|
|
const tailGroups = tail ? tail.split(':') : [];
|
|
const missing = 8 - headGroups.length - tailGroups.length;
|
|
if (ip.includes('::') ? missing < 0 : missing !== 0) return null;
|
|
const groups = [...headGroups, ...Array(Math.max(missing, 0)).fill('0'), ...tailGroups];
|
|
return groups
|
|
.map((g) =>
|
|
parseInt(g || '0', 16)
|
|
.toString(2)
|
|
.padStart(16, '0'),
|
|
)
|
|
.join('');
|
|
}
|
|
|
|
/** Build the stored event. Returns null for entries without a request id. */
|
|
export function toEvent(
|
|
entry: LogEntry,
|
|
hashClient: (key: string) => string,
|
|
ownNetworks: ((ip: string) => boolean)[],
|
|
): Event | null {
|
|
if (!entry.requestId) return null;
|
|
const ip = entry.remoteIp ?? '';
|
|
const userAgent = entry.userAgent ?? '';
|
|
const raw: RawEvent = {
|
|
request_id: entry.requestId,
|
|
ts: entry.timestamp,
|
|
zone_id: entry.pullZoneId,
|
|
host: entry.host,
|
|
status: entry.statusCode,
|
|
cache_status: entry.cacheStatus,
|
|
bytes: entry.bytesSent,
|
|
path: entry.path ?? '',
|
|
user_agent: entry.userAgent,
|
|
country: entry.countryCode,
|
|
asn: entry.asn,
|
|
asn_org: entry.asnOrganization,
|
|
client: hashClient(`${ip}|${entry.asn ?? ''}|${clientPlatform(userAgent)}`),
|
|
own: ip !== '' && ownNetworks.some((matches) => matches(ip)),
|
|
};
|
|
return { ...raw, ...classify(raw) };
|
|
}
|