diff --git a/.crow/download-stats.yaml b/.crow/download-stats.yaml new file mode 100644 index 0000000..ef0f50a --- /dev/null +++ b/.crow/download-stats.yaml @@ -0,0 +1,79 @@ +# Collect CDN access logs and publish the public download statistics. +# +# Each run reads the last 70 hours of cran-rpkgs access logs from Bunny into +# `download_stats.events` (build metadata Postgres), then rewrites +# https://cran.rpkgs.com/stats/downloads.json, which the rpkgs.com /stats page +# reads. Bunny keeps logs for only three days, so a run every hour leaves room +# for two days of failed runs before anything is lost. Re-reading the window is +# safe: events are keyed by Bunny's request id. +# +# The classification rules and what is excluded are documented in +# `stats/README.md`. +# +# Register the `download-stats` cron (hourly, `0 * * * *`) in the crow UI, or +# run manually: +# crow pipeline create --branch main --var download_stats=true devxy/build-cran-binaries +# +# Secrets: +# STATS_CLIENT_SALT HMAC key for the client hash. Rotating it resets +# distinct-client counts, so keep it stable. +# STATS_OWN_NETWORKS Comma-separated CIDRs of our own CI runners. +# Both live in OpenBao at development/agent-dev/rpkgs-download-stats. +variables: + download_stats: + description: 'Collect and publish download statistics. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' + +when: + - event: manual + evaluate: 'download_stats == "true"' + - event: cron + cron: download-stats + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Collect and publish download stats' + image: oven/bun:1.4.4-alpine + pull: true + environment: + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + PGPASS: + from_secret: PGPASS + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + STATS_CLIENT_SALT: + from_secret: STATS_CLIENT_SALT + STATS_OWN_NETWORKS: + from_secret: STATS_OWN_NETWORKS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + commands: + - apk add --no-cache -q git + - git clone -q --depth 1 --branch "$CI_COMMIT_BRANCH" https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - bun test stats + - bun stats/collect.ts + - bun stats/publish.ts + backend_options: + kubernetes: + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 1000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.gitignore b/.gitignore index f4124a2..c072661 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,7 @@ docs/ # Superpowers SDD scratch (briefs, reports, ledger) .superpowers/ + +# Internal design docs (plans and specs) are kept out of the repo. +plans/ +specs/ diff --git a/cdn.tf b/cdn.tf index df30273..46b3e70 100644 --- a/cdn.tf +++ b/cdn.tf @@ -164,6 +164,23 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { add_canonical_header = true + # Access logs feed the public download statistics (`stats/`, published to + # `/stats/downloads.json`). Both settings were already on; declaring them + # keeps an apply from quietly switching off the only source of that data. + # Bunny keeps the logs for three days, which is why `download-stats` runs on + # a schedule. IPs stay anonymized to /24 (/64 for IPv6) at the edge. + log_enabled = true + log_anonymized = true + log_anonymized_style = "OneDigit" + + # The rpkgs.com website reads `/stats/downloads.json` from the browser, so + # JSON needs the CORS header too. The rest is the zone's existing list. + cors_enabled = true + cors_extensions = [ + "css", "eot", "gif", "jpeg", "jpg", "js", "json", "mp3", "mp4", "mpeg", "png", "svg", "ttf", "webm", "webp", + "woff", "woff2", + ] + cache_stale = ["offline", "updating"] block_ips = var.cdn_block_ips diff --git a/stats/README.md b/stats/README.md new file mode 100644 index 0000000..143f3c1 --- /dev/null +++ b/stats/README.md @@ -0,0 +1,63 @@ +# Download statistics + +This directory turns the cran.rpkgs.com CDN access logs into the public download numbers on [rpkgs.com/stats](https://rpkgs.com/stats). +The `download-stats` pipeline runs it every hour. + +## Data flow + +1. `collect.ts` reads the last 70 hours of access logs from the Bunny Logging API v2 and inserts each request into `download_stats.events` in the build metadata database. +2. `classify.ts` derives what each request was (package download, index fetch, redirect, probe, noise) from its path, status and User-Agent. +3. `publish.ts` aggregates the events into `stats/downloads.json` in the binaries bucket and purges its CDN copy, so it is served at . + +Bunny keeps access logs for three days. +Each hourly run re-reads almost all of that window, and events are keyed by Bunny's request id, so re-reading never double counts and up to two days of failed runs lose nothing. +Every run is recorded in `download_stats.runs`, and the dashboard marks any hour that no run covered. + +## Privacy + +Bunny anonymizes IPs at the edge to /24 (IPv4) or /64 (IPv6) before they reach the logs. +Not even that prefix is stored: each event keeps only `client`, an HMAC (keyed with `STATS_CLIENT_SALT`) of the prefix, the network (ASN) and the client platform from the User-Agent. +The public JSON contains counts only, never a client hash, IP or User-Agent. +Countries seen from fewer than three clients are folded into "other". + +## What counts + +| Metric | Definition | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Download | A distinct (client, package, version) served on one UTC day. Raw tarball requests are reported separately, because one CI job can re-download the same package many times a day. | +| Client | A distinct `client` hash that fetched an index or a package. Several machines behind one /24 count once, and one laptop on two networks counts twice, so this is an estimate. | +| Client-day | A client active on a given day, roughly one install session. | + +The same R session seen with and without renv's User-Agent prefix counts as one client, since `clientPlatform()` keys on the R part of the User-Agent. + +## What is separated or excluded + +- **Cloud and CI networks** are kept but reported separately: Microsoft/GitHub, AWS, Google, Hetzner, OVH and other hosting ASNs, plus any network whose name says hosting, cloud or data centre. +- **Our own CI runners** (`STATS_OWN_NETWORKS`) are excluded from every public number and only counted as excluded requests. +- **Internal redirects** (`minor_routing`, `slot_resolution`, `https`) are not counted, only the request they lead to. +- **Redirects away from us** are counted as their own metrics: `unsupported_r_minor` (demand for R versions we do not build), `archive_fallback` (old versions we have no binary for), `unknown_distro` and `macos`. +- **Probes**: renv and pak request `__api__/status` to check for Posit Package Manager. +- **Noise**: vulnerability scanners, crawlers and anything outside the repository layout. + +## Changing the rules + +Change `classify.ts`, add a test case to `classify.test.ts` and bump `RULESET`. +Then recompute every stored event: + +```sh +bun stats/collect.ts --reclassify +``` + +The raw path, status, User-Agent and network are stored for exactly this reason. +Whether a request came from our own runners is decided at ingest, so changing `STATS_OWN_NETWORKS` only affects new events. + +## Running locally + +The secrets are in OpenBao: the Bunny key at `development/agent-dev/bunny`, the database password at `development/agent-dev/rpkgs-metadata-db` and the salt and own networks at `development/agent-dev/rpkgs-download-stats`. + +```sh +bun test stats +bun stats/collect.ts +# write the JSON locally instead of uploading it +bun stats/publish.ts --out downloads.json +``` diff --git a/stats/classify.test.ts b/stats/classify.test.ts new file mode 100644 index 0000000..09c96f8 --- /dev/null +++ b/stats/classify.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, test } from 'bun:test'; +import { classify, clientPlatform, parseNetworks, toEvent, tool, type LogEntry } from './classify'; + +const R_RHEL = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)'; +const RENV_RHEL = 'renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)'; +const R_ALPINE = 'R/4.6.1 (Alpine Linux 3.24.1) (aarch64-unknown-linux-musl aarch64 linux-musl)'; +const R_43 = 'R (4.3.2 x86_64-pc-linux-gnu x86_64 linux-gnu)'; + +const base = { status: 200, user_agent: R_RHEL, asn: 3303, asn_org: 'Swisscom', own: false }; + +describe('classify', () => { + test('flat tarball is a download with package, version and slot', () => { + const d = classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/data.table_1.17.8.tar.gz' }); + expect(d).toMatchObject({ + kind: 'download', + outcome: 'served', + slot: 'amd64/rhel9', + arch: 'amd64', + os: 'rhel9', + package: 'data.table', + version: '1.17.8', + r_minor: '4.5', + r_version: '4.5.3', + tool: 'R', + network: 'other', + }); + }); + + test('per-minor tarball and index keep the slot', () => { + expect(classify({ ...base, path: '/arm64/alpine324/latest/src/contrib/4.6/Rcpp_1.1.0.tar.gz' })).toMatchObject({ + kind: 'download', + slot: 'arm64/alpine324', + package: 'Rcpp', + }); + expect(classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/4.5/PACKAGES.rds' }).kind).toBe('index'); + }); + + test('index redirect with a known minor is internal minor routing', () => { + const d = classify({ ...base, status: 302, path: '/amd64/rhel9/latest/src/contrib/PACKAGES.rds' }); + expect(d).toMatchObject({ kind: 'index', outcome: 'redirect', redirect_reason: 'minor_routing' }); + }); + + test('redirect for an R minor we do not build is unsupported_r_minor', () => { + const d = classify({ ...base, user_agent: R_43, status: 302, path: '/amd64/noble/latest/src/contrib/PACKAGES' }); + expect(d).toMatchObject({ redirect_reason: 'unsupported_r_minor', r_minor: '4.3' }); + }); + + test('archived version without a binary falls back to CRAN', () => { + const d = classify({ + ...base, + status: 302, + path: '/amd64/rhel9/latest/src/contrib/Archive/renv/renv_1.2.3.tar.gz', + }); + expect(d).toMatchObject({ kind: 'download', redirect_reason: 'archive_fallback', package: 'renv' }); + }); + + test('bare URL: identified distro resolves to a slot, stock R UA goes to CRAN', () => { + expect( + classify({ ...base, user_agent: R_ALPINE, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason, + ).toBe('slot_resolution'); + expect(classify({ ...base, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason).toBe( + 'unknown_distro', + ); + }); + + test('renv and pak probing for Posit Package Manager is a probe', () => { + expect(classify({ ...base, status: 404, path: '/amd64/__api__/status' }).kind).toBe('probe'); + }); + + test('scanner paths are noise', () => { + for (const path of ['/.env', '/wp-content/plugins/x.php', '/.git/config', '/xwpg.php']) { + expect(classify({ ...base, status: 404, user_agent: 'Mozilla/5.0', path }).kind).toBe('noise'); + } + }); + + test('the stats file is not counted as repository use', () => { + expect(classify({ ...base, path: '/stats/downloads.json' }).kind).toBe('stats'); + }); + + test('cloud networks are labelled, own networks win over cloud', () => { + expect(classify({ ...base, asn: 8075, asn_org: 'Microsoft Corporation', path: '/' }).network).toBe('cloud'); + expect(classify({ ...base, asn: null, asn_org: 'Some Hosting Ltd', path: '/' }).network).toBe('cloud'); + expect(classify({ ...base, asn: 24940, own: true, path: '/' }).network).toBe('own'); + }); + + test('query strings and doubled slashes do not change the result', () => { + expect(classify({ ...base, path: '/amd64//rhel9/latest/src/contrib/PACKAGES.rds?x=1' }).kind).toBe('index'); + }); +}); + +describe('client identity', () => { + test('renv and plain R from the same machine share one platform key', () => { + expect(clientPlatform(RENV_RHEL)).toBe(clientPlatform(R_RHEL)); + expect(clientPlatform(R_ALPINE)).not.toBe(clientPlatform(R_RHEL)); + }); + + test('tool detection', () => { + expect(tool(RENV_RHEL)).toBe('renv'); + expect(tool(R_RHEL)).toBe('R'); + expect(tool('pak/0.9.0 R/4.5.1 (...)')).toBe('pak'); + expect(tool('curl/8.5.0')).toBe('curl'); + expect(tool('-')).toBe('none'); + expect(tool('Mozilla/5.0 (Windows NT 10.0)')).toBe('browser'); + }); + + test('own networks match by prefix for IPv4 and IPv6', () => { + const own = parseNetworks('203.0.113.0/24, 2001:db8:1:a100::/64'); + const matches = (ip: string) => own.some((m) => m(ip)); + expect(matches('203.0.113.0')).toBe(true); + expect(matches('203.0.114.0')).toBe(false); + expect(matches('2001:db8:1:a100::')).toBe(true); + expect(matches('2001:db8:1:a101::')).toBe(false); + expect(parseNetworks('')).toHaveLength(0); + }); + + test('toEvent hashes the client and never keeps the IP', () => { + const entry: LogEntry = { + timestamp: '2026-11-14T11:45:33.689+00:00', + pullZoneId: 3857050, + requestId: 'abc', + cacheStatus: 'HIT', + statusCode: 200, + bytesSent: 1000, + remoteIp: '203.0.113.0', + countryCode: 'CH', + host: 'cran.rpkgs.com', + path: '/amd64/rhel9/latest/src/contrib/rlang_1.1.6.tar.gz', + userAgent: RENV_RHEL, + asn: 3303, + asnOrganization: 'Swisscom', + }; + const event = toEvent(entry, (key) => `h(${key.length})`, parseNetworks('203.0.113.0/24')); + expect(event).not.toBeNull(); + expect(JSON.stringify(event)).not.toContain('203.0.113'); + expect(event!.own).toBe(true); + expect(event!.network).toBe('own'); + expect(toEvent({ ...entry, requestId: null }, (k) => k, [])).toBeNull(); + }); +}); diff --git a/stats/classify.ts b/stats/classify.ts new file mode 100644 index 0000000..ab09ef6 --- /dev/null +++ b/stats/classify.ts @@ -0,0 +1,410 @@ +/** + * Turns one Bunny access-log entry into a classified download event. + * + * Everything here is pure so the rules can be unit-tested and re-run over the + * stored events (`collect.ts --reclassify`) whenever they change. Bump RULESET + * on any change that alters a derived column, so the public JSON can say which + * rules produced it. + * + * The rules mirror `edge/rpkgs-router.ts`: a redirect is classified by what the + * router would have done with that path and User-Agent, because the Location + * header is not in the logs. + */ + +export const RULESET = 1; + +/** R minors with a per-minor slot; keep in sync with KNOWN_MINORS in cdn.tf. */ +export const KNOWN_MINORS = new Set(['4.4', '4.5', '4.6']); + +export type Kind = + /** A package tarball. */ + | 'download' + /** `PACKAGES*`: one per repository per `install.packages()` call. */ + | 'index' + /** Other repository metadata, e.g. `Meta/archive.rds`. */ + | 'metadata' + /** renv and pak asking whether this is a Posit Package Manager (`__api__/…`). */ + | 'probe' + /** The repository root or `/src/contrib` without a file. */ + | 'root' + /** Our own `stats/` files, so the dashboard does not count itself. */ + | 'stats' + /** Anything else: vulnerability scanners, crawlers, typos. */ + | 'noise'; + +export type Outcome = 'served' | 'redirect' | 'error'; + +export type RedirectReason = + /** Internal hop: index sent to its per-R-minor slot. Not a user-visible event. */ + | 'minor_routing' + /** Internal hop: bare `cran.rpkgs.com` URL resolved to a slot. */ + | 'slot_resolution' + /** Client runs an R minor we do not build for; sent to CRAN for source. */ + | 'unsupported_r_minor' + /** Bare URL from a distro we could not identify; sent to CRAN. */ + | 'unknown_distro' + /** Archived version we have no binary for; sent to the CRAN source tarball. */ + | 'archive_fallback' + /** macOS client; sent to CRAN's macOS binaries. */ + | 'macos' + /** `http://` to `https://`. */ + | 'https' + | 'other'; + +export type Tool = 'R' | 'renv' | 'pak' | 'rig' | 'curl' | 'wget' | 'python' | 'browser' | 'none' | 'other'; + +export type Network = 'cloud' | 'other' | 'own'; + +export interface LogEntry { + timestamp: string; + pullZoneId: number; + requestId: string | null; + cacheStatus: string | null; + statusCode: number; + bytesSent: number; + remoteIp: string | null; + countryCode: string | null; + host: string | null; + path: string | null; + userAgent: string | null; + asn: number | null; + asnOrganization: string | null; +} + +export interface Derived { + ruleset: number; + kind: Kind; + outcome: Outcome; + redirect_reason: RedirectReason | null; + slot: string | null; + arch: string | null; + os: string | null; + r_minor: string | null; + r_version: string | null; + package: string | null; + version: string | null; + tool: Tool; + network: Network; +} + +/** The raw columns kept per event; everything Derived is recomputed from these. */ +export interface RawEvent { + request_id: string; + ts: string; + zone_id: number; + host: string | null; + status: number; + cache_status: string | null; + bytes: number; + path: string; + user_agent: string | null; + country: string | null; + asn: number | null; + asn_org: string | null; + /** HMAC of the (already anonymized) IP prefix, network and client platform. */ + client: string; + /** Matched one of our own networks at ingest. The IP itself is not kept. */ + own: boolean; +} + +export type Event = RawEvent & Derived; + +const SLOT_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib(?:\/(.*))?$/; +const BARE_REGEX = /^\/src\/contrib(?:\/(.*))?$/; +const MACOS_REGEX = /^\/bin\/macosx\/[^/]+\/contrib\/(\d+\.\d+)\/(.+)$/; +const MINOR_DIR_REGEX = /^(\d+\.\d+)\/(.*)$/; +const INDEX_REGEX = /^PACKAGES(\.gz|\.rds|\.json)?$/; +const TARBALL_REGEX = /(?:^|\/)([A-Za-z][A-Za-z0-9.]*)_([0-9][^/_]*)\.(?:tar\.gz|tgz)$/; +const R_VERSION_REGEXES = [/\bR\/(\d+\.\d+\.\d+)/, /\bR \((\d+\.\d+\.\d+)/]; +const ARCH_REGEX = /\b(x86_64|aarch64|arm64)\b/; +/** The distro forms `parseSlot()` in the router recognises. */ +const DISTRO_REGEX = /(almalinux|rocky)[^\d]*\d+|Ubuntu \d+\.\d+|Alpine Linux|alpine-\d/i; + +/** + * Networks whose clients are almost always machines: public clouds, CI + * providers and hosting companies. A request from one of these is reported as + * "cloud / CI" rather than dropped, since a CI pipeline is a real user too, + * just not a person at a laptop. + */ +const CLOUD_ASNS = new Map([ + [8075, 'Microsoft (incl. GitHub Actions runners)'], + [8068, 'Microsoft'], + [8069, 'Microsoft'], + [36459, 'GitHub'], + [16509, 'Amazon'], + [14618, 'Amazon'], + [15169, 'Google'], + [396982, 'Google Cloud'], + [24940, 'Hetzner'], + [213230, 'Hetzner Cloud'], + [212317, 'Hetzner'], + [16276, 'OVH'], + [14061, 'DigitalOcean'], + [51167, 'Contabo'], + [40021, 'Contabo US'], + [63949, 'Akamai / Linode'], + [31898, 'Oracle'], + [45102, 'Alibaba'], + [37963, 'Alibaba'], + [132203, 'Tencent'], + [45090, 'Tencent'], + [13335, 'Cloudflare'], + [62240, 'Clouvider'], + [12876, 'Scaleway'], + [20473, 'Vultr'], + [197540, 'netcup'], + [9009, 'M247'], + [60068, 'Datacamp / CDN77'], + [36352, 'ColoCrossing'], + [47583, 'Hostinger'], + [207990, 'HostRoyale'], +]); + +const CLOUD_ORG_REGEX = /\b(hosting|cloud|server(s)?|data ?cent(er|re)|colo(cation)?|vps|dedicated)\b/i; + +export function isCloudNetwork(asn: number | null, org: string | null): boolean { + if (asn !== null && CLOUD_ASNS.has(asn)) return true; + return org !== null && CLOUD_ORG_REGEX.test(org); +} + +export function normalizePath(path: string): string { + const withoutQuery = path.split('?')[0] ?? ''; + return withoutQuery.replace(/\/{2,}/g, '/'); +} + +export function rVersion(userAgent: string): string | null { + for (const regex of R_VERSION_REGEXES) { + const match = userAgent.match(regex); + if (match) return match[1]; + } + return null; +} + +export function rMinor(version: string | null): string | null { + if (!version) return null; + const [major, minor] = version.split('.'); + return `${major}.${minor}`; +} + +export function tool(userAgent: string): Tool { + const ua = userAgent.trim(); + if (ua === '' || ua === '-') return 'none'; + if (/^renv\b/i.test(ua)) return 'renv'; + if (/\bpak\b/i.test(ua)) return 'pak'; + if (/\brig\b/i.test(ua)) return 'rig'; + if (/^R[ /(]/.test(ua)) return 'R'; + if (/^curl\//i.test(ua)) return 'curl'; + if (/^wget\//i.test(ua)) return 'wget'; + if (/python|aiohttp|httpx/i.test(ua)) return 'python'; + if (/^Mozilla\//.test(ua)) return 'browser'; + return 'other'; +} + +/** + * The part of the User-Agent that identifies the machine rather than the tool. + * + * `renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` and + * `R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` are the same R session, one going + * through renv and one not. Keying distinct clients on the full User-Agent + * would count that machine twice. + */ +export function clientPlatform(userAgent: string): string { + const version = rVersion(userAgent); + if (!version) return userAgent; + const rPart = userAgent.slice(userAgent.search(/\bR[/ ]\(?\d/)); + return rPart.replace(/\bR\/\d+\.\d+\.\d+\s*/, '').trim() || version; +} + +function slotParts(slot: string | null): { arch: string | null; os: string | null } { + if (!slot) return { arch: null, os: null }; + const [arch, os] = slot.split('/'); + return { arch: arch ?? null, os: os ?? null }; +} + +function archFromUserAgent(userAgent: string): string | null { + const match = userAgent.match(ARCH_REGEX); + if (!match) return null; + return match[1] === 'x86_64' ? 'amd64' : 'arm64'; +} + +function fileKind(rest: string): Kind { + if (rest === '') return 'root'; + if (INDEX_REGEX.test(rest)) return 'index'; + if (TARBALL_REGEX.test(rest)) return 'download'; + if (/^Meta\//.test(rest) || /^PACKAGES/.test(rest)) return 'metadata'; + return 'noise'; +} + +function outcome(status: number): Outcome { + if (status === 301 || status === 302 || status === 303 || status === 307 || status === 308) return 'redirect'; + if (status >= 200 && status < 400) return 'served'; + return 'error'; +} + +export function classify( + raw: Pick, + scheme = 'https', +): Derived { + const path = normalizePath(raw.path); + const userAgent = raw.user_agent ?? ''; + const version = rVersion(userAgent); + const minor = rMinor(version); + const result: Derived = { + ruleset: RULESET, + kind: 'noise', + outcome: outcome(raw.status), + redirect_reason: null, + slot: null, + arch: null, + os: null, + r_minor: minor, + r_version: version, + package: null, + version: null, + tool: tool(userAgent), + network: raw.own ? 'own' : isCloudNetwork(raw.asn, raw.asn_org) ? 'cloud' : 'other', + }; + + let rest: string | null = null; + let area: 'slot' | 'bare' | 'macos' | null = null; + + const slot = path.match(SLOT_REGEX); + const bare = path.match(BARE_REGEX); + const macos = path.match(MACOS_REGEX); + + if (path.includes('/__api__/')) { + result.kind = 'probe'; + } else if (path.startsWith('/stats/')) { + result.kind = 'stats'; + } else if (path === '/') { + result.kind = 'root'; + area = 'bare'; + } else if (slot) { + area = 'slot'; + result.slot = `${slot[1]}/${slot[2]}`; + rest = slot[3] ?? ''; + } else if (bare) { + area = 'bare'; + rest = bare[1] ?? ''; + } else if (macos) { + area = 'macos'; + result.os = 'macos'; + rest = macos[2]; + } + + if (rest !== null) { + const minorDir = rest.match(MINOR_DIR_REGEX); + const file = minorDir ? minorDir[2] : rest; + result.kind = fileKind(file); + const tarball = file.match(TARBALL_REGEX); + if (tarball) { + result.package = tarball[1]; + result.version = tarball[2]; + } + } + + const parts = slotParts(result.slot); + result.arch = parts.arch ?? archFromUserAgent(userAgent); + result.os = parts.os ?? result.os; + + if (result.outcome === 'redirect') { + result.redirect_reason = redirectReason(raw.status, area, rest, minor, scheme, userAgent); + } + + return result; +} + +function redirectReason( + status: number, + area: 'slot' | 'bare' | 'macos' | null, + rest: string | null, + minor: string | null, + scheme: string, + userAgent: string, +): RedirectReason { + if (status === 301 || scheme === 'http') return 'https'; + if (area === 'macos' || (area === 'bare' && /darwin/.test(userAgent))) return 'macos'; + if (minor !== null && !KNOWN_MINORS.has(minor) && (area === 'slot' || area === 'bare')) { + return 'unsupported_r_minor'; + } + if (area === 'slot') { + if (rest !== null && rest.startsWith('Archive/')) return 'archive_fallback'; + if (rest !== null && INDEX_REGEX.test(rest)) return 'minor_routing'; + return 'other'; + } + if (area === 'bare') { + // The router resolves a bare URL only when it can identify the distro from + // the User-Agent; otherwise it sends the client to CRAN. A stock R + // User-Agent (`R (4.5.3 x86_64-pc-linux-gnu …)`) names no distro. + return DISTRO_REGEX.test(userAgent) && ARCH_REGEX.test(userAgent) ? 'slot_resolution' : 'unknown_distro'; + } + return 'other'; +} + +/** Parse `"a.b.c.0/24,2001:db8::/64"` into prefix matchers for our own networks. */ +export function parseNetworks(spec: string): ((ip: string) => boolean)[] { + return spec + .split(',') + .map((cidr) => cidr.trim()) + .filter((cidr) => cidr.length > 0) + .map((cidr) => { + const [base, bitsText] = cidr.split('/'); + const v6 = base.includes(':'); + const bits = Number(bitsText ?? (v6 ? 128 : 32)); + const baseBits = ipBits(base); + return (ip: string) => { + if (ip.includes(':') !== v6) return false; + const candidate = ipBits(ip); + return baseBits !== null && candidate !== null && candidate.slice(0, bits) === baseBits.slice(0, bits); + }; + }); +} + +function ipBits(ip: string): string | null { + if (!ip.includes(':')) { + const octets = ip.split('.').map(Number); + if (octets.length !== 4 || octets.some((o) => !Number.isInteger(o) || o < 0 || o > 255)) return null; + return octets.map((o) => o.toString(2).padStart(8, '0')).join(''); + } + const [head, tail] = ip.split('::'); + const headGroups = head ? head.split(':') : []; + const tailGroups = tail ? tail.split(':') : []; + const missing = 8 - headGroups.length - tailGroups.length; + if (ip.includes('::') ? missing < 0 : missing !== 0) return null; + const groups = [...headGroups, ...Array(Math.max(missing, 0)).fill('0'), ...tailGroups]; + return groups + .map((g) => + parseInt(g || '0', 16) + .toString(2) + .padStart(16, '0'), + ) + .join(''); +} + +/** Build the stored event. Returns null for entries without a request id. */ +export function toEvent( + entry: LogEntry, + hashClient: (key: string) => string, + ownNetworks: ((ip: string) => boolean)[], +): Event | null { + if (!entry.requestId) return null; + const ip = entry.remoteIp ?? ''; + const userAgent = entry.userAgent ?? ''; + const raw: RawEvent = { + request_id: entry.requestId, + ts: entry.timestamp, + zone_id: entry.pullZoneId, + host: entry.host, + status: entry.statusCode, + cache_status: entry.cacheStatus, + bytes: entry.bytesSent, + path: entry.path ?? '', + user_agent: entry.userAgent, + country: entry.countryCode, + asn: entry.asn, + asn_org: entry.asnOrganization, + client: hashClient(`${ip}|${entry.asn ?? ''}|${clientPlatform(userAgent)}`), + own: ip !== '' && ownNetworks.some((matches) => matches(ip)), + }; + return { ...raw, ...classify(raw) }; +} diff --git a/stats/collect.ts b/stats/collect.ts new file mode 100644 index 0000000..432c064 --- /dev/null +++ b/stats/collect.ts @@ -0,0 +1,157 @@ +/** + * Pull the CDN access logs into `download_stats.events`. + * + * bun stats/collect.ts # fetch the last 70 hours, insert what is new + * bun stats/collect.ts --reclassify # recompute derived columns after a rule change + * + * Bunny keeps logs for three days, so every run re-reads almost all of that + * window. Events are keyed by Bunny's request id, which makes re-reading free + * of double counts, and a run that fails is simply covered by the next one. + * + * Environment: BUNNYNET_API_KEY, PGPASS, STATS_CLIENT_SALT, and optionally + * STATS_PULL_ZONE (default: cran-rpkgs) and STATS_OWN_NETWORKS (comma-separated + * CIDRs of our own CI runners, kept out of the public numbers). + */ +import { classify, parseNetworks, toEvent, type Event, type LogEntry } from './classify'; +import { connect, migrate } from './db'; + +const PULL_ZONE = Number(process.env.STATS_PULL_ZONE ?? 3857050); +const WINDOW_HOURS = 70; +const PAGE_SIZE = 10000; + +const COLUMNS = [ + 'request_id', + 'ts', + 'zone_id', + 'host', + 'status', + 'cache_status', + 'bytes', + 'path', + 'user_agent', + 'country', + 'asn', + 'asn_org', + 'client', + 'own', + 'ruleset', + 'kind', + 'outcome', + 'redirect_reason', + 'slot', + 'arch', + 'os', + 'r_minor', + 'r_version', + 'package', + 'version', + 'tool', + 'network', +] as const; + +function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is not set`); + return value; +} + +async function fetchPage( + apiKey: string, + from: Date, + to: Date, + offset: number, +): Promise<{ data: LogEntry[]; hasMore: boolean }> { + const url = new URL(`https://logging.bunnycdn.com/v2/pullzones/${PULL_ZONE}/logs`); + url.searchParams.set('from', from.toISOString()); + url.searchParams.set('to', to.toISOString()); + url.searchParams.set('order', 'asc'); + url.searchParams.set('limit', String(PAGE_SIZE)); + url.searchParams.set('offset', String(offset)); + + for (let attempt = 1; ; attempt++) { + const response = await fetch(url, { headers: { AccessKey: apiKey, Accept: 'application/json' } }); + if (response.ok) { + const body = (await response.json()) as { data: LogEntry[]; pagination: { hasMore: boolean } }; + return { data: body.data, hasMore: body.pagination.hasMore }; + } + if ((response.status === 429 || response.status >= 500) && attempt < 6) { + await Bun.sleep(2000 * attempt); + continue; + } + throw new Error(`Bunny logging API returned ${response.status}: ${await response.text()}`); + } +} + +async function collect(): Promise { + const apiKey = requireEnv('BUNNYNET_API_KEY'); + const salt = requireEnv('STATS_CLIENT_SALT'); + const ownNetworks = parseNetworks(process.env.STATS_OWN_NETWORKS ?? ''); + const hashClient = (key: string) => new Bun.CryptoHasher('sha256', salt).update(key).digest('hex').slice(0, 20); + + // Fixing `to` up front keeps offset pagination stable while new requests + // keep arriving. + const to = new Date(); + const from = new Date(to.getTime() - WINDOW_HOURS * 3600 * 1000); + + const events: Event[] = []; + let fetched = 0; + for (let offset = 0; ;) { + const page = await fetchPage(apiKey, from, to, offset); + fetched += page.data.length; + for (const entry of page.data) { + const event = toEvent(entry, hashClient, ownNetworks); + if (event) events.push(event); + } + if (!page.hasMore || page.data.length === 0) break; + offset += page.data.length; + } + + const sql = connect(); + await migrate(sql); + let inserted = 0; + for (let i = 0; i < events.length; i += 1000) { + const batch = events.slice(i, i + 1000); + const rows = await sql` + insert into download_stats.events ${sql(batch, ...COLUMNS)} + on conflict (request_id) do nothing + returning 1`; + inserted += rows.length; + } + await sql` + insert into download_stats.runs (zone_id, window_from, window_to, fetched, inserted) + values (${PULL_ZONE}, ${from}, ${to}, ${fetched}, ${inserted})`; + await sql.close(); + + console.log( + `fetched ${fetched} log entries from ${from.toISOString()} to ${to.toISOString()}, inserted ${inserted} new`, + ); +} + +async function reclassify(): Promise { + const sql = connect(); + await migrate(sql); + const rows: Event[] = + await sql`select request_id, path, status, user_agent, asn, asn_org, own from download_stats.events`; + let updated = 0; + for (let i = 0; i < rows.length; i += 1000) { + const batch = rows.slice(i, i + 1000).map((row) => ({ request_id: row.request_id, ...classify(row) })); + await sql.begin(async (tx) => { + for (const row of batch) { + await tx` + update download_stats.events set + ruleset = ${row.ruleset}, kind = ${row.kind}, outcome = ${row.outcome}, + redirect_reason = ${row.redirect_reason}, slot = ${row.slot}, arch = ${row.arch}, os = ${row.os}, + r_minor = ${row.r_minor}, r_version = ${row.r_version}, package = ${row.package}, + version = ${row.version}, tool = ${row.tool}, network = ${row.network} + where request_id = ${row.request_id}`; + } + }); + updated += batch.length; + } + await sql.close(); + console.log(`reclassified ${updated} events`); +} + +if (import.meta.main) { + await (process.argv.includes('--reclassify') ? reclassify() : collect()); +} diff --git a/stats/db.ts b/stats/db.ts new file mode 100644 index 0000000..2169149 --- /dev/null +++ b/stats/db.ts @@ -0,0 +1,75 @@ +import { SQL } from 'bun'; + +/** + * Download events live in their own `download_stats` schema in the build + * metadata database, next to (but never joined with) `single_builds`. + * + * No IP address is stored, not even the /24 prefix Bunny logs: only a salted + * HMAC of it (`client`) and whether it matched one of our own networks. + */ +const SCHEMA = ` +create schema if not exists download_stats; + +create table if not exists download_stats.events ( + request_id text primary key, + ts timestamptz not null, + zone_id integer not null, + host text, + status smallint not null, + cache_status text, + bytes bigint not null, + path text not null, + user_agent text, + country text, + asn integer, + asn_org text, + client text not null, + own boolean not null, + ruleset smallint not null, + kind text not null, + outcome text not null, + redirect_reason text, + slot text, + arch text, + os text, + r_minor text, + r_version text, + package text, + version text, + tool text not null, + network text not null +); + +create index if not exists events_ts_idx on download_stats.events (ts); +create index if not exists events_package_idx on download_stats.events (package) where kind = 'download'; + +-- One row per collection run. The union of successful windows is what the +-- dashboard reports as covered; anything outside it is shown as a gap. +create table if not exists download_stats.runs ( + id bigserial primary key, + zone_id integer not null, + started_at timestamptz not null default now(), + window_from timestamptz not null, + window_to timestamptz not null, + fetched integer not null, + inserted integer not null +); +`; + +export function connect(): SQL { + const password = process.env.PGPASS; + if (!password) throw new Error('PGPASS is not set'); + return new SQL({ + hostname: process.env.PGHOST ?? 'r-binaries.devxy.io', + port: Number(process.env.PGPORT ?? 15432), + database: process.env.PGDATABASE ?? 'build_metadata', + username: process.env.PGUSER ?? 'rpkgs', + password, + tls: true, + max: 2, + }); +} + +export async function migrate(sql: SQL): Promise { + await sql.unsafe(SCHEMA); +} diff --git a/stats/publish.ts b/stats/publish.ts new file mode 100644 index 0000000..ae6e13d --- /dev/null +++ b/stats/publish.ts @@ -0,0 +1,339 @@ +/** + * Aggregate `download_stats.events` into the public `stats/downloads.json`. + * + * bun stats/publish.ts --out downloads.json # write locally only + * bun stats/publish.ts # upload to the bucket and purge the CDN copy + * + * The output holds counts only. Distinct clients are counted from the stored + * HMAC, which never leaves the database, and countries seen from fewer than + * MIN_CLIENTS clients are folded into "other" so no single client can be + * picked out by where it is. + * + * Every number excludes our own CI runners. Cloud and CI networks are kept but + * reported separately, since one pipeline can outweigh every person combined. + */ +import { S3Client, type SQL } from 'bun'; +import { KNOWN_MINORS, RULESET } from './classify'; +import { connect, migrate } from './db'; + +const MIN_CLIENTS = 3; +const PUBLIC_URL = 'https://cran.rpkgs.com/stats/downloads.json'; +const OBJECT_KEY = 'stats/downloads.json'; + +/** Split of a count by network class. */ +export interface Split { + total: number; + cloud: number; + other: number; +} + +export interface DayStats { + date: string; + /** Hours of this UTC day for which logs were collected (24 = complete). */ + coveredHours: number; + /** Tarball requests served, before any de-duplication. */ + requests: number; + /** Distinct (client, package, version) served on the day. */ + downloads: Split; + /** Distinct clients that fetched an index or a package. */ + clients: Split; + /** Distinct clients sent to CRAN because we do not build for their R minor. */ + unsupportedMinorClients: number; + bytes: number; +} + +export interface WindowStats { + days: number; + downloads: Split; + /** Distinct clients across the whole window. */ + clients: Split; + /** Sum of daily distinct clients: roughly, install sessions. */ + clientDays: Split; +} + +export interface Breakdown { + key: string; + clients: number; + downloads: number; +} + +export interface PackageStats { + name: string; + downloads30d: number; + clients30d: number; + downloadsAll: number; +} + +export interface PublicStats { + schema: 1; + ruleset: number; + generatedAt: string; + firstDay: string | null; + lastDay: string | null; + supportedMinors: string[]; + daily: DayStats[]; + windows: { last7: WindowStats; last30: WindowStats; all: WindowStats }; + /** Distribution of distinct packages fetched per client per day (30 days). */ + packagesPerClientDay: { median: number; p90: number }; + /** Last 30 days. */ + breakdowns: { + network: Breakdown[]; + os: Breakdown[]; + arch: Breakdown[]; + rMinor: Breakdown[]; + tool: Breakdown[]; + country: Breakdown[]; + }; + /** Last 30 days: clients we sent elsewhere, and why. */ + redirects: { reason: string; clients: number; requests: number }[]; + unsupportedMinors: Breakdown[]; + /** Last 30 days: requests that were not repository use at all. */ + excluded: { ownRunners: number; probes: number; noise: number; errors: number }; + packages: PackageStats[]; +} + +const split = (row: Record, prefix: string): Split => ({ + total: Number(row[`${prefix}_total`] ?? 0), + cloud: Number(row[`${prefix}_cloud`] ?? 0), + other: Number(row[`${prefix}_other`] ?? 0), +}); + +async function coverage(sql: SQL): Promise> { + const runs: { window_from: Date; window_to: Date }[] = + await sql`select window_from, window_to from download_stats.runs order by window_from`; + const merged: [number, number][] = []; + for (const run of runs) { + const from = run.window_from.getTime(); + const to = run.window_to.getTime(); + const last = merged.at(-1); + if (last && from <= last[1]) last[1] = Math.max(last[1], to); + else merged.push([from, to]); + } + const hours = new Map(); + const DAY = 86400000; + for (const [from, to] of merged) { + for (let day = Math.floor(from / DAY) * DAY; day < to; day += DAY) { + const overlap = Math.min(to, day + DAY) - Math.max(from, day); + const key = new Date(day).toISOString().slice(0, 10); + hours.set(key, (hours.get(key) ?? 0) + overlap / 3600000); + } + } + return hours; +} + +async function windowStats(sql: SQL, days: number | null): Promise { + const since = days === null ? new Date(0) : new Date(Date.now() - days * 86400000); + const [row] = await sql` + with ev as ( + select client, network, kind, package, version, (ts at time zone 'UTC')::date as day + from download_stats.events + where network <> 'own' and outcome = 'served' and kind in ('download', 'index') and ts >= ${since} + ), + dl as (select distinct client, network, package, version, day from ev where kind = 'download'), + cd as (select distinct client, network, day from ev) + select + (select count(*) from dl) as downloads_total, + (select count(*) from dl where network = 'cloud') as downloads_cloud, + (select count(*) from dl where network = 'other') as downloads_other, + (select count(distinct client) from ev) as clients_total, + (select count(distinct client) from ev where network = 'cloud') as clients_cloud, + (select count(distinct client) from ev where network = 'other') as clients_other, + (select count(*) from cd) as client_days_total, + (select count(*) from cd where network = 'cloud') as client_days_cloud, + (select count(*) from cd where network = 'other') as client_days_other, + (select min(day) from ev) as first_day`; + const firstDay = row.first_day ? new Date(row.first_day) : null; + const span = firstDay ? Math.max(1, Math.ceil((Date.now() - firstDay.getTime()) / 86400000)) : 0; + return { + days: days === null ? span : Math.min(days, span), + downloads: split(row, 'downloads'), + clients: split(row, 'clients'), + clientDays: split(row, 'client_days'), + }; +} + +async function breakdown(sql: SQL, column: string, since: Date): Promise { + const rows: { key: string | null; clients: number; downloads: number }[] = await sql` + with ev as ( + select ${sql(column)}::text as key, client, kind, package, version, (ts at time zone 'UTC')::date as day + from download_stats.events + where network <> 'own' and outcome = 'served' and kind in ('download', 'index') and ts >= ${since} + ) + select key, count(distinct client)::int as clients, + (select count(*) from (select distinct client, package, version, day from ev e2 + where e2.kind = 'download' and e2.key is not distinct from ev.key) d)::int as downloads + from ev group by key order by clients desc`; + return rows.map((row) => ({ key: row.key ?? 'unknown', clients: row.clients, downloads: row.downloads })); +} + +/** Fold rows below the client threshold into one "other" row. */ +function suppress(rows: Breakdown[]): Breakdown[] { + const kept = rows.filter((row) => row.clients >= MIN_CLIENTS); + const folded = rows.filter((row) => row.clients < MIN_CLIENTS); + if (folded.length === 0) return kept; + return [ + ...kept, + { + key: 'other', + clients: folded.reduce((sum, row) => sum + row.clients, 0), + downloads: folded.reduce((sum, row) => sum + row.downloads, 0), + }, + ]; +} + +export async function buildStats(sql: SQL): Promise { + const since30 = new Date(Date.now() - 30 * 86400000); + const covered = await coverage(sql); + + const dailyRows: Record[] = await sql` + with ev as ( + select *, (ts at time zone 'UTC')::date as day from download_stats.events where network <> 'own' + ), + served as (select * from ev where outcome = 'served' and kind in ('download', 'index')), + dl as (select distinct client, network, package, version, day from served where kind = 'download'), + cl as (select distinct client, network, day from served) + select d.day::text as date, + (select count(*) from served s where s.day = d.day and s.kind = 'download')::int as requests, + (select count(*) from dl where dl.day = d.day)::int as downloads_total, + (select count(*) from dl where dl.day = d.day and network = 'cloud')::int as downloads_cloud, + (select count(*) from dl where dl.day = d.day and network = 'other')::int as downloads_other, + (select count(*) from cl where cl.day = d.day)::int as clients_total, + (select count(*) from cl where cl.day = d.day and network = 'cloud')::int as clients_cloud, + (select count(*) from cl where cl.day = d.day and network = 'other')::int as clients_other, + (select count(distinct client) from ev e where e.day = d.day and e.redirect_reason = 'unsupported_r_minor')::int + as unsupported, + (select coalesce(sum(bytes), 0) from served s where s.day = d.day)::bigint as bytes + from (select distinct day from ev) d order by d.day`; + + const daily: DayStats[] = dailyRows.map((row) => ({ + date: String(row.date), + coveredHours: Math.round(Math.min(24, covered.get(String(row.date)) ?? 0) * 10) / 10, + requests: Number(row.requests), + downloads: split(row, 'downloads'), + clients: split(row, 'clients'), + unsupportedMinorClients: Number(row.unsupported), + bytes: Number(row.bytes), + })); + + const [perSession] = await sql` + with per as ( + select client, (ts at time zone 'UTC')::date as day, count(distinct (package, version)) as n + from download_stats.events + where network <> 'own' and outcome = 'served' and kind = 'download' and ts >= ${since30} + group by 1, 2 + ) + select coalesce(percentile_cont(0.5) within group (order by n), 0) as median, + coalesce(percentile_cont(0.9) within group (order by n), 0) as p90 + from per`; + + const redirects: { reason: string; clients: number; requests: number }[] = await sql` + select redirect_reason as reason, count(distinct client)::int as clients, count(*)::int as requests + from download_stats.events + where network <> 'own' and outcome = 'redirect' and ts >= ${since30} + and redirect_reason not in ('minor_routing', 'slot_resolution', 'https') + group by 1 order by clients desc`; + + const unsupportedMinors: Breakdown[] = await sql` + select r_minor as key, count(distinct client)::int as clients, count(*)::int as downloads + from download_stats.events + where network <> 'own' and redirect_reason = 'unsupported_r_minor' and ts >= ${since30} + group by 1 order by clients desc`; + + const [excluded] = await sql` + select + count(*) filter (where network = 'own')::int as own, + count(*) filter (where network <> 'own' and kind = 'probe')::int as probes, + count(*) filter (where network <> 'own' and kind = 'noise')::int as noise, + count(*) filter (where network <> 'own' and kind in ('download', 'index') and outcome = 'error')::int as errors + from download_stats.events where ts >= ${since30}`; + + const packages: PackageStats[] = ( + await sql` + with dl as ( + select distinct client, package, version, (ts at time zone 'UTC')::date as day, ts >= ${since30} as recent + from download_stats.events + where network <> 'own' and outcome = 'served' and kind = 'download' and package is not null + ) + select package as name, + count(*) filter (where recent)::int as downloads30d, + count(distinct client) filter (where recent)::int as clients30d, + count(*)::int as downloadsall + from dl group by package order by downloads30d desc, downloadsall desc, name` + ).map((row: Record) => ({ + name: String(row.name), + downloads30d: Number(row.downloads30d), + clients30d: Number(row.clients30d), + downloadsAll: Number(row.downloadsall), + })); + + return { + schema: 1, + ruleset: RULESET, + generatedAt: new Date().toISOString(), + firstDay: daily[0]?.date ?? null, + lastDay: daily.at(-1)?.date ?? null, + supportedMinors: [...KNOWN_MINORS], + daily, + windows: { + last7: await windowStats(sql, 7), + last30: await windowStats(sql, 30), + all: await windowStats(sql, null), + }, + packagesPerClientDay: { median: Number(perSession.median), p90: Number(perSession.p90) }, + breakdowns: { + network: await breakdown(sql, 'network', since30), + os: await breakdown(sql, 'os', since30), + arch: await breakdown(sql, 'arch', since30), + rMinor: await breakdown(sql, 'r_minor', since30), + tool: await breakdown(sql, 'tool', since30), + country: suppress(await breakdown(sql, 'country', since30)), + }, + redirects, + unsupportedMinors, + excluded: { + ownRunners: Number(excluded.own), + probes: Number(excluded.probes), + noise: Number(excluded.noise), + errors: Number(excluded.errors), + }, + packages, + }; +} + +async function upload(body: string): Promise { + const s3 = new S3Client({ + accessKeyId: process.env.B2_S3_ACCESS_KEY, + secretAccessKey: process.env.B2_S3_SECRET_KEY, + bucket: process.env.STATS_BUCKET ?? 'devxy-rpkgs-binaries', + endpoint: 'https://s3.eu-central-003.backblazeb2.com', + region: 'eu-central-003', + }); + await s3.write(OBJECT_KEY, body, { type: 'application/json' }); + + // The pull zone overrides origin caching with a one-year TTL, so the edge + // copy has to be purged for the new file to show up. + const purge = new URL('https://api.bunny.net/purge'); + purge.searchParams.set('url', PUBLIC_URL); + purge.searchParams.set('async', 'false'); + const response = await fetch(purge, { method: 'POST', headers: { AccessKey: process.env.BUNNYNET_API_KEY ?? '' } }); + if (!response.ok) throw new Error(`purging ${PUBLIC_URL} failed with ${response.status}: ${await response.text()}`); +} + +if (import.meta.main) { + const sql = connect(); + await migrate(sql); + const stats = await buildStats(sql); + await sql.close(); + const body = JSON.stringify(stats); + + const outIndex = process.argv.indexOf('--out'); + if (outIndex !== -1) { + await Bun.write(process.argv[outIndex + 1], JSON.stringify(stats, null, 2)); + console.log(`wrote ${process.argv[outIndex + 1]} (${body.length} bytes)`); + } else { + await upload(body); + console.log( + `published ${PUBLIC_URL} (${body.length} bytes, ${stats.daily.length} days, ${stats.packages.length} packages)`, + ); + } +}