feat(stats): collect CDN access logs and publish download statistics #251
8 changed files with 1279 additions and 0 deletions
feat(stats): collect CDN access logs and publish download statistics
Add an hourly download-stats pipeline that reads the cran-rpkgs Bunny access logs into a download_stats schema in the build metadata database and publishes aggregated counts to /stats/downloads.json for the rpkgs.com /stats page. - Classify each request (download, index, redirect reason, probe, noise) mirroring the edge router, and split cloud/CI networks from the rest. - De-duplicate downloads per client, package version and day, and key clients on a salted HMAC so no IP is stored or published. - Exclude our own CI runners and record run windows to show gaps. - Declare the zone's logging settings and allow CORS for JSON.
commit
658254afc5
79
.crow/download-stats.yaml
Normal file
79
.crow/download-stats.yaml
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
# Collect CDN access logs and publish the public download statistics.
|
||||
#
|
||||
# Each run reads the last 70 hours of cran-rpkgs access logs from Bunny into
|
||||
# `download_stats.events` (build metadata Postgres), then rewrites
|
||||
# https://cran.rpkgs.com/stats/downloads.json, which the rpkgs.com /stats page
|
||||
# reads. Bunny keeps logs for only three days, so a run every hour leaves room
|
||||
# for two days of failed runs before anything is lost. Re-reading the window is
|
||||
# safe: events are keyed by Bunny's request id.
|
||||
#
|
||||
# The classification rules and what is excluded are documented in
|
||||
# `stats/README.md`.
|
||||
#
|
||||
# Register the `download-stats` cron (hourly, `0 * * * *`) in the crow UI, or
|
||||
# run manually:
|
||||
# crow pipeline create --branch main --var download_stats=true devxy/build-cran-binaries
|
||||
#
|
||||
# Secrets:
|
||||
# STATS_CLIENT_SALT HMAC key for the client hash. Rotating it resets
|
||||
# distinct-client counts, so keep it stable.
|
||||
# STATS_OWN_NETWORKS Comma-separated CIDRs of our own CI runners.
|
||||
# Both live in OpenBao at development/agent-dev/rpkgs-download-stats.
|
||||
variables:
|
||||
download_stats:
|
||||
description: 'Collect and publish download statistics. Also gates this pipeline.'
|
||||
options:
|
||||
- 'true'
|
||||
- 'false'
|
||||
default: 'false'
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'download_stats == "true"'
|
||||
- event: cron
|
||||
cron: download-stats
|
||||
|
||||
skip_clone: true
|
||||
|
||||
labels:
|
||||
group: rpkgs-amd64
|
||||
|
||||
steps:
|
||||
- name: 'Collect and publish download stats'
|
||||
image: oven/bun:1.4.4-alpine
|
||||
pull: true
|
||||
environment:
|
||||
BUNNYNET_API_KEY:
|
||||
from_secret: BUNNYNET_API_KEY
|
||||
PGPASS:
|
||||
from_secret: PGPASS
|
||||
B2_S3_ACCESS_KEY:
|
||||
from_secret: B2_S3_ACCESS_KEY
|
||||
B2_S3_SECRET_KEY:
|
||||
from_secret: B2_S3_SECRET_KEY
|
||||
STATS_CLIENT_SALT:
|
||||
from_secret: STATS_CLIENT_SALT
|
||||
STATS_OWN_NETWORKS:
|
||||
from_secret: STATS_OWN_NETWORKS
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
commands:
|
||||
- apk add --no-cache -q git
|
||||
- git clone -q --depth 1 --branch "$CI_COMMIT_BRANCH" https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- bun test stats
|
||||
- bun stats/collect.ts
|
||||
- bun stats/publish.ts
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: 256Mi
|
||||
cpu: 250m
|
||||
limits:
|
||||
memory: 1Gi
|
||||
cpu: 1000m
|
||||
tolerations:
|
||||
- key: 'CI'
|
||||
operator: 'Equal'
|
||||
value: 'true'
|
||||
effect: 'NoSchedule'
|
||||
17
cdn.tf
17
cdn.tf
|
|
@ -164,6 +164,23 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" {
|
|||
|
||||
add_canonical_header = true
|
||||
|
||||
# Access logs feed the public download statistics (`stats/`, published to
|
||||
# `/stats/downloads.json`). Both settings were already on; declaring them
|
||||
# keeps an apply from quietly switching off the only source of that data.
|
||||
# Bunny keeps the logs for three days, which is why `download-stats` runs on
|
||||
# a schedule. IPs stay anonymized to /24 (/64 for IPv6) at the edge.
|
||||
log_enabled = true
|
||||
log_anonymized = true
|
||||
log_anonymized_style = "OneDigit"
|
||||
|
||||
# The rpkgs.com website reads `/stats/downloads.json` from the browser, so
|
||||
# JSON needs the CORS header too. The rest is the zone's existing list.
|
||||
cors_enabled = true
|
||||
cors_extensions = [
|
||||
"css", "eot", "gif", "jpeg", "jpg", "js", "json", "mp3", "mp4", "mpeg", "png", "svg", "ttf", "webm", "webp",
|
||||
"woff", "woff2",
|
||||
]
|
||||
|
||||
cache_stale = ["offline", "updating"]
|
||||
|
||||
block_ips = var.cdn_block_ips
|
||||
|
|
|
|||
63
stats/README.md
Normal file
63
stats/README.md
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
# Download statistics
|
||||
|
||||
This directory turns the cran.rpkgs.com CDN access logs into the public download numbers on [rpkgs.com/stats](https://rpkgs.com/stats).
|
||||
The `download-stats` pipeline runs it every hour.
|
||||
|
||||
## Data flow
|
||||
|
||||
1. `collect.ts` reads the last 70 hours of access logs from the Bunny Logging API v2 and inserts each request into `download_stats.events` in the build metadata database.
|
||||
2. `classify.ts` derives what each request was (package download, index fetch, redirect, probe, noise) from its path, status and User-Agent.
|
||||
3. `publish.ts` aggregates the events into `stats/downloads.json` in the binaries bucket and purges its CDN copy, so it is served at <https://cran.rpkgs.com/stats/downloads.json>.
|
||||
|
||||
Bunny keeps access logs for three days.
|
||||
Each hourly run re-reads almost all of that window, and events are keyed by Bunny's request id, so re-reading never double counts and up to two days of failed runs lose nothing.
|
||||
Every run is recorded in `download_stats.runs`, and the dashboard marks any hour that no run covered.
|
||||
|
||||
## Privacy
|
||||
|
||||
Bunny anonymizes IPs at the edge to /24 (IPv4) or /64 (IPv6) before they reach the logs.
|
||||
Not even that prefix is stored: each event keeps only `client`, an HMAC (keyed with `STATS_CLIENT_SALT`) of the prefix, the network (ASN) and the client platform from the User-Agent.
|
||||
The public JSON contains counts only, never a client hash, IP or User-Agent.
|
||||
Countries seen from fewer than three clients are folded into "other".
|
||||
|
||||
## What counts
|
||||
|
||||
| Metric | Definition |
|
||||
| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Download | A distinct (client, package, version) served on one UTC day. Raw tarball requests are reported separately, because one CI job can re-download the same package many times a day. |
|
||||
| Client | A distinct `client` hash that fetched an index or a package. Several machines behind one /24 count once, and one laptop on two networks counts twice, so this is an estimate. |
|
||||
| Client-day | A client active on a given day, roughly one install session. |
|
||||
|
||||
The same R session seen with and without renv's User-Agent prefix counts as one client, since `clientPlatform()` keys on the R part of the User-Agent.
|
||||
|
||||
## What is separated or excluded
|
||||
|
||||
- **Cloud and CI networks** are kept but reported separately: Microsoft/GitHub, AWS, Google, Hetzner, OVH and other hosting ASNs, plus any network whose name says hosting, cloud or data centre.
|
||||
- **Our own CI runners** (`STATS_OWN_NETWORKS`) are excluded from every public number and only counted as excluded requests.
|
||||
- **Internal redirects** (`minor_routing`, `slot_resolution`, `https`) are not counted, only the request they lead to.
|
||||
- **Redirects away from us** are counted as their own metrics: `unsupported_r_minor` (demand for R versions we do not build), `archive_fallback` (old versions we have no binary for), `unknown_distro` and `macos`.
|
||||
- **Probes**: renv and pak request `__api__/status` to check for Posit Package Manager.
|
||||
- **Noise**: vulnerability scanners, crawlers and anything outside the repository layout.
|
||||
|
||||
## Changing the rules
|
||||
|
||||
Change `classify.ts`, add a test case to `classify.test.ts` and bump `RULESET`.
|
||||
Then recompute every stored event:
|
||||
|
||||
```sh
|
||||
bun stats/collect.ts --reclassify
|
||||
```
|
||||
|
||||
The raw path, status, User-Agent and network are stored for exactly this reason.
|
||||
Whether a request came from our own runners is decided at ingest, so changing `STATS_OWN_NETWORKS` only affects new events.
|
||||
|
||||
## Running locally
|
||||
|
||||
The secrets are in OpenBao: the Bunny key at `development/agent-dev/bunny`, the database password at `development/agent-dev/rpkgs-metadata-db` and the salt and own networks at `development/agent-dev/rpkgs-download-stats`.
|
||||
|
||||
```sh
|
||||
bun test stats
|
||||
bun stats/collect.ts
|
||||
# write the JSON locally instead of uploading it
|
||||
bun stats/publish.ts --out downloads.json
|
||||
```
|
||||
139
stats/classify.test.ts
Normal file
139
stats/classify.test.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import { describe, expect, test } from 'bun:test';
|
||||
import { classify, clientPlatform, parseNetworks, toEvent, tool, type LogEntry } from './classify';
|
||||
|
||||
const R_RHEL = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)';
|
||||
const RENV_RHEL = 'renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu x86_64 linux-gnu)';
|
||||
const R_ALPINE = 'R/4.6.1 (Alpine Linux 3.24.1) (aarch64-unknown-linux-musl aarch64 linux-musl)';
|
||||
const R_43 = 'R (4.3.2 x86_64-pc-linux-gnu x86_64 linux-gnu)';
|
||||
|
||||
const base = { status: 200, user_agent: R_RHEL, asn: 3303, asn_org: 'Swisscom', own: false };
|
||||
|
||||
describe('classify', () => {
|
||||
test('flat tarball is a download with package, version and slot', () => {
|
||||
const d = classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/data.table_1.17.8.tar.gz' });
|
||||
expect(d).toMatchObject({
|
||||
kind: 'download',
|
||||
outcome: 'served',
|
||||
slot: 'amd64/rhel9',
|
||||
arch: 'amd64',
|
||||
os: 'rhel9',
|
||||
package: 'data.table',
|
||||
version: '1.17.8',
|
||||
r_minor: '4.5',
|
||||
r_version: '4.5.3',
|
||||
tool: 'R',
|
||||
network: 'other',
|
||||
});
|
||||
});
|
||||
|
||||
test('per-minor tarball and index keep the slot', () => {
|
||||
expect(classify({ ...base, path: '/arm64/alpine324/latest/src/contrib/4.6/Rcpp_1.1.0.tar.gz' })).toMatchObject({
|
||||
kind: 'download',
|
||||
slot: 'arm64/alpine324',
|
||||
package: 'Rcpp',
|
||||
});
|
||||
expect(classify({ ...base, path: '/amd64/rhel9/latest/src/contrib/4.5/PACKAGES.rds' }).kind).toBe('index');
|
||||
});
|
||||
|
||||
test('index redirect with a known minor is internal minor routing', () => {
|
||||
const d = classify({ ...base, status: 302, path: '/amd64/rhel9/latest/src/contrib/PACKAGES.rds' });
|
||||
expect(d).toMatchObject({ kind: 'index', outcome: 'redirect', redirect_reason: 'minor_routing' });
|
||||
});
|
||||
|
||||
test('redirect for an R minor we do not build is unsupported_r_minor', () => {
|
||||
const d = classify({ ...base, user_agent: R_43, status: 302, path: '/amd64/noble/latest/src/contrib/PACKAGES' });
|
||||
expect(d).toMatchObject({ redirect_reason: 'unsupported_r_minor', r_minor: '4.3' });
|
||||
});
|
||||
|
||||
test('archived version without a binary falls back to CRAN', () => {
|
||||
const d = classify({
|
||||
...base,
|
||||
status: 302,
|
||||
path: '/amd64/rhel9/latest/src/contrib/Archive/renv/renv_1.2.3.tar.gz',
|
||||
});
|
||||
expect(d).toMatchObject({ kind: 'download', redirect_reason: 'archive_fallback', package: 'renv' });
|
||||
});
|
||||
|
||||
test('bare URL: identified distro resolves to a slot, stock R UA goes to CRAN', () => {
|
||||
expect(
|
||||
classify({ ...base, user_agent: R_ALPINE, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason,
|
||||
).toBe('slot_resolution');
|
||||
expect(classify({ ...base, status: 302, path: '/src/contrib/PACKAGES.rds' }).redirect_reason).toBe(
|
||||
'unknown_distro',
|
||||
);
|
||||
});
|
||||
|
||||
test('renv and pak probing for Posit Package Manager is a probe', () => {
|
||||
expect(classify({ ...base, status: 404, path: '/amd64/__api__/status' }).kind).toBe('probe');
|
||||
});
|
||||
|
||||
test('scanner paths are noise', () => {
|
||||
for (const path of ['/.env', '/wp-content/plugins/x.php', '/.git/config', '/xwpg.php']) {
|
||||
expect(classify({ ...base, status: 404, user_agent: 'Mozilla/5.0', path }).kind).toBe('noise');
|
||||
}
|
||||
});
|
||||
|
||||
test('the stats file is not counted as repository use', () => {
|
||||
expect(classify({ ...base, path: '/stats/downloads.json' }).kind).toBe('stats');
|
||||
});
|
||||
|
||||
test('cloud networks are labelled, own networks win over cloud', () => {
|
||||
expect(classify({ ...base, asn: 8075, asn_org: 'Microsoft Corporation', path: '/' }).network).toBe('cloud');
|
||||
expect(classify({ ...base, asn: null, asn_org: 'Some Hosting Ltd', path: '/' }).network).toBe('cloud');
|
||||
expect(classify({ ...base, asn: 24940, own: true, path: '/' }).network).toBe('own');
|
||||
});
|
||||
|
||||
test('query strings and doubled slashes do not change the result', () => {
|
||||
expect(classify({ ...base, path: '/amd64//rhel9/latest/src/contrib/PACKAGES.rds?x=1' }).kind).toBe('index');
|
||||
});
|
||||
});
|
||||
|
||||
describe('client identity', () => {
|
||||
test('renv and plain R from the same machine share one platform key', () => {
|
||||
expect(clientPlatform(RENV_RHEL)).toBe(clientPlatform(R_RHEL));
|
||||
expect(clientPlatform(R_ALPINE)).not.toBe(clientPlatform(R_RHEL));
|
||||
});
|
||||
|
||||
test('tool detection', () => {
|
||||
expect(tool(RENV_RHEL)).toBe('renv');
|
||||
expect(tool(R_RHEL)).toBe('R');
|
||||
expect(tool('pak/0.9.0 R/4.5.1 (...)')).toBe('pak');
|
||||
expect(tool('curl/8.5.0')).toBe('curl');
|
||||
expect(tool('-')).toBe('none');
|
||||
expect(tool('Mozilla/5.0 (Windows NT 10.0)')).toBe('browser');
|
||||
});
|
||||
|
||||
test('own networks match by prefix for IPv4 and IPv6', () => {
|
||||
const own = parseNetworks('203.0.113.0/24, 2001:db8:1:a100::/64');
|
||||
const matches = (ip: string) => own.some((m) => m(ip));
|
||||
expect(matches('203.0.113.0')).toBe(true);
|
||||
expect(matches('203.0.114.0')).toBe(false);
|
||||
expect(matches('2001:db8:1:a100::')).toBe(true);
|
||||
expect(matches('2001:db8:1:a101::')).toBe(false);
|
||||
expect(parseNetworks('')).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('toEvent hashes the client and never keeps the IP', () => {
|
||||
const entry: LogEntry = {
|
||||
timestamp: '2026-11-14T11:45:33.689+00:00',
|
||||
pullZoneId: 3857050,
|
||||
requestId: 'abc',
|
||||
cacheStatus: 'HIT',
|
||||
statusCode: 200,
|
||||
bytesSent: 1000,
|
||||
remoteIp: '203.0.113.0',
|
||||
countryCode: 'CH',
|
||||
host: 'cran.rpkgs.com',
|
||||
path: '/amd64/rhel9/latest/src/contrib/rlang_1.1.6.tar.gz',
|
||||
userAgent: RENV_RHEL,
|
||||
asn: 3303,
|
||||
asnOrganization: 'Swisscom',
|
||||
};
|
||||
const event = toEvent(entry, (key) => `h(${key.length})`, parseNetworks('203.0.113.0/24'));
|
||||
expect(event).not.toBeNull();
|
||||
expect(JSON.stringify(event)).not.toContain('203.0.113');
|
||||
expect(event!.own).toBe(true);
|
||||
expect(event!.network).toBe('own');
|
||||
expect(toEvent({ ...entry, requestId: null }, (k) => k, [])).toBeNull();
|
||||
});
|
||||
});
|
||||
410
stats/classify.ts
Normal file
410
stats/classify.ts
Normal file
|
|
@ -0,0 +1,410 @@
|
|||
/**
|
||||
* Turns one Bunny access-log entry into a classified download event.
|
||||
*
|
||||
* Everything here is pure so the rules can be unit-tested and re-run over the
|
||||
* stored events (`collect.ts --reclassify`) whenever they change. Bump RULESET
|
||||
* on any change that alters a derived column, so the public JSON can say which
|
||||
* rules produced it.
|
||||
*
|
||||
* The rules mirror `edge/rpkgs-router.ts`: a redirect is classified by what the
|
||||
* router would have done with that path and User-Agent, because the Location
|
||||
* header is not in the logs.
|
||||
*/
|
||||
|
||||
export const RULESET = 1;
|
||||
|
||||
/** R minors with a per-minor slot; keep in sync with KNOWN_MINORS in cdn.tf. */
|
||||
export const KNOWN_MINORS = new Set(['4.4', '4.5', '4.6']);
|
||||
|
||||
export type Kind =
|
||||
/** A package tarball. */
|
||||
| 'download'
|
||||
/** `PACKAGES*`: one per repository per `install.packages()` call. */
|
||||
| 'index'
|
||||
/** Other repository metadata, e.g. `Meta/archive.rds`. */
|
||||
| 'metadata'
|
||||
/** renv and pak asking whether this is a Posit Package Manager (`__api__/…`). */
|
||||
| 'probe'
|
||||
/** The repository root or `/src/contrib` without a file. */
|
||||
| 'root'
|
||||
/** Our own `stats/` files, so the dashboard does not count itself. */
|
||||
| 'stats'
|
||||
/** Anything else: vulnerability scanners, crawlers, typos. */
|
||||
| 'noise';
|
||||
|
||||
export type Outcome = 'served' | 'redirect' | 'error';
|
||||
|
||||
export type RedirectReason =
|
||||
/** Internal hop: index sent to its per-R-minor slot. Not a user-visible event. */
|
||||
| 'minor_routing'
|
||||
/** Internal hop: bare `cran.rpkgs.com` URL resolved to a slot. */
|
||||
| 'slot_resolution'
|
||||
/** Client runs an R minor we do not build for; sent to CRAN for source. */
|
||||
| 'unsupported_r_minor'
|
||||
/** Bare URL from a distro we could not identify; sent to CRAN. */
|
||||
| 'unknown_distro'
|
||||
/** Archived version we have no binary for; sent to the CRAN source tarball. */
|
||||
| 'archive_fallback'
|
||||
/** macOS client; sent to CRAN's macOS binaries. */
|
||||
| 'macos'
|
||||
/** `http://` to `https://`. */
|
||||
| 'https'
|
||||
| 'other';
|
||||
|
||||
export type Tool = 'R' | 'renv' | 'pak' | 'rig' | 'curl' | 'wget' | 'python' | 'browser' | 'none' | 'other';
|
||||
|
||||
export type Network = 'cloud' | 'other' | 'own';
|
||||
|
||||
export interface LogEntry {
|
||||
timestamp: string;
|
||||
pullZoneId: number;
|
||||
requestId: string | null;
|
||||
cacheStatus: string | null;
|
||||
statusCode: number;
|
||||
bytesSent: number;
|
||||
remoteIp: string | null;
|
||||
countryCode: string | null;
|
||||
host: string | null;
|
||||
path: string | null;
|
||||
userAgent: string | null;
|
||||
asn: number | null;
|
||||
asnOrganization: string | null;
|
||||
}
|
||||
|
||||
export interface Derived {
|
||||
ruleset: number;
|
||||
kind: Kind;
|
||||
outcome: Outcome;
|
||||
redirect_reason: RedirectReason | null;
|
||||
slot: string | null;
|
||||
arch: string | null;
|
||||
os: string | null;
|
||||
r_minor: string | null;
|
||||
r_version: string | null;
|
||||
package: string | null;
|
||||
version: string | null;
|
||||
tool: Tool;
|
||||
network: Network;
|
||||
}
|
||||
|
||||
/** The raw columns kept per event; everything Derived is recomputed from these. */
|
||||
export interface RawEvent {
|
||||
request_id: string;
|
||||
ts: string;
|
||||
zone_id: number;
|
||||
host: string | null;
|
||||
status: number;
|
||||
cache_status: string | null;
|
||||
bytes: number;
|
||||
path: string;
|
||||
user_agent: string | null;
|
||||
country: string | null;
|
||||
asn: number | null;
|
||||
asn_org: string | null;
|
||||
/** HMAC of the (already anonymized) IP prefix, network and client platform. */
|
||||
client: string;
|
||||
/** Matched one of our own networks at ingest. The IP itself is not kept. */
|
||||
own: boolean;
|
||||
}
|
||||
|
||||
export type Event = RawEvent & Derived;
|
||||
|
||||
const SLOT_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib(?:\/(.*))?$/;
|
||||
const BARE_REGEX = /^\/src\/contrib(?:\/(.*))?$/;
|
||||
const MACOS_REGEX = /^\/bin\/macosx\/[^/]+\/contrib\/(\d+\.\d+)\/(.+)$/;
|
||||
const MINOR_DIR_REGEX = /^(\d+\.\d+)\/(.*)$/;
|
||||
const INDEX_REGEX = /^PACKAGES(\.gz|\.rds|\.json)?$/;
|
||||
const TARBALL_REGEX = /(?:^|\/)([A-Za-z][A-Za-z0-9.]*)_([0-9][^/_]*)\.(?:tar\.gz|tgz)$/;
|
||||
const R_VERSION_REGEXES = [/\bR\/(\d+\.\d+\.\d+)/, /\bR \((\d+\.\d+\.\d+)/];
|
||||
const ARCH_REGEX = /\b(x86_64|aarch64|arm64)\b/;
|
||||
/** The distro forms `parseSlot()` in the router recognises. */
|
||||
const DISTRO_REGEX = /(almalinux|rocky)[^\d]*\d+|Ubuntu \d+\.\d+|Alpine Linux|alpine-\d/i;
|
||||
|
||||
/**
|
||||
* Networks whose clients are almost always machines: public clouds, CI
|
||||
* providers and hosting companies. A request from one of these is reported as
|
||||
* "cloud / CI" rather than dropped, since a CI pipeline is a real user too,
|
||||
* just not a person at a laptop.
|
||||
*/
|
||||
const CLOUD_ASNS = new Map<number, string>([
|
||||
[8075, 'Microsoft (incl. GitHub Actions runners)'],
|
||||
[8068, 'Microsoft'],
|
||||
[8069, 'Microsoft'],
|
||||
[36459, 'GitHub'],
|
||||
[16509, 'Amazon'],
|
||||
[14618, 'Amazon'],
|
||||
[15169, 'Google'],
|
||||
[396982, 'Google Cloud'],
|
||||
[24940, 'Hetzner'],
|
||||
[213230, 'Hetzner Cloud'],
|
||||
[212317, 'Hetzner'],
|
||||
[16276, 'OVH'],
|
||||
[14061, 'DigitalOcean'],
|
||||
[51167, 'Contabo'],
|
||||
[40021, 'Contabo US'],
|
||||
[63949, 'Akamai / Linode'],
|
||||
[31898, 'Oracle'],
|
||||
[45102, 'Alibaba'],
|
||||
[37963, 'Alibaba'],
|
||||
[132203, 'Tencent'],
|
||||
[45090, 'Tencent'],
|
||||
[13335, 'Cloudflare'],
|
||||
[62240, 'Clouvider'],
|
||||
[12876, 'Scaleway'],
|
||||
[20473, 'Vultr'],
|
||||
[197540, 'netcup'],
|
||||
[9009, 'M247'],
|
||||
[60068, 'Datacamp / CDN77'],
|
||||
[36352, 'ColoCrossing'],
|
||||
[47583, 'Hostinger'],
|
||||
[207990, 'HostRoyale'],
|
||||
]);
|
||||
|
||||
const CLOUD_ORG_REGEX = /\b(hosting|cloud|server(s)?|data ?cent(er|re)|colo(cation)?|vps|dedicated)\b/i;
|
||||
|
||||
export function isCloudNetwork(asn: number | null, org: string | null): boolean {
|
||||
if (asn !== null && CLOUD_ASNS.has(asn)) return true;
|
||||
return org !== null && CLOUD_ORG_REGEX.test(org);
|
||||
}
|
||||
|
||||
export function normalizePath(path: string): string {
|
||||
const withoutQuery = path.split('?')[0] ?? '';
|
||||
return withoutQuery.replace(/\/{2,}/g, '/');
|
||||
}
|
||||
|
||||
export function rVersion(userAgent: string): string | null {
|
||||
for (const regex of R_VERSION_REGEXES) {
|
||||
const match = userAgent.match(regex);
|
||||
if (match) return match[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function rMinor(version: string | null): string | null {
|
||||
if (!version) return null;
|
||||
const [major, minor] = version.split('.');
|
||||
return `${major}.${minor}`;
|
||||
}
|
||||
|
||||
export function tool(userAgent: string): Tool {
|
||||
const ua = userAgent.trim();
|
||||
if (ua === '' || ua === '-') return 'none';
|
||||
if (/^renv\b/i.test(ua)) return 'renv';
|
||||
if (/\bpak\b/i.test(ua)) return 'pak';
|
||||
if (/\brig\b/i.test(ua)) return 'rig';
|
||||
if (/^R[ /(]/.test(ua)) return 'R';
|
||||
if (/^curl\//i.test(ua)) return 'curl';
|
||||
if (/^wget\//i.test(ua)) return 'wget';
|
||||
if (/python|aiohttp|httpx/i.test(ua)) return 'python';
|
||||
if (/^Mozilla\//.test(ua)) return 'browser';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
/**
|
||||
* The part of the User-Agent that identifies the machine rather than the tool.
|
||||
*
|
||||
* `renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` and
|
||||
* `R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` are the same R session, one going
|
||||
* through renv and one not. Keying distinct clients on the full User-Agent
|
||||
* would count that machine twice.
|
||||
*/
|
||||
export function clientPlatform(userAgent: string): string {
|
||||
const version = rVersion(userAgent);
|
||||
if (!version) return userAgent;
|
||||
const rPart = userAgent.slice(userAgent.search(/\bR[/ ]\(?\d/));
|
||||
return rPart.replace(/\bR\/\d+\.\d+\.\d+\s*/, '').trim() || version;
|
||||
}
|
||||
|
||||
function slotParts(slot: string | null): { arch: string | null; os: string | null } {
|
||||
if (!slot) return { arch: null, os: null };
|
||||
const [arch, os] = slot.split('/');
|
||||
return { arch: arch ?? null, os: os ?? null };
|
||||
}
|
||||
|
||||
function archFromUserAgent(userAgent: string): string | null {
|
||||
const match = userAgent.match(ARCH_REGEX);
|
||||
if (!match) return null;
|
||||
return match[1] === 'x86_64' ? 'amd64' : 'arm64';
|
||||
}
|
||||
|
||||
function fileKind(rest: string): Kind {
|
||||
if (rest === '') return 'root';
|
||||
if (INDEX_REGEX.test(rest)) return 'index';
|
||||
if (TARBALL_REGEX.test(rest)) return 'download';
|
||||
if (/^Meta\//.test(rest) || /^PACKAGES/.test(rest)) return 'metadata';
|
||||
return 'noise';
|
||||
}
|
||||
|
||||
function outcome(status: number): Outcome {
|
||||
if (status === 301 || status === 302 || status === 303 || status === 307 || status === 308) return 'redirect';
|
||||
if (status >= 200 && status < 400) return 'served';
|
||||
return 'error';
|
||||
}
|
||||
|
||||
export function classify(
|
||||
raw: Pick<RawEvent, 'path' | 'status' | 'user_agent' | 'asn' | 'asn_org' | 'own'>,
|
||||
scheme = 'https',
|
||||
): Derived {
|
||||
const path = normalizePath(raw.path);
|
||||
const userAgent = raw.user_agent ?? '';
|
||||
const version = rVersion(userAgent);
|
||||
const minor = rMinor(version);
|
||||
const result: Derived = {
|
||||
ruleset: RULESET,
|
||||
kind: 'noise',
|
||||
outcome: outcome(raw.status),
|
||||
redirect_reason: null,
|
||||
slot: null,
|
||||
arch: null,
|
||||
os: null,
|
||||
r_minor: minor,
|
||||
r_version: version,
|
||||
package: null,
|
||||
version: null,
|
||||
tool: tool(userAgent),
|
||||
network: raw.own ? 'own' : isCloudNetwork(raw.asn, raw.asn_org) ? 'cloud' : 'other',
|
||||
};
|
||||
|
||||
let rest: string | null = null;
|
||||
let area: 'slot' | 'bare' | 'macos' | null = null;
|
||||
|
||||
const slot = path.match(SLOT_REGEX);
|
||||
const bare = path.match(BARE_REGEX);
|
||||
const macos = path.match(MACOS_REGEX);
|
||||
|
||||
if (path.includes('/__api__/')) {
|
||||
result.kind = 'probe';
|
||||
} else if (path.startsWith('/stats/')) {
|
||||
result.kind = 'stats';
|
||||
} else if (path === '/') {
|
||||
result.kind = 'root';
|
||||
area = 'bare';
|
||||
} else if (slot) {
|
||||
area = 'slot';
|
||||
result.slot = `${slot[1]}/${slot[2]}`;
|
||||
rest = slot[3] ?? '';
|
||||
} else if (bare) {
|
||||
area = 'bare';
|
||||
rest = bare[1] ?? '';
|
||||
} else if (macos) {
|
||||
area = 'macos';
|
||||
result.os = 'macos';
|
||||
rest = macos[2];
|
||||
}
|
||||
|
||||
if (rest !== null) {
|
||||
const minorDir = rest.match(MINOR_DIR_REGEX);
|
||||
const file = minorDir ? minorDir[2] : rest;
|
||||
result.kind = fileKind(file);
|
||||
const tarball = file.match(TARBALL_REGEX);
|
||||
if (tarball) {
|
||||
result.package = tarball[1];
|
||||
result.version = tarball[2];
|
||||
}
|
||||
}
|
||||
|
||||
const parts = slotParts(result.slot);
|
||||
result.arch = parts.arch ?? archFromUserAgent(userAgent);
|
||||
result.os = parts.os ?? result.os;
|
||||
|
||||
if (result.outcome === 'redirect') {
|
||||
result.redirect_reason = redirectReason(raw.status, area, rest, minor, scheme, userAgent);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function redirectReason(
|
||||
status: number,
|
||||
area: 'slot' | 'bare' | 'macos' | null,
|
||||
rest: string | null,
|
||||
minor: string | null,
|
||||
scheme: string,
|
||||
userAgent: string,
|
||||
): RedirectReason {
|
||||
if (status === 301 || scheme === 'http') return 'https';
|
||||
if (area === 'macos' || (area === 'bare' && /darwin/.test(userAgent))) return 'macos';
|
||||
if (minor !== null && !KNOWN_MINORS.has(minor) && (area === 'slot' || area === 'bare')) {
|
||||
return 'unsupported_r_minor';
|
||||
}
|
||||
if (area === 'slot') {
|
||||
if (rest !== null && rest.startsWith('Archive/')) return 'archive_fallback';
|
||||
if (rest !== null && INDEX_REGEX.test(rest)) return 'minor_routing';
|
||||
return 'other';
|
||||
}
|
||||
if (area === 'bare') {
|
||||
// The router resolves a bare URL only when it can identify the distro from
|
||||
// the User-Agent; otherwise it sends the client to CRAN. A stock R
|
||||
// User-Agent (`R (4.5.3 x86_64-pc-linux-gnu …)`) names no distro.
|
||||
return DISTRO_REGEX.test(userAgent) && ARCH_REGEX.test(userAgent) ? 'slot_resolution' : 'unknown_distro';
|
||||
}
|
||||
return 'other';
|
||||
}
|
||||
|
||||
/** Parse `"a.b.c.0/24,2001:db8::/64"` into prefix matchers for our own networks. */
|
||||
export function parseNetworks(spec: string): ((ip: string) => boolean)[] {
|
||||
return spec
|
||||
.split(',')
|
||||
.map((cidr) => cidr.trim())
|
||||
.filter((cidr) => cidr.length > 0)
|
||||
.map((cidr) => {
|
||||
const [base, bitsText] = cidr.split('/');
|
||||
const v6 = base.includes(':');
|
||||
const bits = Number(bitsText ?? (v6 ? 128 : 32));
|
||||
const baseBits = ipBits(base);
|
||||
return (ip: string) => {
|
||||
if (ip.includes(':') !== v6) return false;
|
||||
const candidate = ipBits(ip);
|
||||
return baseBits !== null && candidate !== null && candidate.slice(0, bits) === baseBits.slice(0, bits);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function ipBits(ip: string): string | null {
|
||||
if (!ip.includes(':')) {
|
||||
const octets = ip.split('.').map(Number);
|
||||
if (octets.length !== 4 || octets.some((o) => !Number.isInteger(o) || o < 0 || o > 255)) return null;
|
||||
return octets.map((o) => o.toString(2).padStart(8, '0')).join('');
|
||||
}
|
||||
const [head, tail] = ip.split('::');
|
||||
const headGroups = head ? head.split(':') : [];
|
||||
const tailGroups = tail ? tail.split(':') : [];
|
||||
const missing = 8 - headGroups.length - tailGroups.length;
|
||||
if (ip.includes('::') ? missing < 0 : missing !== 0) return null;
|
||||
const groups = [...headGroups, ...Array(Math.max(missing, 0)).fill('0'), ...tailGroups];
|
||||
return groups
|
||||
.map((g) =>
|
||||
parseInt(g || '0', 16)
|
||||
.toString(2)
|
||||
.padStart(16, '0'),
|
||||
)
|
||||
.join('');
|
||||
}
|
||||
|
||||
/** Build the stored event. Returns null for entries without a request id. */
|
||||
export function toEvent(
|
||||
entry: LogEntry,
|
||||
hashClient: (key: string) => string,
|
||||
ownNetworks: ((ip: string) => boolean)[],
|
||||
): Event | null {
|
||||
if (!entry.requestId) return null;
|
||||
const ip = entry.remoteIp ?? '';
|
||||
const userAgent = entry.userAgent ?? '';
|
||||
const raw: RawEvent = {
|
||||
request_id: entry.requestId,
|
||||
ts: entry.timestamp,
|
||||
zone_id: entry.pullZoneId,
|
||||
host: entry.host,
|
||||
status: entry.statusCode,
|
||||
cache_status: entry.cacheStatus,
|
||||
bytes: entry.bytesSent,
|
||||
path: entry.path ?? '',
|
||||
user_agent: entry.userAgent,
|
||||
country: entry.countryCode,
|
||||
asn: entry.asn,
|
||||
asn_org: entry.asnOrganization,
|
||||
client: hashClient(`${ip}|${entry.asn ?? ''}|${clientPlatform(userAgent)}`),
|
||||
own: ip !== '' && ownNetworks.some((matches) => matches(ip)),
|
||||
};
|
||||
return { ...raw, ...classify(raw) };
|
||||
}
|
||||
157
stats/collect.ts
Normal file
157
stats/collect.ts
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
/**
|
||||
* Pull the CDN access logs into `download_stats.events`.
|
||||
*
|
||||
* bun stats/collect.ts # fetch the last 70 hours, insert what is new
|
||||
* bun stats/collect.ts --reclassify # recompute derived columns after a rule change
|
||||
*
|
||||
* Bunny keeps logs for three days, so every run re-reads almost all of that
|
||||
* window. Events are keyed by Bunny's request id, which makes re-reading free
|
||||
* of double counts, and a run that fails is simply covered by the next one.
|
||||
*
|
||||
* Environment: BUNNYNET_API_KEY, PGPASS, STATS_CLIENT_SALT, and optionally
|
||||
* STATS_PULL_ZONE (default: cran-rpkgs) and STATS_OWN_NETWORKS (comma-separated
|
||||
* CIDRs of our own CI runners, kept out of the public numbers).
|
||||
*/
|
||||
import { classify, parseNetworks, toEvent, type Event, type LogEntry } from './classify';
|
||||
import { connect, migrate } from './db';
|
||||
|
||||
const PULL_ZONE = Number(process.env.STATS_PULL_ZONE ?? 3857050);
|
||||
const WINDOW_HOURS = 70;
|
||||
const PAGE_SIZE = 10000;
|
||||
|
||||
const COLUMNS = [
|
||||
'request_id',
|
||||
'ts',
|
||||
'zone_id',
|
||||
'host',
|
||||
'status',
|
||||
'cache_status',
|
||||
'bytes',
|
||||
'path',
|
||||
'user_agent',
|
||||
'country',
|
||||
'asn',
|
||||
'asn_org',
|
||||
'client',
|
||||
'own',
|
||||
'ruleset',
|
||||
'kind',
|
||||
'outcome',
|
||||
'redirect_reason',
|
||||
'slot',
|
||||
'arch',
|
||||
'os',
|
||||
'r_minor',
|
||||
'r_version',
|
||||
'package',
|
||||
'version',
|
||||
'tool',
|
||||
'network',
|
||||
] as const;
|
||||
|
||||
function requireEnv(name: string): string {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(`${name} is not set`);
|
||||
return value;
|
||||
}
|
||||
|
||||
async function fetchPage(
|
||||
apiKey: string,
|
||||
from: Date,
|
||||
to: Date,
|
||||
offset: number,
|
||||
): Promise<{ data: LogEntry[]; hasMore: boolean }> {
|
||||
const url = new URL(`https://logging.bunnycdn.com/v2/pullzones/${PULL_ZONE}/logs`);
|
||||
url.searchParams.set('from', from.toISOString());
|
||||
url.searchParams.set('to', to.toISOString());
|
||||
url.searchParams.set('order', 'asc');
|
||||
url.searchParams.set('limit', String(PAGE_SIZE));
|
||||
url.searchParams.set('offset', String(offset));
|
||||
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
const response = await fetch(url, { headers: { AccessKey: apiKey, Accept: 'application/json' } });
|
||||
if (response.ok) {
|
||||
const body = (await response.json()) as { data: LogEntry[]; pagination: { hasMore: boolean } };
|
||||
return { data: body.data, hasMore: body.pagination.hasMore };
|
||||
}
|
||||
if ((response.status === 429 || response.status >= 500) && attempt < 6) {
|
||||
await Bun.sleep(2000 * attempt);
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Bunny logging API returned ${response.status}: ${await response.text()}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function collect(): Promise<void> {
|
||||
const apiKey = requireEnv('BUNNYNET_API_KEY');
|
||||
const salt = requireEnv('STATS_CLIENT_SALT');
|
||||
const ownNetworks = parseNetworks(process.env.STATS_OWN_NETWORKS ?? '');
|
||||
const hashClient = (key: string) => new Bun.CryptoHasher('sha256', salt).update(key).digest('hex').slice(0, 20);
|
||||
|
||||
// Fixing `to` up front keeps offset pagination stable while new requests
|
||||
// keep arriving.
|
||||
const to = new Date();
|
||||
const from = new Date(to.getTime() - WINDOW_HOURS * 3600 * 1000);
|
||||
|
||||
const events: Event[] = [];
|
||||
let fetched = 0;
|
||||
for (let offset = 0; ;) {
|
||||
const page = await fetchPage(apiKey, from, to, offset);
|
||||
fetched += page.data.length;
|
||||
for (const entry of page.data) {
|
||||
const event = toEvent(entry, hashClient, ownNetworks);
|
||||
if (event) events.push(event);
|
||||
}
|
||||
if (!page.hasMore || page.data.length === 0) break;
|
||||
offset += page.data.length;
|
||||
}
|
||||
|
||||
const sql = connect();
|
||||
await migrate(sql);
|
||||
let inserted = 0;
|
||||
for (let i = 0; i < events.length; i += 1000) {
|
||||
const batch = events.slice(i, i + 1000);
|
||||
const rows = await sql`
|
||||
insert into download_stats.events ${sql(batch, ...COLUMNS)}
|
||||
on conflict (request_id) do nothing
|
||||
returning 1`;
|
||||
inserted += rows.length;
|
||||
}
|
||||
await sql`
|
||||
insert into download_stats.runs (zone_id, window_from, window_to, fetched, inserted)
|
||||
values (${PULL_ZONE}, ${from}, ${to}, ${fetched}, ${inserted})`;
|
||||
await sql.close();
|
||||
|
||||
console.log(
|
||||
`fetched ${fetched} log entries from ${from.toISOString()} to ${to.toISOString()}, inserted ${inserted} new`,
|
||||
);
|
||||
}
|
||||
|
||||
async function reclassify(): Promise<void> {
|
||||
const sql = connect();
|
||||
await migrate(sql);
|
||||
const rows: Event[] =
|
||||
await sql`select request_id, path, status, user_agent, asn, asn_org, own from download_stats.events`;
|
||||
let updated = 0;
|
||||
for (let i = 0; i < rows.length; i += 1000) {
|
||||
const batch = rows.slice(i, i + 1000).map((row) => ({ request_id: row.request_id, ...classify(row) }));
|
||||
await sql.begin(async (tx) => {
|
||||
for (const row of batch) {
|
||||
await tx`
|
||||
update download_stats.events set
|
||||
ruleset = ${row.ruleset}, kind = ${row.kind}, outcome = ${row.outcome},
|
||||
redirect_reason = ${row.redirect_reason}, slot = ${row.slot}, arch = ${row.arch}, os = ${row.os},
|
||||
r_minor = ${row.r_minor}, r_version = ${row.r_version}, package = ${row.package},
|
||||
version = ${row.version}, tool = ${row.tool}, network = ${row.network}
|
||||
where request_id = ${row.request_id}`;
|
||||
}
|
||||
});
|
||||
updated += batch.length;
|
||||
}
|
||||
await sql.close();
|
||||
console.log(`reclassified ${updated} events`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
await (process.argv.includes('--reclassify') ? reclassify() : collect());
|
||||
}
|
||||
75
stats/db.ts
Normal file
75
stats/db.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import { SQL } from 'bun';
|
||||
|
||||
/**
|
||||
* Download events live in their own `download_stats` schema in the build
|
||||
* metadata database, next to (but never joined with) `single_builds`.
|
||||
*
|
||||
* No IP address is stored, not even the /24 prefix Bunny logs: only a salted
|
||||
* HMAC of it (`client`) and whether it matched one of our own networks.
|
||||
*/
|
||||
const SCHEMA = `
|
||||
create schema if not exists download_stats;
|
||||
|
||||
create table if not exists download_stats.events (
|
||||
request_id text primary key,
|
||||
ts timestamptz not null,
|
||||
zone_id integer not null,
|
||||
host text,
|
||||
status smallint not null,
|
||||
cache_status text,
|
||||
bytes bigint not null,
|
||||
path text not null,
|
||||
user_agent text,
|
||||
country text,
|
||||
asn integer,
|
||||
asn_org text,
|
||||
client text not null,
|
||||
own boolean not null,
|
||||
ruleset smallint not null,
|
||||
kind text not null,
|
||||
outcome text not null,
|
||||
redirect_reason text,
|
||||
slot text,
|
||||
arch text,
|
||||
os text,
|
||||
r_minor text,
|
||||
r_version text,
|
||||
package text,
|
||||
version text,
|
||||
tool text not null,
|
||||
network text not null
|
||||
);
|
||||
|
||||
create index if not exists events_ts_idx on download_stats.events (ts);
|
||||
create index if not exists events_package_idx on download_stats.events (package) where kind = 'download';
|
||||
|
||||
-- One row per collection run. The union of successful windows is what the
|
||||
-- dashboard reports as covered; anything outside it is shown as a gap.
|
||||
create table if not exists download_stats.runs (
|
||||
id bigserial primary key,
|
||||
zone_id integer not null,
|
||||
started_at timestamptz not null default now(),
|
||||
window_from timestamptz not null,
|
||||
window_to timestamptz not null,
|
||||
fetched integer not null,
|
||||
inserted integer not null
|
||||
);
|
||||
`;
|
||||
|
||||
export function connect(): SQL {
|
||||
const password = process.env.PGPASS;
|
||||
if (!password) throw new Error('PGPASS is not set');
|
||||
return new SQL({
|
||||
hostname: process.env.PGHOST ?? 'r-binaries.devxy.io',
|
||||
port: Number(process.env.PGPORT ?? 15432),
|
||||
database: process.env.PGDATABASE ?? 'build_metadata',
|
||||
username: process.env.PGUSER ?? 'rpkgs',
|
||||
password,
|
||||
tls: true,
|
||||
max: 2,
|
||||
});
|
||||
}
|
||||
|
||||
export async function migrate(sql: SQL): Promise<void> {
|
||||
await sql.unsafe(SCHEMA);
|
||||
}
|
||||
339
stats/publish.ts
Normal file
339
stats/publish.ts
Normal file
|
|
@ -0,0 +1,339 @@
|
|||
/**
|
||||
* Aggregate `download_stats.events` into the public `stats/downloads.json`.
|
||||
*
|
||||
* bun stats/publish.ts --out downloads.json # write locally only
|
||||
* bun stats/publish.ts # upload to the bucket and purge the CDN copy
|
||||
*
|
||||
* The output holds counts only. Distinct clients are counted from the stored
|
||||
* HMAC, which never leaves the database, and countries seen from fewer than
|
||||
* MIN_CLIENTS clients are folded into "other" so no single client can be
|
||||
* picked out by where it is.
|
||||
*
|
||||
* Every number excludes our own CI runners. Cloud and CI networks are kept but
|
||||
* reported separately, since one pipeline can outweigh every person combined.
|
||||
*/
|
||||
import { S3Client, type SQL } from 'bun';
|
||||
import { KNOWN_MINORS, RULESET } from './classify';
|
||||
import { connect, migrate } from './db';
|
||||
|
||||
const MIN_CLIENTS = 3;
|
||||
const PUBLIC_URL = 'https://cran.rpkgs.com/stats/downloads.json';
|
||||
const OBJECT_KEY = 'stats/downloads.json';
|
||||
|
||||
/** Split of a count by network class. */
|
||||
export interface Split {
|
||||
total: number;
|
||||
cloud: number;
|
||||
other: number;
|
||||
}
|
||||
|
||||
export interface DayStats {
|
||||
date: string;
|
||||
/** Hours of this UTC day for which logs were collected (24 = complete). */
|
||||
coveredHours: number;
|
||||
/** Tarball requests served, before any de-duplication. */
|
||||
requests: number;
|
||||
/** Distinct (client, package, version) served on the day. */
|
||||
downloads: Split;
|
||||
/** Distinct clients that fetched an index or a package. */
|
||||
clients: Split;
|
||||
/** Distinct clients sent to CRAN because we do not build for their R minor. */
|
||||
unsupportedMinorClients: number;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface WindowStats {
|
||||
days: number;
|
||||
downloads: Split;
|
||||
/** Distinct clients across the whole window. */
|
||||
clients: Split;
|
||||
/** Sum of daily distinct clients: roughly, install sessions. */
|
||||
clientDays: Split;
|
||||
}
|
||||
|
||||
export interface Breakdown {
|
||||
key: string;
|
||||
clients: number;
|
||||
downloads: number;
|
||||
}
|
||||
|
||||
export interface PackageStats {
|
||||
name: string;
|
||||
downloads30d: number;
|
||||
clients30d: number;
|
||||
downloadsAll: number;
|
||||
}
|
||||
|
||||
export interface PublicStats {
|
||||
schema: 1;
|
||||
ruleset: number;
|
||||
generatedAt: string;
|
||||
firstDay: string | null;
|
||||
lastDay: string | null;
|
||||
supportedMinors: string[];
|
||||
daily: DayStats[];
|
||||
windows: { last7: WindowStats; last30: WindowStats; all: WindowStats };
|
||||
/** Distribution of distinct packages fetched per client per day (30 days). */
|
||||
packagesPerClientDay: { median: number; p90: number };
|
||||
/** Last 30 days. */
|
||||
breakdowns: {
|
||||
network: Breakdown[];
|
||||
os: Breakdown[];
|
||||
arch: Breakdown[];
|
||||
rMinor: Breakdown[];
|
||||
tool: Breakdown[];
|
||||
country: Breakdown[];
|
||||
};
|
||||
/** Last 30 days: clients we sent elsewhere, and why. */
|
||||
redirects: { reason: string; clients: number; requests: number }[];
|
||||
unsupportedMinors: Breakdown[];
|
||||
/** Last 30 days: requests that were not repository use at all. */
|
||||
excluded: { ownRunners: number; probes: number; noise: number; errors: number };
|
||||
packages: PackageStats[];
|
||||
}
|
||||
|
||||
const split = (row: Record<string, unknown>, prefix: string): Split => ({
|
||||
total: Number(row[`${prefix}_total`] ?? 0),
|
||||
cloud: Number(row[`${prefix}_cloud`] ?? 0),
|
||||
other: Number(row[`${prefix}_other`] ?? 0),
|
||||
});
|
||||
|
||||
async function coverage(sql: SQL): Promise<Map<string, number>> {
|
||||
const runs: { window_from: Date; window_to: Date }[] =
|
||||
await sql`select window_from, window_to from download_stats.runs order by window_from`;
|
||||
const merged: [number, number][] = [];
|
||||
for (const run of runs) {
|
||||
const from = run.window_from.getTime();
|
||||
const to = run.window_to.getTime();
|
||||
const last = merged.at(-1);
|
||||
if (last && from <= last[1]) last[1] = Math.max(last[1], to);
|
||||
else merged.push([from, to]);
|
||||
}
|
||||
const hours = new Map<string, number>();
|
||||
const DAY = 86400000;
|
||||
for (const [from, to] of merged) {
|
||||
for (let day = Math.floor(from / DAY) * DAY; day < to; day += DAY) {
|
||||
const overlap = Math.min(to, day + DAY) - Math.max(from, day);
|
||||
const key = new Date(day).toISOString().slice(0, 10);
|
||||
hours.set(key, (hours.get(key) ?? 0) + overlap / 3600000);
|
||||
}
|
||||
}
|
||||
return hours;
|
||||
}
|
||||
|
||||
async function windowStats(sql: SQL, days: number | null): Promise<WindowStats> {
|
||||
const since = days === null ? new Date(0) : new Date(Date.now() - days * 86400000);
|
||||
const [row] = await sql`
|
||||
with ev as (
|
||||
select client, network, kind, package, version, (ts at time zone 'UTC')::date as day
|
||||
from download_stats.events
|
||||
where network <> 'own' and outcome = 'served' and kind in ('download', 'index') and ts >= ${since}
|
||||
),
|
||||
dl as (select distinct client, network, package, version, day from ev where kind = 'download'),
|
||||
cd as (select distinct client, network, day from ev)
|
||||
select
|
||||
(select count(*) from dl) as downloads_total,
|
||||
(select count(*) from dl where network = 'cloud') as downloads_cloud,
|
||||
(select count(*) from dl where network = 'other') as downloads_other,
|
||||
(select count(distinct client) from ev) as clients_total,
|
||||
(select count(distinct client) from ev where network = 'cloud') as clients_cloud,
|
||||
(select count(distinct client) from ev where network = 'other') as clients_other,
|
||||
(select count(*) from cd) as client_days_total,
|
||||
(select count(*) from cd where network = 'cloud') as client_days_cloud,
|
||||
(select count(*) from cd where network = 'other') as client_days_other,
|
||||
(select min(day) from ev) as first_day`;
|
||||
const firstDay = row.first_day ? new Date(row.first_day) : null;
|
||||
const span = firstDay ? Math.max(1, Math.ceil((Date.now() - firstDay.getTime()) / 86400000)) : 0;
|
||||
return {
|
||||
days: days === null ? span : Math.min(days, span),
|
||||
downloads: split(row, 'downloads'),
|
||||
clients: split(row, 'clients'),
|
||||
clientDays: split(row, 'client_days'),
|
||||
};
|
||||
}
|
||||
|
||||
async function breakdown(sql: SQL, column: string, since: Date): Promise<Breakdown[]> {
|
||||
const rows: { key: string | null; clients: number; downloads: number }[] = await sql`
|
||||
with ev as (
|
||||
select ${sql(column)}::text as key, client, kind, package, version, (ts at time zone 'UTC')::date as day
|
||||
from download_stats.events
|
||||
where network <> 'own' and outcome = 'served' and kind in ('download', 'index') and ts >= ${since}
|
||||
)
|
||||
select key, count(distinct client)::int as clients,
|
||||
(select count(*) from (select distinct client, package, version, day from ev e2
|
||||
where e2.kind = 'download' and e2.key is not distinct from ev.key) d)::int as downloads
|
||||
from ev group by key order by clients desc`;
|
||||
return rows.map((row) => ({ key: row.key ?? 'unknown', clients: row.clients, downloads: row.downloads }));
|
||||
}
|
||||
|
||||
/** Fold rows below the client threshold into one "other" row. */
|
||||
function suppress(rows: Breakdown[]): Breakdown[] {
|
||||
const kept = rows.filter((row) => row.clients >= MIN_CLIENTS);
|
||||
const folded = rows.filter((row) => row.clients < MIN_CLIENTS);
|
||||
if (folded.length === 0) return kept;
|
||||
return [
|
||||
...kept,
|
||||
{
|
||||
key: 'other',
|
||||
clients: folded.reduce((sum, row) => sum + row.clients, 0),
|
||||
downloads: folded.reduce((sum, row) => sum + row.downloads, 0),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
export async function buildStats(sql: SQL): Promise<PublicStats> {
|
||||
const since30 = new Date(Date.now() - 30 * 86400000);
|
||||
const covered = await coverage(sql);
|
||||
|
||||
const dailyRows: Record<string, unknown>[] = await sql`
|
||||
with ev as (
|
||||
select *, (ts at time zone 'UTC')::date as day from download_stats.events where network <> 'own'
|
||||
),
|
||||
served as (select * from ev where outcome = 'served' and kind in ('download', 'index')),
|
||||
dl as (select distinct client, network, package, version, day from served where kind = 'download'),
|
||||
cl as (select distinct client, network, day from served)
|
||||
select d.day::text as date,
|
||||
(select count(*) from served s where s.day = d.day and s.kind = 'download')::int as requests,
|
||||
(select count(*) from dl where dl.day = d.day)::int as downloads_total,
|
||||
(select count(*) from dl where dl.day = d.day and network = 'cloud')::int as downloads_cloud,
|
||||
(select count(*) from dl where dl.day = d.day and network = 'other')::int as downloads_other,
|
||||
(select count(*) from cl where cl.day = d.day)::int as clients_total,
|
||||
(select count(*) from cl where cl.day = d.day and network = 'cloud')::int as clients_cloud,
|
||||
(select count(*) from cl where cl.day = d.day and network = 'other')::int as clients_other,
|
||||
(select count(distinct client) from ev e where e.day = d.day and e.redirect_reason = 'unsupported_r_minor')::int
|
||||
as unsupported,
|
||||
(select coalesce(sum(bytes), 0) from served s where s.day = d.day)::bigint as bytes
|
||||
from (select distinct day from ev) d order by d.day`;
|
||||
|
||||
const daily: DayStats[] = dailyRows.map((row) => ({
|
||||
date: String(row.date),
|
||||
coveredHours: Math.round(Math.min(24, covered.get(String(row.date)) ?? 0) * 10) / 10,
|
||||
requests: Number(row.requests),
|
||||
downloads: split(row, 'downloads'),
|
||||
clients: split(row, 'clients'),
|
||||
unsupportedMinorClients: Number(row.unsupported),
|
||||
bytes: Number(row.bytes),
|
||||
}));
|
||||
|
||||
const [perSession] = await sql`
|
||||
with per as (
|
||||
select client, (ts at time zone 'UTC')::date as day, count(distinct (package, version)) as n
|
||||
from download_stats.events
|
||||
where network <> 'own' and outcome = 'served' and kind = 'download' and ts >= ${since30}
|
||||
group by 1, 2
|
||||
)
|
||||
select coalesce(percentile_cont(0.5) within group (order by n), 0) as median,
|
||||
coalesce(percentile_cont(0.9) within group (order by n), 0) as p90
|
||||
from per`;
|
||||
|
||||
const redirects: { reason: string; clients: number; requests: number }[] = await sql`
|
||||
select redirect_reason as reason, count(distinct client)::int as clients, count(*)::int as requests
|
||||
from download_stats.events
|
||||
where network <> 'own' and outcome = 'redirect' and ts >= ${since30}
|
||||
and redirect_reason not in ('minor_routing', 'slot_resolution', 'https')
|
||||
group by 1 order by clients desc`;
|
||||
|
||||
const unsupportedMinors: Breakdown[] = await sql`
|
||||
select r_minor as key, count(distinct client)::int as clients, count(*)::int as downloads
|
||||
from download_stats.events
|
||||
where network <> 'own' and redirect_reason = 'unsupported_r_minor' and ts >= ${since30}
|
||||
group by 1 order by clients desc`;
|
||||
|
||||
const [excluded] = await sql`
|
||||
select
|
||||
count(*) filter (where network = 'own')::int as own,
|
||||
count(*) filter (where network <> 'own' and kind = 'probe')::int as probes,
|
||||
count(*) filter (where network <> 'own' and kind = 'noise')::int as noise,
|
||||
count(*) filter (where network <> 'own' and kind in ('download', 'index') and outcome = 'error')::int as errors
|
||||
from download_stats.events where ts >= ${since30}`;
|
||||
|
||||
const packages: PackageStats[] = (
|
||||
await sql`
|
||||
with dl as (
|
||||
select distinct client, package, version, (ts at time zone 'UTC')::date as day, ts >= ${since30} as recent
|
||||
from download_stats.events
|
||||
where network <> 'own' and outcome = 'served' and kind = 'download' and package is not null
|
||||
)
|
||||
select package as name,
|
||||
count(*) filter (where recent)::int as downloads30d,
|
||||
count(distinct client) filter (where recent)::int as clients30d,
|
||||
count(*)::int as downloadsall
|
||||
from dl group by package order by downloads30d desc, downloadsall desc, name`
|
||||
).map((row: Record<string, unknown>) => ({
|
||||
name: String(row.name),
|
||||
downloads30d: Number(row.downloads30d),
|
||||
clients30d: Number(row.clients30d),
|
||||
downloadsAll: Number(row.downloadsall),
|
||||
}));
|
||||
|
||||
return {
|
||||
schema: 1,
|
||||
ruleset: RULESET,
|
||||
generatedAt: new Date().toISOString(),
|
||||
firstDay: daily[0]?.date ?? null,
|
||||
lastDay: daily.at(-1)?.date ?? null,
|
||||
supportedMinors: [...KNOWN_MINORS],
|
||||
daily,
|
||||
windows: {
|
||||
last7: await windowStats(sql, 7),
|
||||
last30: await windowStats(sql, 30),
|
||||
all: await windowStats(sql, null),
|
||||
},
|
||||
packagesPerClientDay: { median: Number(perSession.median), p90: Number(perSession.p90) },
|
||||
breakdowns: {
|
||||
network: await breakdown(sql, 'network', since30),
|
||||
os: await breakdown(sql, 'os', since30),
|
||||
arch: await breakdown(sql, 'arch', since30),
|
||||
rMinor: await breakdown(sql, 'r_minor', since30),
|
||||
tool: await breakdown(sql, 'tool', since30),
|
||||
country: suppress(await breakdown(sql, 'country', since30)),
|
||||
},
|
||||
redirects,
|
||||
unsupportedMinors,
|
||||
excluded: {
|
||||
ownRunners: Number(excluded.own),
|
||||
probes: Number(excluded.probes),
|
||||
noise: Number(excluded.noise),
|
||||
errors: Number(excluded.errors),
|
||||
},
|
||||
packages,
|
||||
};
|
||||
}
|
||||
|
||||
async function upload(body: string): Promise<void> {
|
||||
const s3 = new S3Client({
|
||||
accessKeyId: process.env.B2_S3_ACCESS_KEY,
|
||||
secretAccessKey: process.env.B2_S3_SECRET_KEY,
|
||||
bucket: process.env.STATS_BUCKET ?? 'devxy-rpkgs-binaries',
|
||||
endpoint: 'https://s3.eu-central-003.backblazeb2.com',
|
||||
region: 'eu-central-003',
|
||||
});
|
||||
await s3.write(OBJECT_KEY, body, { type: 'application/json' });
|
||||
|
||||
// The pull zone overrides origin caching with a one-year TTL, so the edge
|
||||
// copy has to be purged for the new file to show up.
|
||||
const purge = new URL('https://api.bunny.net/purge');
|
||||
purge.searchParams.set('url', PUBLIC_URL);
|
||||
purge.searchParams.set('async', 'false');
|
||||
const response = await fetch(purge, { method: 'POST', headers: { AccessKey: process.env.BUNNYNET_API_KEY ?? '' } });
|
||||
if (!response.ok) throw new Error(`purging ${PUBLIC_URL} failed with ${response.status}: ${await response.text()}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const sql = connect();
|
||||
await migrate(sql);
|
||||
const stats = await buildStats(sql);
|
||||
await sql.close();
|
||||
const body = JSON.stringify(stats);
|
||||
|
||||
const outIndex = process.argv.indexOf('--out');
|
||||
if (outIndex !== -1) {
|
||||
await Bun.write(process.argv[outIndex + 1], JSON.stringify(stats, null, 2));
|
||||
console.log(`wrote ${process.argv[outIndex + 1]} (${body.length} bytes)`);
|
||||
} else {
|
||||
await upload(body);
|
||||
console.log(
|
||||
`published ${PUBLIC_URL} (${body.length} bytes, ${stats.daily.length} days, ${stats.packages.length} packages)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue