feat(stats): collect CDN access logs and publish download statistics
Add an hourly download-stats pipeline that reads the cran-rpkgs Bunny access logs into a download_stats schema in the build metadata database and publishes aggregated counts to /stats/downloads.json for the rpkgs.com /stats page. - Classify each request (download, index, redirect reason, probe, noise) mirroring the edge router, and split cloud/CI networks from the rest. - De-duplicate downloads per client, package version and day, and key clients on a salted HMAC so no IP is stored or published. - Exclude our own CI runners and record run windows to show gaps. - Declare the zone's logging settings and allow CORS for JSON.
This commit is contained in:
parent
987958fc6e
commit
2614d40bd9
1 changed files with 1370 additions and 46 deletions
75
stats/db.ts
Normal file
75
stats/db.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import { SQL } from 'bun';
|
||||
|
||||
/**
|
||||
* Download events live in their own `download_stats` schema in the build
|
||||
* metadata database, next to (but never joined with) `single_builds`.
|
||||
*
|
||||
* No IP address is stored, not even the /24 prefix Bunny logs: only a salted
|
||||
* HMAC of it (`client`) and whether it matched one of our own networks.
|
||||
*/
|
||||
const SCHEMA = `
|
||||
create schema if not exists download_stats;
|
||||
|
||||
create table if not exists download_stats.events (
|
||||
request_id text primary key,
|
||||
ts timestamptz not null,
|
||||
zone_id integer not null,
|
||||
host text,
|
||||
status smallint not null,
|
||||
cache_status text,
|
||||
bytes bigint not null,
|
||||
path text not null,
|
||||
user_agent text,
|
||||
country text,
|
||||
asn integer,
|
||||
asn_org text,
|
||||
client text not null,
|
||||
own boolean not null,
|
||||
ruleset smallint not null,
|
||||
kind text not null,
|
||||
outcome text not null,
|
||||
redirect_reason text,
|
||||
slot text,
|
||||
arch text,
|
||||
os text,
|
||||
r_minor text,
|
||||
r_version text,
|
||||
package text,
|
||||
version text,
|
||||
tool text not null,
|
||||
network text not null
|
||||
);
|
||||
|
||||
create index if not exists events_ts_idx on download_stats.events (ts);
|
||||
create index if not exists events_package_idx on download_stats.events (package) where kind = 'download';
|
||||
|
||||
-- One row per collection run. The union of successful windows is what the
|
||||
-- dashboard reports as covered; anything outside it is shown as a gap.
|
||||
create table if not exists download_stats.runs (
|
||||
id bigserial primary key,
|
||||
zone_id integer not null,
|
||||
started_at timestamptz not null default now(),
|
||||
window_from timestamptz not null,
|
||||
window_to timestamptz not null,
|
||||
fetched integer not null,
|
||||
inserted integer not null
|
||||
);
|
||||
`;
|
||||
|
||||
export function connect(): SQL {
|
||||
const password = process.env.PGPASS;
|
||||
if (!password) throw new Error('PGPASS is not set');
|
||||
return new SQL({
|
||||
hostname: process.env.PGHOST ?? 'r-binaries.devxy.io',
|
||||
port: Number(process.env.PGPORT ?? 15432),
|
||||
database: process.env.PGDATABASE ?? 'build_metadata',
|
||||
username: process.env.PGUSER ?? 'rpkgs',
|
||||
password,
|
||||
tls: true,
|
||||
max: 2,
|
||||
});
|
||||
}
|
||||
|
||||
export async function migrate(sql: SQL): Promise<void> {
|
||||
await sql.unsafe(SCHEMA);
|
||||
}
|
||||
Loading…
Reference in a new issue