build-cran-binaries/stats/classify.ts

/**
 * Turns one Bunny access-log entry into a classified download event.
 *
 * Everything here is pure so the rules can be unit-tested and re-run over the
 * stored events (`collect.ts --reclassify`) whenever they change. Bump RULESET
 * on any change that alters a derived column, so the public JSON can say which
 * rules produced it.
 *
 * The rules mirror `edge/rpkgs-router.ts`: a redirect is classified by what the
 * router would have done with that path and User-Agent, because the Location
 * header is not in the logs.
 */

export const RULESET = 1;

/** R minors with a per-minor slot; keep in sync with KNOWN_MINORS in cdn.tf. */
export const KNOWN_MINORS = new Set(['4.4', '4.5', '4.6']);

export type Kind =
  /** A package tarball. */
  | 'download'
  /** `PACKAGES*`: one per repository per `install.packages()` call. */
  | 'index'
  /** Other repository metadata, e.g. `Meta/archive.rds`. */
  | 'metadata'
  /** renv and pak asking whether this is a Posit Package Manager (`__api__/…`). */
  | 'probe'
  /** The repository root or `/src/contrib` without a file. */
  | 'root'
  /** Our own `stats/` files, so the dashboard does not count itself. */
  | 'stats'
  /** Anything else: vulnerability scanners, crawlers, typos. */
  | 'noise';

export type Outcome = 'served' | 'redirect' | 'error';

export type RedirectReason =
  /** Internal hop: index sent to its per-R-minor slot. Not a user-visible event. */
  | 'minor_routing'
  /** Internal hop: bare `cran.rpkgs.com` URL resolved to a slot. */
  | 'slot_resolution'
  /** Client runs an R minor we do not build for; sent to CRAN for source. */
  | 'unsupported_r_minor'
  /** Bare URL from a distro we could not identify; sent to CRAN. */
  | 'unknown_distro'
  /** Archived version we have no binary for; sent to the CRAN source tarball. */
  | 'archive_fallback'
  /** macOS client; sent to CRAN's macOS binaries. */
  | 'macos'
  /** `http://` to `https://`. */
  | 'https'
  | 'other';

export type Tool = 'R' | 'renv' | 'pak' | 'rig' | 'curl' | 'wget' | 'python' | 'browser' | 'none' | 'other';

export type Network = 'cloud' | 'other' | 'own';

export interface LogEntry {
  timestamp: string;
  pullZoneId: number;
  requestId: string | null;
  cacheStatus: string | null;
  statusCode: number;
  bytesSent: number;
  remoteIp: string | null;
  countryCode: string | null;
  host: string | null;
  path: string | null;
  userAgent: string | null;
  asn: number | null;
  asnOrganization: string | null;
}

export interface Derived {
  ruleset: number;
  kind: Kind;
  outcome: Outcome;
  redirect_reason: RedirectReason | null;
  slot: string | null;
  arch: string | null;
  os: string | null;
  r_minor: string | null;
  r_version: string | null;
  package: string | null;
  version: string | null;
  tool: Tool;
  network: Network;
}

/** The raw columns kept per event; everything Derived is recomputed from these. */
export interface RawEvent {
  request_id: string;
  ts: string;
  zone_id: number;
  host: string | null;
  status: number;
  cache_status: string | null;
  bytes: number;
  path: string;
  user_agent: string | null;
  country: string | null;
  asn: number | null;
  asn_org: string | null;
  /** HMAC of the (already anonymized) IP prefix, network and client platform. */
  client: string;
  /** Matched one of our own networks at ingest. The IP itself is not kept. */
  own: boolean;
}

export type Event = RawEvent & Derived;

const SLOT_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib(?:\/(.*))?$/;
const BARE_REGEX = /^\/src\/contrib(?:\/(.*))?$/;
const MACOS_REGEX = /^\/bin\/macosx\/[^/]+\/contrib\/(\d+\.\d+)\/(.+)$/;
const MINOR_DIR_REGEX = /^(\d+\.\d+)\/(.*)$/;
const INDEX_REGEX = /^PACKAGES(\.gz|\.rds|\.json)?$/;
const TARBALL_REGEX = /(?:^|\/)([A-Za-z][A-Za-z0-9.]*)_([0-9][^/_]*)\.(?:tar\.gz|tgz)$/;
const R_VERSION_REGEXES = [/\bR\/(\d+\.\d+\.\d+)/, /\bR \((\d+\.\d+\.\d+)/];
const ARCH_REGEX = /\b(x86_64|aarch64|arm64)\b/;
/** The distro forms `parseSlot()` in the router recognises. */
const DISTRO_REGEX = /(almalinux|rocky)[^\d]*\d+|Ubuntu \d+\.\d+|Alpine Linux|alpine-\d/i;

/**
 * Networks whose clients are almost always machines: public clouds, CI
 * providers and hosting companies. A request from one of these is reported as
 * "cloud / CI" rather than dropped, since a CI pipeline is a real user too,
 * just not a person at a laptop.
 */
const CLOUD_ASNS = new Map<number, string>([
  [8075, 'Microsoft (incl. GitHub Actions runners)'],
  [8068, 'Microsoft'],
  [8069, 'Microsoft'],
  [36459, 'GitHub'],
  [16509, 'Amazon'],
  [14618, 'Amazon'],
  [15169, 'Google'],
  [396982, 'Google Cloud'],
  [24940, 'Hetzner'],
  [213230, 'Hetzner Cloud'],
  [212317, 'Hetzner'],
  [16276, 'OVH'],
  [14061, 'DigitalOcean'],
  [51167, 'Contabo'],
  [40021, 'Contabo US'],
  [63949, 'Akamai / Linode'],
  [31898, 'Oracle'],
  [45102, 'Alibaba'],
  [37963, 'Alibaba'],
  [132203, 'Tencent'],
  [45090, 'Tencent'],
  [13335, 'Cloudflare'],
  [62240, 'Clouvider'],
  [12876, 'Scaleway'],
  [20473, 'Vultr'],
  [197540, 'netcup'],
  [9009, 'M247'],
  [60068, 'Datacamp / CDN77'],
  [36352, 'ColoCrossing'],
  [47583, 'Hostinger'],
  [207990, 'HostRoyale'],
]);

const CLOUD_ORG_REGEX = /\b(hosting|cloud|server(s)?|data ?cent(er|re)|colo(cation)?|vps|dedicated)\b/i;

export function isCloudNetwork(asn: number | null, org: string | null): boolean {
  if (asn !== null && CLOUD_ASNS.has(asn)) return true;
  return org !== null && CLOUD_ORG_REGEX.test(org);
}

export function normalizePath(path: string): string {
  const withoutQuery = path.split('?')[0] ?? '';
  return withoutQuery.replace(/\/{2,}/g, '/');
}

export function rVersion(userAgent: string): string | null {
  for (const regex of R_VERSION_REGEXES) {
    const match = userAgent.match(regex);
    if (match) return match[1];
  }
  return null;
}

export function rMinor(version: string | null): string | null {
  if (!version) return null;
  const [major, minor] = version.split('.');
  return `${major}.${minor}`;
}

export function tool(userAgent: string): Tool {
  const ua = userAgent.trim();
  if (ua === '' || ua === '-') return 'none';
  if (/^renv\b/i.test(ua)) return 'renv';
  if (/\bpak\b/i.test(ua)) return 'pak';
  if (/\brig\b/i.test(ua)) return 'rig';
  if (/^R[ /(]/.test(ua)) return 'R';
  if (/^curl\//i.test(ua)) return 'curl';
  if (/^wget\//i.test(ua)) return 'wget';
  if (/python|aiohttp|httpx/i.test(ua)) return 'python';
  if (/^Mozilla\//.test(ua)) return 'browser';
  return 'other';
}

/**
 * The part of the User-Agent that identifies the machine rather than the tool.
 *
 * `renv (1.2.3); R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` and
 * `R/4.5.3 R (4.5.3 x86_64-pc-linux-gnu …)` are the same R session, one going
 * through renv and one not. Keying distinct clients on the full User-Agent
 * would count that machine twice.
 */
export function clientPlatform(userAgent: string): string {
  const version = rVersion(userAgent);
  if (!version) return userAgent;
  const rPart = userAgent.slice(userAgent.search(/\bR[/ ]\(?\d/));
  return rPart.replace(/\bR\/\d+\.\d+\.\d+\s*/, '').trim() || version;
}

function slotParts(slot: string | null): { arch: string | null; os: string | null } {
  if (!slot) return { arch: null, os: null };
  const [arch, os] = slot.split('/');
  return { arch: arch ?? null, os: os ?? null };
}

function archFromUserAgent(userAgent: string): string | null {
  const match = userAgent.match(ARCH_REGEX);
  if (!match) return null;
  return match[1] === 'x86_64' ? 'amd64' : 'arm64';
}

function fileKind(rest: string): Kind {
  if (rest === '') return 'root';
  if (INDEX_REGEX.test(rest)) return 'index';
  if (TARBALL_REGEX.test(rest)) return 'download';
  if (/^Meta\//.test(rest) || /^PACKAGES/.test(rest)) return 'metadata';
  return 'noise';
}

function outcome(status: number): Outcome {
  if (status === 301 || status === 302 || status === 303 || status === 307 || status === 308) return 'redirect';
  if (status >= 200 && status < 400) return 'served';
  return 'error';
}

export function classify(
  raw: Pick<RawEvent, 'path' | 'status' | 'user_agent' | 'asn' | 'asn_org' | 'own'>,
  scheme = 'https',
): Derived {
  const path = normalizePath(raw.path);
  const userAgent = raw.user_agent ?? '';
  const version = rVersion(userAgent);
  const minor = rMinor(version);
  const result: Derived = {
    ruleset: RULESET,
    kind: 'noise',
    outcome: outcome(raw.status),
    redirect_reason: null,
    slot: null,
    arch: null,
    os: null,
    r_minor: minor,
    r_version: version,
    package: null,
    version: null,
    tool: tool(userAgent),
    network: raw.own ? 'own' : isCloudNetwork(raw.asn, raw.asn_org) ? 'cloud' : 'other',
  };

  let rest: string | null = null;
  let area: 'slot' | 'bare' | 'macos' | null = null;

  const slot = path.match(SLOT_REGEX);
  const bare = path.match(BARE_REGEX);
  const macos = path.match(MACOS_REGEX);

  if (path.includes('/__api__/')) {
    result.kind = 'probe';
  } else if (path.startsWith('/stats/')) {
    result.kind = 'stats';
  } else if (path === '/') {
    result.kind = 'root';
    area = 'bare';
  } else if (slot) {
    area = 'slot';
    result.slot = `${slot[1]}/${slot[2]}`;
    rest = slot[3] ?? '';
  } else if (bare) {
    area = 'bare';
    rest = bare[1] ?? '';
  } else if (macos) {
    area = 'macos';
    result.os = 'macos';
    rest = macos[2];
  }

  if (rest !== null) {
    const minorDir = rest.match(MINOR_DIR_REGEX);
    const file = minorDir ? minorDir[2] : rest;
    result.kind = fileKind(file);
    const tarball = file.match(TARBALL_REGEX);
    if (tarball) {
      result.package = tarball[1];
      result.version = tarball[2];
    }
  }

  const parts = slotParts(result.slot);
  result.arch = parts.arch ?? archFromUserAgent(userAgent);
  result.os = parts.os ?? result.os;

  if (result.outcome === 'redirect') {
    result.redirect_reason = redirectReason(raw.status, area, rest, minor, scheme, userAgent);
  }

  return result;
}

function redirectReason(
  status: number,
  area: 'slot' | 'bare' | 'macos' | null,
  rest: string | null,
  minor: string | null,
  scheme: string,
  userAgent: string,
): RedirectReason {
  if (status === 301 || scheme === 'http') return 'https';
  if (area === 'macos' || (area === 'bare' && /darwin/.test(userAgent))) return 'macos';
  if (minor !== null && !KNOWN_MINORS.has(minor) && (area === 'slot' || area === 'bare')) {
    return 'unsupported_r_minor';
  }
  if (area === 'slot') {
    if (rest !== null && rest.startsWith('Archive/')) return 'archive_fallback';
    if (rest !== null && INDEX_REGEX.test(rest)) return 'minor_routing';
    return 'other';
  }
  if (area === 'bare') {
    // The router resolves a bare URL only when it can identify the distro from
    // the User-Agent; otherwise it sends the client to CRAN. A stock R
    // User-Agent (`R (4.5.3 x86_64-pc-linux-gnu …)`) names no distro.
    return DISTRO_REGEX.test(userAgent) && ARCH_REGEX.test(userAgent) ? 'slot_resolution' : 'unknown_distro';
  }
  return 'other';
}

/** Parse `"a.b.c.0/24,2001:db8::/64"` into prefix matchers for our own networks. */
export function parseNetworks(spec: string): ((ip: string) => boolean)[] {
  return spec
    .split(',')
    .map((cidr) => cidr.trim())
    .filter((cidr) => cidr.length > 0)
    .map((cidr) => {
      const [base, bitsText] = cidr.split('/');
      const v6 = base.includes(':');
      const bits = Number(bitsText ?? (v6 ? 128 : 32));
      const baseBits = ipBits(base);
      return (ip: string) => {
        if (ip.includes(':') !== v6) return false;
        const candidate = ipBits(ip);
        return baseBits !== null && candidate !== null && candidate.slice(0, bits) === baseBits.slice(0, bits);
      };
    });
}

function ipBits(ip: string): string | null {
  if (!ip.includes(':')) {
    const octets = ip.split('.').map(Number);
    if (octets.length !== 4 || octets.some((o) => !Number.isInteger(o) || o < 0 || o > 255)) return null;
    return octets.map((o) => o.toString(2).padStart(8, '0')).join('');
  }
  const [head, tail] = ip.split('::');
  const headGroups = head ? head.split(':') : [];
  const tailGroups = tail ? tail.split(':') : [];
  const missing = 8 - headGroups.length - tailGroups.length;
  if (ip.includes('::') ? missing < 0 : missing !== 0) return null;
  const groups = [...headGroups, ...Array(Math.max(missing, 0)).fill('0'), ...tailGroups];
  return groups
    .map((g) =>
      parseInt(g || '0', 16)
        .toString(2)
        .padStart(16, '0'),
    )
    .join('');
}

/** Build the stored event. Returns null for entries without a request id. */
export function toEvent(
  entry: LogEntry,
  hashClient: (key: string) => string,
  ownNetworks: ((ip: string) => boolean)[],
): Event | null {
  if (!entry.requestId) return null;
  const ip = entry.remoteIp ?? '';
  const userAgent = entry.userAgent ?? '';
  const raw: RawEvent = {
    request_id: entry.requestId,
    ts: entry.timestamp,
    zone_id: entry.pullZoneId,
    host: entry.host,
    status: entry.statusCode,
    cache_status: entry.cacheStatus,
    bytes: entry.bytesSent,
    path: entry.path ?? '',
    user_agent: entry.userAgent,
    country: entry.countryCode,
    asn: entry.asn,
    asn_org: entry.asnOrganization,
    client: hashClient(`${ip}|${entry.asn ?? ''}|${clientPlatform(userAgent)}`),
    own: ip !== '' && ownNetworks.some((matches) => matches(ip)),
  };
  return { ...raw, ...classify(raw) };
}