pat-s
658254afc5
Add an hourly download-stats pipeline that reads the cran-rpkgs Bunny access logs into a download_stats schema in the build metadata database and publishes aggregated counts to /stats/downloads.json for the rpkgs.com /stats page. - Classify each request (download, index, redirect reason, probe, noise) mirroring the edge router, and split cloud/CI networks from the rest. - De-duplicate downloads per client, package version and day, and key clients on a salted HMAC so no IP is stored or published. - Exclude our own CI runners and record run windows to show gaps. - Declare the zone's logging settings and allow CORS for JSON.
79 lines
2.5 KiB
YAML
79 lines
2.5 KiB
YAML
# Collect CDN access logs and publish the public download statistics.
|
|
#
|
|
# Each run reads the last 70 hours of cran-rpkgs access logs from Bunny into
|
|
# `download_stats.events` (build metadata Postgres), then rewrites
|
|
# https://cran.rpkgs.com/stats/downloads.json, which the rpkgs.com /stats page
|
|
# reads. Bunny keeps logs for only three days, so a run every hour leaves room
|
|
# for two days of failed runs before anything is lost. Re-reading the window is
|
|
# safe: events are keyed by Bunny's request id.
|
|
#
|
|
# The classification rules and what is excluded are documented in
|
|
# `stats/README.md`.
|
|
#
|
|
# Register the `download-stats` cron (hourly, `0 * * * *`) in the crow UI, or
|
|
# run manually:
|
|
# crow pipeline create --branch main --var download_stats=true devxy/build-cran-binaries
|
|
#
|
|
# Secrets:
|
|
# STATS_CLIENT_SALT HMAC key for the client hash. Rotating it resets
|
|
# distinct-client counts, so keep it stable.
|
|
# STATS_OWN_NETWORKS Comma-separated CIDRs of our own CI runners.
|
|
# Both live in OpenBao at development/agent-dev/rpkgs-download-stats.
|
|
variables:
|
|
download_stats:
|
|
description: 'Collect and publish download statistics. Also gates this pipeline.'
|
|
options:
|
|
- 'true'
|
|
- 'false'
|
|
default: 'false'
|
|
|
|
when:
|
|
- event: manual
|
|
evaluate: 'download_stats == "true"'
|
|
- event: cron
|
|
cron: download-stats
|
|
|
|
skip_clone: true
|
|
|
|
labels:
|
|
group: rpkgs-amd64
|
|
|
|
steps:
|
|
- name: 'Collect and publish download stats'
|
|
image: oven/bun:1.4.4-alpine
|
|
pull: true
|
|
environment:
|
|
BUNNYNET_API_KEY:
|
|
from_secret: BUNNYNET_API_KEY
|
|
PGPASS:
|
|
from_secret: PGPASS
|
|
B2_S3_ACCESS_KEY:
|
|
from_secret: B2_S3_ACCESS_KEY
|
|
B2_S3_SECRET_KEY:
|
|
from_secret: B2_S3_SECRET_KEY
|
|
STATS_CLIENT_SALT:
|
|
from_secret: STATS_CLIENT_SALT
|
|
STATS_OWN_NETWORKS:
|
|
from_secret: STATS_OWN_NETWORKS
|
|
REPO_RO_TOKEN:
|
|
from_secret: REPO_RO_TOKEN
|
|
commands:
|
|
- apk add --no-cache -q git
|
|
- git clone -q --depth 1 --branch "$CI_COMMIT_BRANCH" https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
|
- bun test stats
|
|
- bun stats/collect.ts
|
|
- bun stats/publish.ts
|
|
backend_options:
|
|
kubernetes:
|
|
resources:
|
|
requests:
|
|
memory: 256Mi
|
|
cpu: 250m
|
|
limits:
|
|
memory: 1Gi
|
|
cpu: 1000m
|
|
tolerations:
|
|
- key: 'CI'
|
|
operator: 'Equal'
|
|
value: 'true'
|
|
effect: 'NoSchedule'
|