pat-s
bfe56ffd22Retiring a slot means three separate things in three separate places, and nothing tied them together. Alpine 3.22 is what that costs: it was removed from `local.rpkgs_slots` in `cdn.tf` at some point, so the edge stopped routing it, but it kept a matrix row in twelve pipelines, six cron jobs on the Crow server and ~50000 objects in the bucket. Nightly builds kept producing binaries nobody could reach, and because an unrouted slot serves the flat index to every client, R 4.4 and R 4.6 users got `LOAD_FAILED` rather than nothing. This change will: - Add `local/check-slots.R`, which fails when any `.crow/*.yaml` still targets a slot missing from `local.rpkgs_slots`. Run it with `just check-slots`. - Add `local/retire-slot.R`, which strips a slot from the pipeline matrices and deletes its objects from the bucket. Dry run by default, and it refuses any slot still listed in `cdn.tf`, so it cannot be pointed at a live slot. - Add `.crow/retire-slot.yaml`, which runs the asset half in CI where the B2 credentials exist. - Document the ordering in README, including the cron jobs, which live on the Crow server and cannot be removed from here. The check is one-directional on purpose. A pipeline targeting an unpublished slot is an error; a published slot missing from *some* pipeline is only reported, because several pipelines cover a subset deliberately. Verified against the tree as it stood before #213: the checker names exactly the twelve files that PR edited by hand, and `retire-slot.R --config-only` reproduces that commit's structural changes byte for byte across all 18 pipeline files. The only differences are two prose lines, a usage example and a dropdown `description:`, which the script leaves alone because rewriting free text automatically is guesswork.
97 lines
4.2 KiB
Makefile
97 lines
4.2 KiB
Makefile
# Local helpers for build-cran-binaries.
|
|
#
|
|
# `rebuild` (re)builds specific versions of a single package on a given OS/arch
|
|
# by dispatching to a remote buildx builder (the build runs there, not locally).
|
|
# Sensitivity is auto-detected: risky packages land in the per-minor slot
|
|
# (contrib/<x.y>/), everything else in the generic slot; the touched index is
|
|
# refreshed so the result is immediately servable.
|
|
#
|
|
# Prerequisites:
|
|
# - buildx builders named `artemis` (amd64) and `gaia` (arm64), created with the
|
|
# docker-container driver (runs BuildKit on the remote host's docker daemon over
|
|
# SSH). The default `remote` driver does NOT work with an ssh:// docker host.
|
|
# Pass --config docker/buildkitd.toml so BuildKit's GC keeps a usable cache
|
|
# (the default caps the cache-mount tier at 512 MB, forcing re-downloads).
|
|
# docker buildx create --name artemis --driver docker-container --config docker/buildkitd.toml ssh://<user@host-amd64>
|
|
# docker buildx create --name gaia --driver docker-container --config docker/buildkitd.toml ssh://<user@host-arm64>
|
|
# - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional)
|
|
#
|
|
# Overridable (env or `just VAR=… rebuild …`):
|
|
# R_VERSION (default 4.5.3) — selects the R minor → the per-minor slot
|
|
# AMD64_BUILDER / ARM64_BUILDER — buildx builder names
|
|
#
|
|
# Examples:
|
|
# just rebuild alpine 3.23 amd64 rlang 1.1.4 1.1.3
|
|
# just rebuild redhat 10 arm64 data.table 1.15.4
|
|
# R_VERSION=4.4.3 just rebuild ubuntu noble amd64 Rcpp 1.0.12
|
|
|
|
r_version := env_var_or_default("R_VERSION", "4.5.3")
|
|
amd64_builder := env_var_or_default("AMD64_BUILDER", "artemis")
|
|
arm64_builder := env_var_or_default("ARM64_BUILDER", "gaia")
|
|
|
|
# (re)build PACKAGE at one or more VERSIONS on OS/TAG/ARCH via a remote builder
|
|
rebuild os tag arch package *versions:
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
if [ -z "{{ versions }}" ]; then
|
|
echo "error: provide at least one version, e.g. just rebuild alpine 3.23 amd64 rlang 1.1.4" >&2
|
|
exit 1
|
|
fi
|
|
case "{{ arch }}" in
|
|
amd64) builder="{{ amd64_builder }}" ;;
|
|
arm64) builder="{{ arm64_builder }}" ;;
|
|
*) echo "error: arch must be 'amd64' or 'arm64'" >&2; exit 1 ;;
|
|
esac
|
|
# Accept TF_VAR_-prefixed names (direnv) or plain names.
|
|
export B2_S3_ACCESS_KEY="${B2_S3_ACCESS_KEY:-${TF_VAR_B2_S3_ACCESS_KEY:-}}"
|
|
export B2_S3_SECRET_KEY="${B2_S3_SECRET_KEY:-${TF_VAR_B2_S3_SECRET_KEY:-}}"
|
|
: "${B2_S3_ACCESS_KEY:?set B2_S3_ACCESS_KEY or TF_VAR_B2_S3_ACCESS_KEY}"
|
|
: "${B2_S3_SECRET_KEY:?set B2_S3_SECRET_KEY or TF_VAR_B2_S3_SECRET_KEY}"
|
|
: "${PGPASS:?set PGPASS}"
|
|
secret_args=(
|
|
--secret id=b2_access,env=B2_S3_ACCESS_KEY
|
|
--secret id=b2_secret,env=B2_S3_SECRET_KEY
|
|
--secret id=pgpass,env=PGPASS
|
|
)
|
|
# GITHUB_PAT is optional (raises the GitHub API rate limit); pass only if set.
|
|
if [ -n "${GITHUB_PAT:-}" ]; then
|
|
secret_args+=(--secret id=github_pat,env=GITHUB_PAT)
|
|
fi
|
|
echo "Dispatching build of {{ package }} ({{ versions }}) on {{ os }}:{{ tag }}/{{ arch }} (R {{ r_version }}) to builder '$builder'"
|
|
docker buildx build \
|
|
--builder "$builder" \
|
|
--platform "linux/{{ arch }}" \
|
|
--no-cache \
|
|
--progress=plain \
|
|
--output type=cacheonly \
|
|
"${secret_args[@]}" \
|
|
--build-arg OS="{{ os }}" \
|
|
--build-arg OS_VERSION="{{ tag }}" \
|
|
--build-arg R_VERSION="{{ r_version }}" \
|
|
--build-arg PACKAGE="{{ package }}" \
|
|
--build-arg VERSIONS="{{ versions }}" \
|
|
--build-arg CACHEBUST="$(date +%s)" \
|
|
-f docker/build-one.Dockerfile \
|
|
local
|
|
|
|
# run the edge middleware routing matrix (uses a local deno, else the deno image)
|
|
edge-test:
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
if command -v deno >/dev/null 2>&1; then
|
|
deno test -A edge/rpkgs-router.test.ts
|
|
else
|
|
docker run --rm \
|
|
-v "$PWD:/w" -w /w \
|
|
-v deno-cache:/deno-dir \
|
|
denoland/deno:latest \
|
|
deno test -A edge/rpkgs-router.test.ts
|
|
fi
|
|
|
|
# fail if any pipeline still targets a slot the edge does not route
|
|
check-slots:
|
|
Rscript local/check-slots.R
|
|
|
|
# strip a retired slot from .crow/*.yaml (dry run; pass --apply to write)
|
|
retire-slot codename *flags:
|
|
Rscript local/retire-slot.R {{ codename }} --config-only {{ flags }}
|