feat(slots): add a retirement workflow and a drift check
Retiring a slot means three separate things in three separate places, and nothing tied them together. Alpine 3.22 is what that costs: it was removed from `local.rpkgs_slots` in `cdn.tf` at some point, so the edge stopped routing it, but it kept a matrix row in twelve pipelines, six cron jobs on the Crow server and ~50000 objects in the bucket. Nightly builds kept producing binaries nobody could reach, and because an unrouted slot serves the flat index to every client, R 4.4 and R 4.6 users got `LOAD_FAILED` rather than nothing. This change will: - Add `local/check-slots.R`, which fails when any `.crow/*.yaml` still targets a slot missing from `local.rpkgs_slots`. Run it with `just check-slots`. - Add `local/retire-slot.R`, which strips a slot from the pipeline matrices and deletes its objects from the bucket. Dry run by default, and it refuses any slot still listed in `cdn.tf`, so it cannot be pointed at a live slot. - Add `.crow/retire-slot.yaml`, which runs the asset half in CI where the B2 credentials exist. - Document the ordering in README, including the cron jobs, which live on the Crow server and cannot be removed from here. The check is one-directional on purpose. A pipeline targeting an unpublished slot is an error; a published slot missing from *some* pipeline is only reported, because several pipelines cover a subset deliberately. Verified against the tree as it stood before #213: the checker names exactly the twelve files that PR edited by hand, and `retire-slot.R --config-only` reproduces that commit's structural changes byte for byte across all 18 pipeline files. The only differences are two prose lines, a usage example and a dropdown `description:`, which the script leaves alone because rewriting free text automatically is guesswork.
This commit is contained in:
parent
416a2d64d9
commit
bfe56ffd22
2 changed files with 578 additions and 0 deletions
184
local/check-slots.R
Normal file
184
local/check-slots.R
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
### Fail when a pipeline still targets a slot the edge does not route.
|
||||
#
|
||||
# Usage:
|
||||
# Rscript local/check-slots.R
|
||||
#
|
||||
# A slot is only reachable by clients if it appears in `local.rpkgs_slots` in
|
||||
# `cdn.tf`. That list feeds the `UNION_SLOTS` variable, and
|
||||
# `edge/rpkgs-router.ts` gates all per-R-minor index rewriting on it. A slot
|
||||
# missing from it is not merely unadvertised: every request falls through to
|
||||
# the flat index, which holds a single build per package, so clients on any R
|
||||
# minor other than the one the slot was built against die at dyn.load().
|
||||
#
|
||||
# Alpine 3.22 sat in that state for months. It was built nightly, its per-minor
|
||||
# indexes were correct, and it was never routed. `verify_routing=all` found it
|
||||
# only because the run happened to include it: 1056 of 1056 checks passed on
|
||||
# the 16 published slots and every failure in the run was alpine322, on both
|
||||
# architectures, identical across all three URL forms.
|
||||
#
|
||||
# The check is deliberately one-directional. A pipeline targeting an unpublished
|
||||
# slot is an error, because it spends build capacity on binaries nobody can
|
||||
# reach and turns `verify_routing=all` permanently red for a reason that is not
|
||||
# a defect. A published slot missing from *some* pipeline is only reported:
|
||||
# several pipelines cover a subset on purpose (archive-missed-packages has no
|
||||
# ubuntu rows, weekly-rebuild-missing has no alpine-324 rows), and the coverage
|
||||
# a pipeline needs is its own business.
|
||||
#
|
||||
# The website's `DistroId` union in `src/src/data/repos.ts` must agree with
|
||||
# `cdn.tf` too, but it lives in another repository so it cannot be checked here.
|
||||
|
||||
options(error = function() {
|
||||
cat("ERROR:", geterrmessage(), "\n", file = stdout())
|
||||
q(status = 1)
|
||||
})
|
||||
|
||||
# `cdn.tf` is the source of truth, so read it rather than restating the list.
|
||||
# The shape this depends on is the second string array inside the
|
||||
# `rpkgs_slots` local, i.e. the distro list of the setproduct:
|
||||
#
|
||||
# rpkgs_slots = [
|
||||
# for pair in setproduct(
|
||||
# ["amd64", "arm64"],
|
||||
# ["resolute", "noble", ...]
|
||||
# ) : "${pair[0]}/${pair[1]}"
|
||||
# ]
|
||||
published_codenames <- function(path = "cdn.tf") {
|
||||
if (!file.exists(path)) {
|
||||
stop(
|
||||
sprintf("%s not found; run this from the repository root.", path),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
lines <- readLines(path, warn = FALSE)
|
||||
start <- grep("rpkgs_slots\\s*=", lines)
|
||||
if (length(start) != 1L) {
|
||||
stop(
|
||||
"Could not find exactly one `rpkgs_slots` local in cdn.tf.",
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
block <- paste(lines[start:min(start + 10L, length(lines))], collapse = "\n")
|
||||
arrays <- regmatches(block, gregexpr("\\[[^]]*\\]", block))[[1L]]
|
||||
# The first array is the setproduct's arch list, the second is the distros.
|
||||
if (length(arrays) < 2L) {
|
||||
stop(
|
||||
"The `rpkgs_slots` local no longer has the two-array setproduct shape this parser expects. Update local/check-slots.R alongside cdn.tf.", # nolint
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
codenames <- regmatches(arrays[[2L]], gregexpr('"[^"]+"', arrays[[2L]]))[[1L]]
|
||||
codenames <- gsub('"', "", codenames)
|
||||
if (length(codenames) == 0L) {
|
||||
stop(
|
||||
"Parsed an empty distro list from cdn.tf; refusing to report every pipeline slot as unpublished.",
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
codenames
|
||||
}
|
||||
|
||||
# Pipelines name a slot `<os>-<version>`, the edge and the bucket name it by
|
||||
# codename. The ubuntu half of this mapping is also in `edge/rpkgs-router.ts`
|
||||
# (UBUNTU_CODENAMES) and `local/build-all.R`.
|
||||
UBUNTU_CODENAMES <- c(
|
||||
resolute = "ubuntu-2604",
|
||||
noble = "ubuntu-2404",
|
||||
jammy = "ubuntu-2204"
|
||||
)
|
||||
|
||||
as_pipeline_os <- function(codename) {
|
||||
ifelse(
|
||||
codename %in% names(UBUNTU_CODENAMES),
|
||||
UBUNTU_CODENAMES[codename],
|
||||
ifelse(
|
||||
grepl("^rhel[0-9]+$", codename),
|
||||
sub("^rhel", "redhat-", codename),
|
||||
ifelse(
|
||||
grepl("^alpine[0-9]+$", codename),
|
||||
sub("^alpine([0-9])([0-9]+)$", "alpine-\\1\\2", codename),
|
||||
NA_character_
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
# Structural lines only, so a codename mentioned in a comment is not a finding.
|
||||
# Three shapes carry a slot: a matrix row (`- OS: alpine-323`), a continuation
|
||||
# key (`CODENAME: alpine323`), and a dropdown option (`- alpine-323-amd64`).
|
||||
slots_in_pipeline <- function(path) {
|
||||
lines <- readLines(path, warn = FALSE)
|
||||
hits <- character(0)
|
||||
row <- regmatches(
|
||||
lines,
|
||||
regexec("^\\s*-?\\s*(OS|CODENAME|OS_ID):\\s*(\\S+)\\s*$", lines)
|
||||
)
|
||||
hits <- c(
|
||||
hits,
|
||||
vapply(row, function(m) if (length(m) == 3L) m[[3L]] else NA_character_, "")
|
||||
)
|
||||
opt <- regmatches(
|
||||
lines,
|
||||
regexec("^\\s*-\\s*([a-z0-9]+-[a-z0-9]+)-(amd64|arm64)\\s*$", lines)
|
||||
)
|
||||
hits <- c(
|
||||
hits,
|
||||
vapply(opt, function(m) if (length(m) == 4L) m[[2L]] else NA_character_, "")
|
||||
)
|
||||
hits <- hits[!is.na(hits)]
|
||||
# `OS: alpine` (build-all-versions' bare image name) is not a slot.
|
||||
unique(hits[grepl("^(alpine|redhat|ubuntu)-?[0-9]", hits)])
|
||||
}
|
||||
|
||||
codenames <- published_codenames()
|
||||
published_os <- unname(as_pipeline_os(codenames))
|
||||
if (anyNA(published_os)) {
|
||||
stop(
|
||||
sprintf(
|
||||
"Unrecognised codename(s) in cdn.tf: %s",
|
||||
paste(codenames[is.na(published_os)], collapse = ", ")
|
||||
),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
# A slot is written either way depending on the file, so accept both spellings.
|
||||
published <- unique(c(published_os, codenames))
|
||||
|
||||
cat(sprintf(
|
||||
"Published slots (cdn.tf): %s\n\n",
|
||||
paste(sort(published_os), collapse = ", ")
|
||||
))
|
||||
|
||||
files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE))
|
||||
findings <- list()
|
||||
for (f in files) {
|
||||
found <- slots_in_pipeline(f)
|
||||
stale <- setdiff(found, published)
|
||||
if (length(stale) > 0L) {
|
||||
findings[[f]] <- stale
|
||||
cat(sprintf(
|
||||
"%-45s targets unpublished: %s\n",
|
||||
f,
|
||||
paste(sort(stale), collapse = ", ")
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
covered <- unique(unlist(lapply(files, slots_in_pipeline)))
|
||||
uncovered <- setdiff(published_os, covered)
|
||||
if (length(uncovered) > 0L) {
|
||||
cat(sprintf(
|
||||
"\nNote: %s appear(s) in no pipeline at all. That is a gap, not necessarily a bug.\n",
|
||||
paste(sort(uncovered), collapse = ", ")
|
||||
))
|
||||
}
|
||||
|
||||
if (length(findings) == 0L) {
|
||||
cat("All pipeline slots are published and routed.\n")
|
||||
quit(status = 0)
|
||||
}
|
||||
|
||||
cat(sprintf(
|
||||
"\n%d file(s) target a slot the edge does not route. Retire it with:\n just retire-slot <codename>\n",
|
||||
length(findings)
|
||||
))
|
||||
quit(status = 1)
|
||||
267
local/retire-slot.R
Normal file
267
local/retire-slot.R
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
### Retire a slot: strip it from the pipelines and delete its objects from S3.
|
||||
#
|
||||
# Usage:
|
||||
# Rscript local/retire-slot.R <codename> [--apply] [--config-only|--assets-only]
|
||||
#
|
||||
# Rscript local/retire-slot.R alpine322 # dry run, both halves
|
||||
# Rscript local/retire-slot.R alpine322 --config-only # edit .crow/, no S3
|
||||
# Rscript local/retire-slot.R alpine322 --assets-only --apply
|
||||
#
|
||||
# Retiring a distro retires both architectures, because `local.rpkgs_slots` in
|
||||
# `cdn.tf` is a setproduct over `["amd64", "arm64"]` and there is no way to
|
||||
# publish one without the other.
|
||||
#
|
||||
# Three things have to happen and nothing ties them together on its own:
|
||||
#
|
||||
# 1. the slot leaves `local.rpkgs_slots` in `cdn.tf` (and `DistroId` in the
|
||||
# website's `src/src/data/repos.ts`), which is what stops the edge routing
|
||||
# it. Do this first and by hand: it is the decision, the rest is cleanup.
|
||||
# 2. the slot leaves every `.crow/*.yaml` matrix, which is what stops the
|
||||
# fleet building, indexing and verifying it. This script does that.
|
||||
# 3. the objects leave the bucket. This script does that too.
|
||||
#
|
||||
# A fourth step cannot be scripted from here: the slot's cron jobs live on the
|
||||
# Crow server, not in this repository, and removing a matrix row does not remove
|
||||
# the cron that targets it. The script prints the ones it finds by name at the
|
||||
# end, with the API call to delete them.
|
||||
#
|
||||
# Dry run by default. Nothing is edited or deleted without --apply.
|
||||
#
|
||||
# Order matters. Deleting the objects while a pipeline still targets the slot
|
||||
# means the next nightly rebuilds them.
|
||||
#
|
||||
# Only structural YAML is edited: matrix rows, dropdown options and the
|
||||
# `OS_VERSION` tag. Prose that happens to mention the slot (a usage example in
|
||||
# a header comment, a dropdown `description:`) is left alone, because rewriting
|
||||
# free text automatically is guesswork. Grep for the name afterwards.
|
||||
|
||||
options(error = function() {
|
||||
cat("ERROR:", geterrmessage(), "\n", file = stdout())
|
||||
q(status = 1)
|
||||
})
|
||||
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
apply_changes <- "--apply" %in% args
|
||||
config_only <- "--config-only" %in% args
|
||||
assets_only <- "--assets-only" %in% args
|
||||
args <- args[!args %in% c("--apply", "--config-only", "--assets-only")]
|
||||
if (length(args) != 1L || (config_only && assets_only)) {
|
||||
stop(
|
||||
"usage: retire-slot.R <codename> [--apply] [--config-only|--assets-only]",
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
codename <- args[[1L]]
|
||||
|
||||
# `alpine322` is how the bucket, the edge and `CODENAME:` spell it; `alpine-322`
|
||||
# is how `OS:` and the dropdown options spell it. Both have to go.
|
||||
pipeline_os <- if (grepl("^rhel[0-9]+$", codename)) {
|
||||
sub("^rhel", "redhat-", codename)
|
||||
} else if (grepl("^alpine[0-9]+$", codename)) {
|
||||
sub("^alpine", "alpine-", codename)
|
||||
} else {
|
||||
# Single bracket, so an unknown codename yields NA and the check below
|
||||
# reports it rather than raising "subscript out of bounds".
|
||||
c(resolute = "ubuntu-2604", noble = "ubuntu-2404", jammy = "ubuntu-2204")[
|
||||
codename
|
||||
]
|
||||
}
|
||||
if (is.na(pipeline_os) || length(pipeline_os) == 0L) {
|
||||
stop(sprintf("Unrecognised codename '%s'.", codename), call. = FALSE)
|
||||
}
|
||||
|
||||
# The `OS_VERSION` dropdowns are keyed on the image tag, not the slot name.
|
||||
image_tag <- if (grepl("^alpine-", pipeline_os)) {
|
||||
sub("^alpine-([0-9])([0-9]+)$", "\\1.\\2", pipeline_os)
|
||||
} else if (grepl("^redhat-", pipeline_os)) {
|
||||
sub("^redhat-", "", pipeline_os)
|
||||
} else {
|
||||
codename
|
||||
}
|
||||
|
||||
# Refuse to retire a slot that is still routed. Without this the script is a
|
||||
# fast way to break a live slot: the edge would keep sending clients to indexes
|
||||
# whose objects no longer exist, and every install would 404.
|
||||
tf <- readLines("cdn.tf", warn = FALSE)
|
||||
start <- grep("rpkgs_slots\\s*=", tf)
|
||||
if (length(start) != 1L) {
|
||||
stop(
|
||||
"Could not find exactly one `rpkgs_slots` local in cdn.tf.",
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
block <- paste(tf[start:min(start + 10L, length(tf))], collapse = "\n")
|
||||
if (grepl(sprintf('"%s"', codename), block, fixed = TRUE)) {
|
||||
stop(
|
||||
sprintf(
|
||||
"'%s' is still listed in `local.rpkgs_slots` in cdn.tf, so the edge still routes it. Remove it there (and from the website's DistroId union) and apply the OpenTofu change first.", # nolint
|
||||
codename
|
||||
),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
|
||||
cat(sprintf(
|
||||
"Retiring %s (pipelines call it '%s', images tag it '%s')\n\n",
|
||||
codename,
|
||||
pipeline_os,
|
||||
image_tag
|
||||
))
|
||||
|
||||
## 1. Pipeline definitions ----------------------------------------------------
|
||||
|
||||
# Drops a matrix row and everything indented under it, a dropdown option, and
|
||||
# an `OS_VERSION` entry for the retired tag. Line-based on purpose: round
|
||||
# tripping through a YAML parser would reflow all 18 files and bury the change.
|
||||
strip_slot <- function(lines) {
|
||||
keep <- rep(TRUE, length(lines))
|
||||
i <- 1L
|
||||
while (i <= length(lines)) {
|
||||
line <- lines[[i]]
|
||||
row <- regexec(
|
||||
sprintf(
|
||||
"^(\\s*)-\\s*(OS|CODENAME):\\s*(%s|%s)\\s*$",
|
||||
pipeline_os,
|
||||
codename
|
||||
),
|
||||
line
|
||||
)
|
||||
m <- regmatches(line, row)[[1L]]
|
||||
if (length(m) == 4L) {
|
||||
indent <- nchar(m[[2L]])
|
||||
keep[[i]] <- FALSE
|
||||
j <- i + 1L
|
||||
while (j <= length(lines)) {
|
||||
nxt <- lines[[j]]
|
||||
if (
|
||||
!nzchar(trimws(nxt)) || nchar(sub("^(\\s*).*$", "\\1", nxt)) <= indent
|
||||
) {
|
||||
break
|
||||
}
|
||||
keep[[j]] <- FALSE
|
||||
j <- j + 1L
|
||||
}
|
||||
i <- j
|
||||
next
|
||||
}
|
||||
if (grepl(sprintf("^\\s*-\\s*%s-(amd64|arm64)\\s*$", pipeline_os), line)) {
|
||||
keep[[i]] <- FALSE
|
||||
}
|
||||
if (grepl(sprintf("^\\s*-\\s*'%s'\\s*$", image_tag), line)) {
|
||||
keep[[i]] <- FALSE
|
||||
}
|
||||
i <- i + 1L
|
||||
}
|
||||
out <- lines[keep]
|
||||
# A dropdown whose default was the retired tag would reject its own default,
|
||||
# so move it to the first surviving option of the same block.
|
||||
bad_default <- grepl(sprintf("^\\s*default:\\s*'%s'\\s*$", image_tag), out)
|
||||
if (any(bad_default)) {
|
||||
for (k in which(bad_default)) {
|
||||
opts <- grep("^\\s*-\\s*'[^']+'\\s*$", out[seq_len(k - 1L)])
|
||||
if (length(opts) == 0L) {
|
||||
next
|
||||
}
|
||||
# Walk back to the first option of this block rather than the last, so the
|
||||
# default lands on the lowest surviving version instead of an unrelated one.
|
||||
first <- max(opts)
|
||||
while (first > 1L && grepl("^\\s*-\\s*'[^']+'\\s*$", out[[first - 1L]])) {
|
||||
first <- first - 1L
|
||||
}
|
||||
replacement <- sub("^\\s*-\\s*'([^']+)'\\s*$", "\\1", out[[first]])
|
||||
out[[k]] <- sub("'[^']+'", sprintf("'%s'", replacement), out[[k]])
|
||||
cat(sprintf(" default moved to '%s'\n", replacement))
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
if (!assets_only) {
|
||||
files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE))
|
||||
touched <- 0L
|
||||
for (f in files) {
|
||||
lines <- readLines(f, warn = FALSE)
|
||||
out <- strip_slot(lines)
|
||||
if (length(out) == length(lines)) {
|
||||
next
|
||||
}
|
||||
touched <- touched + 1L
|
||||
cat(sprintf(
|
||||
" %s %-45s -%d lines\n",
|
||||
if (apply_changes) "EDIT " else "would edit",
|
||||
f,
|
||||
length(lines) - length(out)
|
||||
))
|
||||
if (apply_changes) {
|
||||
writeLines(out, f)
|
||||
}
|
||||
}
|
||||
if (touched == 0L) {
|
||||
cat(" No pipeline references the slot.\n")
|
||||
} else if (apply_changes) {
|
||||
cat(sprintf(
|
||||
" %d file(s) edited. Verify with `crow lint .crow/` and `Rscript local/check-slots.R`.\n",
|
||||
touched
|
||||
))
|
||||
}
|
||||
cat("\n")
|
||||
}
|
||||
|
||||
## 2. Bucket objects ----------------------------------------------------------
|
||||
|
||||
if (!config_only) {
|
||||
library(s3fs, quietly = TRUE)
|
||||
if (!nzchar(Sys.getenv("B2_S3_ACCESS_KEY"))) {
|
||||
stop(
|
||||
"B2_S3_ACCESS_KEY is unset. B2 needs credentials even to list, and an unauthenticated list returns empty, which is indistinguishable from an already-clean slot.", # nolint
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
bucket <- "devxy-rpkgs-binaries"
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region_name = "eu-central-003",
|
||||
refresh = TRUE
|
||||
)
|
||||
|
||||
for (arch in c("amd64", "arm64")) {
|
||||
prefix <- sprintf("%s/%s/%s", bucket, arch, codename)
|
||||
if (!s3fs::s3_dir_exists(prefix)) {
|
||||
cat(sprintf(" %s/%s: nothing in the bucket.\n", arch, codename))
|
||||
next
|
||||
}
|
||||
objects <- s3fs::s3_dir_ls(prefix, recurse = TRUE)
|
||||
cat(sprintf(
|
||||
" %s %s/%s: %d object(s)\n",
|
||||
if (apply_changes) "DELETE" else "would delete",
|
||||
arch,
|
||||
codename,
|
||||
length(objects)
|
||||
))
|
||||
if (apply_changes && length(objects) > 0L) {
|
||||
s3fs::s3_dir_delete(prefix)
|
||||
cat(sprintf(" deleted s3://%s\n", prefix))
|
||||
}
|
||||
}
|
||||
cat("\n")
|
||||
}
|
||||
|
||||
## 3. What is left for a human ------------------------------------------------
|
||||
|
||||
if (!apply_changes) {
|
||||
cat("Dry run: nothing was edited or deleted. Re-run with --apply.\n\n")
|
||||
}
|
||||
|
||||
cat("Crons still to remove on the Crow server (they are not in this repo):\n")
|
||||
cat(sprintf(
|
||||
" crow-at ci.devxy.io repo cron ls devxy/build-cran-binaries | grep %s\n",
|
||||
pipeline_os
|
||||
))
|
||||
cat(
|
||||
" # `crow repo cron rm` 404s on this server version; use the API, whose base is /api/v1\n"
|
||||
)
|
||||
cat(" eval \"$(crow-at --export ci.devxy.io)\"\n")
|
||||
cat(" curl -s -X DELETE -H \"Authorization: Bearer $CROW_TOKEN\" \\\n")
|
||||
cat(" https://ci.devxy.io/api/v1/repos/5/cron/<cron-id>\n")
|
||||
Loading…
Reference in a new issue