pat-s
bfe56ffd22Retiring a slot means three separate things in three separate places, and nothing tied them together. Alpine 3.22 is what that costs: it was removed from `local.rpkgs_slots` in `cdn.tf` at some point, so the edge stopped routing it, but it kept a matrix row in twelve pipelines, six cron jobs on the Crow server and ~50000 objects in the bucket. Nightly builds kept producing binaries nobody could reach, and because an unrouted slot serves the flat index to every client, R 4.4 and R 4.6 users got `LOAD_FAILED` rather than nothing. This change will: - Add `local/check-slots.R`, which fails when any `.crow/*.yaml` still targets a slot missing from `local.rpkgs_slots`. Run it with `just check-slots`. - Add `local/retire-slot.R`, which strips a slot from the pipeline matrices and deletes its objects from the bucket. Dry run by default, and it refuses any slot still listed in `cdn.tf`, so it cannot be pointed at a live slot. - Add `.crow/retire-slot.yaml`, which runs the asset half in CI where the B2 credentials exist. - Document the ordering in README, including the cron jobs, which live on the Crow server and cannot be removed from here. The check is one-directional on purpose. A pipeline targeting an unpublished slot is an error; a published slot missing from *some* pipeline is only reported, because several pipelines cover a subset deliberately. Verified against the tree as it stood before #213: the checker names exactly the twelve files that PR edited by hand, and `retire-slot.R --config-only` reproduces that commit's structural changes byte for byte across all 18 pipeline files. The only differences are two prose lines, a usage example and a dropdown `description:`, which the script leaves alone because rewriting free text automatically is guesswork.
267 lines
9 KiB
R
267 lines
9 KiB
R
### Retire a slot: strip it from the pipelines and delete its objects from S3.
|
|
#
|
|
# Usage:
|
|
# Rscript local/retire-slot.R <codename> [--apply] [--config-only|--assets-only]
|
|
#
|
|
# Rscript local/retire-slot.R alpine322 # dry run, both halves
|
|
# Rscript local/retire-slot.R alpine322 --config-only # edit .crow/, no S3
|
|
# Rscript local/retire-slot.R alpine322 --assets-only --apply
|
|
#
|
|
# Retiring a distro retires both architectures, because `local.rpkgs_slots` in
|
|
# `cdn.tf` is a setproduct over `["amd64", "arm64"]` and there is no way to
|
|
# publish one without the other.
|
|
#
|
|
# Three things have to happen and nothing ties them together on its own:
|
|
#
|
|
# 1. the slot leaves `local.rpkgs_slots` in `cdn.tf` (and `DistroId` in the
|
|
# website's `src/src/data/repos.ts`), which is what stops the edge routing
|
|
# it. Do this first and by hand: it is the decision, the rest is cleanup.
|
|
# 2. the slot leaves every `.crow/*.yaml` matrix, which is what stops the
|
|
# fleet building, indexing and verifying it. This script does that.
|
|
# 3. the objects leave the bucket. This script does that too.
|
|
#
|
|
# A fourth step cannot be scripted from here: the slot's cron jobs live on the
|
|
# Crow server, not in this repository, and removing a matrix row does not remove
|
|
# the cron that targets it. The script prints the ones it finds by name at the
|
|
# end, with the API call to delete them.
|
|
#
|
|
# Dry run by default. Nothing is edited or deleted without --apply.
|
|
#
|
|
# Order matters. Deleting the objects while a pipeline still targets the slot
|
|
# means the next nightly rebuilds them.
|
|
#
|
|
# Only structural YAML is edited: matrix rows, dropdown options and the
|
|
# `OS_VERSION` tag. Prose that happens to mention the slot (a usage example in
|
|
# a header comment, a dropdown `description:`) is left alone, because rewriting
|
|
# free text automatically is guesswork. Grep for the name afterwards.
|
|
|
|
options(error = function() {
|
|
cat("ERROR:", geterrmessage(), "\n", file = stdout())
|
|
q(status = 1)
|
|
})
|
|
|
|
args <- commandArgs(trailingOnly = TRUE)
|
|
apply_changes <- "--apply" %in% args
|
|
config_only <- "--config-only" %in% args
|
|
assets_only <- "--assets-only" %in% args
|
|
args <- args[!args %in% c("--apply", "--config-only", "--assets-only")]
|
|
if (length(args) != 1L || (config_only && assets_only)) {
|
|
stop(
|
|
"usage: retire-slot.R <codename> [--apply] [--config-only|--assets-only]",
|
|
call. = FALSE
|
|
)
|
|
}
|
|
codename <- args[[1L]]
|
|
|
|
# `alpine322` is how the bucket, the edge and `CODENAME:` spell it; `alpine-322`
|
|
# is how `OS:` and the dropdown options spell it. Both have to go.
|
|
pipeline_os <- if (grepl("^rhel[0-9]+$", codename)) {
|
|
sub("^rhel", "redhat-", codename)
|
|
} else if (grepl("^alpine[0-9]+$", codename)) {
|
|
sub("^alpine", "alpine-", codename)
|
|
} else {
|
|
# Single bracket, so an unknown codename yields NA and the check below
|
|
# reports it rather than raising "subscript out of bounds".
|
|
c(resolute = "ubuntu-2604", noble = "ubuntu-2404", jammy = "ubuntu-2204")[
|
|
codename
|
|
]
|
|
}
|
|
if (is.na(pipeline_os) || length(pipeline_os) == 0L) {
|
|
stop(sprintf("Unrecognised codename '%s'.", codename), call. = FALSE)
|
|
}
|
|
|
|
# The `OS_VERSION` dropdowns are keyed on the image tag, not the slot name.
|
|
image_tag <- if (grepl("^alpine-", pipeline_os)) {
|
|
sub("^alpine-([0-9])([0-9]+)$", "\\1.\\2", pipeline_os)
|
|
} else if (grepl("^redhat-", pipeline_os)) {
|
|
sub("^redhat-", "", pipeline_os)
|
|
} else {
|
|
codename
|
|
}
|
|
|
|
# Refuse to retire a slot that is still routed. Without this the script is a
|
|
# fast way to break a live slot: the edge would keep sending clients to indexes
|
|
# whose objects no longer exist, and every install would 404.
|
|
tf <- readLines("cdn.tf", warn = FALSE)
|
|
start <- grep("rpkgs_slots\\s*=", tf)
|
|
if (length(start) != 1L) {
|
|
stop(
|
|
"Could not find exactly one `rpkgs_slots` local in cdn.tf.",
|
|
call. = FALSE
|
|
)
|
|
}
|
|
block <- paste(tf[start:min(start + 10L, length(tf))], collapse = "\n")
|
|
if (grepl(sprintf('"%s"', codename), block, fixed = TRUE)) {
|
|
stop(
|
|
sprintf(
|
|
"'%s' is still listed in `local.rpkgs_slots` in cdn.tf, so the edge still routes it. Remove it there (and from the website's DistroId union) and apply the OpenTofu change first.", # nolint
|
|
codename
|
|
),
|
|
call. = FALSE
|
|
)
|
|
}
|
|
|
|
cat(sprintf(
|
|
"Retiring %s (pipelines call it '%s', images tag it '%s')\n\n",
|
|
codename,
|
|
pipeline_os,
|
|
image_tag
|
|
))
|
|
|
|
## 1. Pipeline definitions ----------------------------------------------------
|
|
|
|
# Drops a matrix row and everything indented under it, a dropdown option, and
|
|
# an `OS_VERSION` entry for the retired tag. Line-based on purpose: round
|
|
# tripping through a YAML parser would reflow all 18 files and bury the change.
|
|
strip_slot <- function(lines) {
|
|
keep <- rep(TRUE, length(lines))
|
|
i <- 1L
|
|
while (i <= length(lines)) {
|
|
line <- lines[[i]]
|
|
row <- regexec(
|
|
sprintf(
|
|
"^(\\s*)-\\s*(OS|CODENAME):\\s*(%s|%s)\\s*$",
|
|
pipeline_os,
|
|
codename
|
|
),
|
|
line
|
|
)
|
|
m <- regmatches(line, row)[[1L]]
|
|
if (length(m) == 4L) {
|
|
indent <- nchar(m[[2L]])
|
|
keep[[i]] <- FALSE
|
|
j <- i + 1L
|
|
while (j <= length(lines)) {
|
|
nxt <- lines[[j]]
|
|
if (
|
|
!nzchar(trimws(nxt)) || nchar(sub("^(\\s*).*$", "\\1", nxt)) <= indent
|
|
) {
|
|
break
|
|
}
|
|
keep[[j]] <- FALSE
|
|
j <- j + 1L
|
|
}
|
|
i <- j
|
|
next
|
|
}
|
|
if (grepl(sprintf("^\\s*-\\s*%s-(amd64|arm64)\\s*$", pipeline_os), line)) {
|
|
keep[[i]] <- FALSE
|
|
}
|
|
if (grepl(sprintf("^\\s*-\\s*'%s'\\s*$", image_tag), line)) {
|
|
keep[[i]] <- FALSE
|
|
}
|
|
i <- i + 1L
|
|
}
|
|
out <- lines[keep]
|
|
# A dropdown whose default was the retired tag would reject its own default,
|
|
# so move it to the first surviving option of the same block.
|
|
bad_default <- grepl(sprintf("^\\s*default:\\s*'%s'\\s*$", image_tag), out)
|
|
if (any(bad_default)) {
|
|
for (k in which(bad_default)) {
|
|
opts <- grep("^\\s*-\\s*'[^']+'\\s*$", out[seq_len(k - 1L)])
|
|
if (length(opts) == 0L) {
|
|
next
|
|
}
|
|
# Walk back to the first option of this block rather than the last, so the
|
|
# default lands on the lowest surviving version instead of an unrelated one.
|
|
first <- max(opts)
|
|
while (first > 1L && grepl("^\\s*-\\s*'[^']+'\\s*$", out[[first - 1L]])) {
|
|
first <- first - 1L
|
|
}
|
|
replacement <- sub("^\\s*-\\s*'([^']+)'\\s*$", "\\1", out[[first]])
|
|
out[[k]] <- sub("'[^']+'", sprintf("'%s'", replacement), out[[k]])
|
|
cat(sprintf(" default moved to '%s'\n", replacement))
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
if (!assets_only) {
|
|
files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE))
|
|
touched <- 0L
|
|
for (f in files) {
|
|
lines <- readLines(f, warn = FALSE)
|
|
out <- strip_slot(lines)
|
|
if (length(out) == length(lines)) {
|
|
next
|
|
}
|
|
touched <- touched + 1L
|
|
cat(sprintf(
|
|
" %s %-45s -%d lines\n",
|
|
if (apply_changes) "EDIT " else "would edit",
|
|
f,
|
|
length(lines) - length(out)
|
|
))
|
|
if (apply_changes) {
|
|
writeLines(out, f)
|
|
}
|
|
}
|
|
if (touched == 0L) {
|
|
cat(" No pipeline references the slot.\n")
|
|
} else if (apply_changes) {
|
|
cat(sprintf(
|
|
" %d file(s) edited. Verify with `crow lint .crow/` and `Rscript local/check-slots.R`.\n",
|
|
touched
|
|
))
|
|
}
|
|
cat("\n")
|
|
}
|
|
|
|
## 2. Bucket objects ----------------------------------------------------------
|
|
|
|
if (!config_only) {
|
|
library(s3fs, quietly = TRUE)
|
|
if (!nzchar(Sys.getenv("B2_S3_ACCESS_KEY"))) {
|
|
stop(
|
|
"B2_S3_ACCESS_KEY is unset. B2 needs credentials even to list, and an unauthenticated list returns empty, which is indistinguishable from an already-clean slot.", # nolint
|
|
call. = FALSE
|
|
)
|
|
}
|
|
bucket <- "devxy-rpkgs-binaries"
|
|
s3fs::s3_file_system(
|
|
aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
|
aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
|
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
|
region_name = "eu-central-003",
|
|
refresh = TRUE
|
|
)
|
|
|
|
for (arch in c("amd64", "arm64")) {
|
|
prefix <- sprintf("%s/%s/%s", bucket, arch, codename)
|
|
if (!s3fs::s3_dir_exists(prefix)) {
|
|
cat(sprintf(" %s/%s: nothing in the bucket.\n", arch, codename))
|
|
next
|
|
}
|
|
objects <- s3fs::s3_dir_ls(prefix, recurse = TRUE)
|
|
cat(sprintf(
|
|
" %s %s/%s: %d object(s)\n",
|
|
if (apply_changes) "DELETE" else "would delete",
|
|
arch,
|
|
codename,
|
|
length(objects)
|
|
))
|
|
if (apply_changes && length(objects) > 0L) {
|
|
s3fs::s3_dir_delete(prefix)
|
|
cat(sprintf(" deleted s3://%s\n", prefix))
|
|
}
|
|
}
|
|
cat("\n")
|
|
}
|
|
|
|
## 3. What is left for a human ------------------------------------------------
|
|
|
|
if (!apply_changes) {
|
|
cat("Dry run: nothing was edited or deleted. Re-run with --apply.\n\n")
|
|
}
|
|
|
|
cat("Crons still to remove on the Crow server (they are not in this repo):\n")
|
|
cat(sprintf(
|
|
" crow-at ci.devxy.io repo cron ls devxy/build-cran-binaries | grep %s\n",
|
|
pipeline_os
|
|
))
|
|
cat(
|
|
" # `crow repo cron rm` 404s on this server version; use the API, whose base is /api/v1\n"
|
|
)
|
|
cat(" eval \"$(crow-at --export ci.devxy.io)\"\n")
|
|
cat(" curl -s -X DELETE -H \"Authorization: Bearer $CROW_TOKEN\" \\\n")
|
|
cat(" https://ci.devxy.io/api/v1/repos/5/cron/<cron-id>\n")
|