From bfe56ffd226303b85c02992cbd9a9e46b93c58fc Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 11 Sep 2026 08:17:21 +0000 Subject: [PATCH 1/2] feat(slots): add a retirement workflow and a drift check Retiring a slot means three separate things in three separate places, and nothing tied them together. Alpine 3.22 is what that costs: it was removed from `local.rpkgs_slots` in `cdn.tf` at some point, so the edge stopped routing it, but it kept a matrix row in twelve pipelines, six cron jobs on the Crow server and ~50000 objects in the bucket. Nightly builds kept producing binaries nobody could reach, and because an unrouted slot serves the flat index to every client, R 4.4 and R 4.6 users got `LOAD_FAILED` rather than nothing. This change will: - Add `local/check-slots.R`, which fails when any `.crow/*.yaml` still targets a slot missing from `local.rpkgs_slots`. Run it with `just check-slots`. - Add `local/retire-slot.R`, which strips a slot from the pipeline matrices and deletes its objects from the bucket. Dry run by default, and it refuses any slot still listed in `cdn.tf`, so it cannot be pointed at a live slot. - Add `.crow/retire-slot.yaml`, which runs the asset half in CI where the B2 credentials exist. - Document the ordering in README, including the cron jobs, which live on the Crow server and cannot be removed from here. The check is one-directional on purpose. A pipeline targeting an unpublished slot is an error; a published slot missing from *some* pipeline is only reported, because several pipelines cover a subset deliberately. Verified against the tree as it stood before #213: the checker names exactly the twelve files that PR edited by hand, and `retire-slot.R --config-only` reproduces that commit's structural changes byte for byte across all 18 pipeline files. The only differences are two prose lines, a usage example and a dropdown `description:`, which the script leaves alone because rewriting free text automatically is guesswork. --- .crow/retire-slot.yaml | 83 +++++++++++++ README.md | 36 ++++++ justfile | 8 ++ local/check-slots.R | 184 ++++++++++++++++++++++++++++ local/retire-slot.R | 267 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 578 insertions(+) create mode 100644 .crow/retire-slot.yaml create mode 100644 local/check-slots.R create mode 100644 local/retire-slot.R diff --git a/.crow/retire-slot.yaml b/.crow/retire-slot.yaml new file mode 100644 index 0000000..323a00e --- /dev/null +++ b/.crow/retire-slot.yaml @@ -0,0 +1,83 @@ +# Delete a retired slot's objects from the bucket. +# +# The config half of a retirement (`.crow/*.yaml` matrices) is a commit, so it +# happens on a workstation: +# +# Rscript local/retire-slot.R --config-only --apply +# +# The asset half needs the B2 credentials, which only exist in CI, so it +# happens here. `local/retire-slot.R` refuses any slot still listed in +# `local.rpkgs_slots` in cdn.tf, so this pipeline cannot delete a live slot +# however it is invoked. +# +# Dry run unless RETIRE_APPLY is 'yes'. The dry run reports the object count, +# which is the number worth looking at before deleting a five-figure prefix. +# +# crow pipeline create devxy/build-cran-binaries --branch main \ +# --var retire_slot=alpine322 --var RETIRE_APPLY=yes +# +# Retiring a distro retires both architectures: `local.rpkgs_slots` is a +# setproduct over both, so there is no way to publish one without the other, +# and the script deletes `amd64//` and `arm64//` together. +# +# The slot's cron jobs are not in this repository and are not removed here. The +# script prints them by name, with the API call, at the end of the run. + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the + # default must be a value the script never accepts. + retire_slot: + description: "Codename of the slot to retire (e.g. alpine322), or 'none'." + default: none + RETIRE_APPLY: + description: "Set to 'yes' to delete. Anything else reports and changes nothing." + options: + - 'no' + - 'yes' + default: 'no' + +when: + - event: manual + evaluate: 'retire_slot != "none"' + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Delete the retired slot from the bucket' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + R_LIBS_USER: /tmp/R-pkgs + # Pipeline variables are substituted into the YAML, not exported, so they + # have to be bound here to be readable as env vars in the script below. + RETIRE_SLOT: ${retire_slot} + RETIRE_APPLY: ${RETIRE_APPLY} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /tmp/R-pkgs + - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R + - | + set -eu + APPLY="" + if [ "$$RETIRE_APPLY" = "yes" ]; then + APPLY="--apply" + fi + /opt/R/4.5.3/bin/Rscript local/retire-slot.R "$$RETIRE_SLOT" --assets-only $$APPLY + backend_options: + docker: + resources: + requests: + memory: 2Gi + cpu: 1000m diff --git a/README.md b/README.md index 989069e..1b63e59 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,42 @@ Example: Technically, no date-based snapshots are planned, so this component from the Posit PM structure is not included. +## Retiring a slot + +A slot is only reachable by clients if it appears in `local.rpkgs_slots` in `cdn.tf`. +That list feeds the `UNION_SLOTS` variable and `edge/rpkgs-router.ts` gates all per-R-minor index rewriting on it. +A slot missing from it is not merely unadvertised, it is broken: every request falls through to the flat index, which holds a single build per package, so clients on any R minor other than the one the slot was built against die at `dyn.load()`. + +Alpine 3.22 sat in that state for months, built nightly and never routed. +Retire a slot in this order: + +1. Remove it from `local.rpkgs_slots` in `cdn.tf` and from `DistroId` in the website's `src/src/data/repos.ts`, then `tofu apply`. + This is the decision; everything below is cleanup. +2. Strip it from the pipeline matrices and commit the result: + + ```sh + # dry run + just retire-slot alpine322 + # write the changes + just retire-slot alpine322 --apply + ``` + +3. Delete its cron jobs on the Crow server. + They are not in this repository, and removing a matrix row does not remove the cron that targets it. + `local/retire-slot.R` prints the ones it finds, with the API call. +4. Delete its objects from the bucket, which needs the B2 credentials and therefore runs in CI: + + ```sh + crow pipeline create devxy/build-cran-binaries --branch main \ + --var retire_slot=alpine322 --var RETIRE_APPLY=yes + ``` + +`local/retire-slot.R` refuses any slot still listed in `cdn.tf`, so steps 2 and 4 cannot run against a live slot. +Doing step 4 before step 2 is the one ordering that actively breaks things: the next nightly would rebuild everything just deleted. + +`just check-slots` fails when any pipeline still targets a slot the edge does not route, which is the drift that put Alpine 3.22 where it was. +It is one-directional on purpose: a published slot missing from _some_ pipeline is only reported, because several pipelines cover a subset deliberately. + ## Helpers Helper scripts are located in `local/`. diff --git a/justfile b/justfile index 6c75b13..36ad0e1 100644 --- a/justfile +++ b/justfile @@ -87,3 +87,11 @@ edge-test: denoland/deno:latest \ deno test -A edge/rpkgs-router.test.ts fi + +# fail if any pipeline still targets a slot the edge does not route +check-slots: + Rscript local/check-slots.R + +# strip a retired slot from .crow/*.yaml (dry run; pass --apply to write) +retire-slot codename *flags: + Rscript local/retire-slot.R {{ codename }} --config-only {{ flags }} diff --git a/local/check-slots.R b/local/check-slots.R new file mode 100644 index 0000000..22e3a9a --- /dev/null +++ b/local/check-slots.R @@ -0,0 +1,184 @@ +### Fail when a pipeline still targets a slot the edge does not route. +# +# Usage: +# Rscript local/check-slots.R +# +# A slot is only reachable by clients if it appears in `local.rpkgs_slots` in +# `cdn.tf`. That list feeds the `UNION_SLOTS` variable, and +# `edge/rpkgs-router.ts` gates all per-R-minor index rewriting on it. A slot +# missing from it is not merely unadvertised: every request falls through to +# the flat index, which holds a single build per package, so clients on any R +# minor other than the one the slot was built against die at dyn.load(). +# +# Alpine 3.22 sat in that state for months. It was built nightly, its per-minor +# indexes were correct, and it was never routed. `verify_routing=all` found it +# only because the run happened to include it: 1056 of 1056 checks passed on +# the 16 published slots and every failure in the run was alpine322, on both +# architectures, identical across all three URL forms. +# +# The check is deliberately one-directional. A pipeline targeting an unpublished +# slot is an error, because it spends build capacity on binaries nobody can +# reach and turns `verify_routing=all` permanently red for a reason that is not +# a defect. A published slot missing from *some* pipeline is only reported: +# several pipelines cover a subset on purpose (archive-missed-packages has no +# ubuntu rows, weekly-rebuild-missing has no alpine-324 rows), and the coverage +# a pipeline needs is its own business. +# +# The website's `DistroId` union in `src/src/data/repos.ts` must agree with +# `cdn.tf` too, but it lives in another repository so it cannot be checked here. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +# `cdn.tf` is the source of truth, so read it rather than restating the list. +# The shape this depends on is the second string array inside the +# `rpkgs_slots` local, i.e. the distro list of the setproduct: +# +# rpkgs_slots = [ +# for pair in setproduct( +# ["amd64", "arm64"], +# ["resolute", "noble", ...] +# ) : "${pair[0]}/${pair[1]}" +# ] +published_codenames <- function(path = "cdn.tf") { + if (!file.exists(path)) { + stop( + sprintf("%s not found; run this from the repository root.", path), + call. = FALSE + ) + } + lines <- readLines(path, warn = FALSE) + start <- grep("rpkgs_slots\\s*=", lines) + if (length(start) != 1L) { + stop( + "Could not find exactly one `rpkgs_slots` local in cdn.tf.", + call. = FALSE + ) + } + block <- paste(lines[start:min(start + 10L, length(lines))], collapse = "\n") + arrays <- regmatches(block, gregexpr("\\[[^]]*\\]", block))[[1L]] + # The first array is the setproduct's arch list, the second is the distros. + if (length(arrays) < 2L) { + stop( + "The `rpkgs_slots` local no longer has the two-array setproduct shape this parser expects. Update local/check-slots.R alongside cdn.tf.", # nolint + call. = FALSE + ) + } + codenames <- regmatches(arrays[[2L]], gregexpr('"[^"]+"', arrays[[2L]]))[[1L]] + codenames <- gsub('"', "", codenames) + if (length(codenames) == 0L) { + stop( + "Parsed an empty distro list from cdn.tf; refusing to report every pipeline slot as unpublished.", + call. = FALSE + ) + } + codenames +} + +# Pipelines name a slot `-`, the edge and the bucket name it by +# codename. The ubuntu half of this mapping is also in `edge/rpkgs-router.ts` +# (UBUNTU_CODENAMES) and `local/build-all.R`. +UBUNTU_CODENAMES <- c( + resolute = "ubuntu-2604", + noble = "ubuntu-2404", + jammy = "ubuntu-2204" +) + +as_pipeline_os <- function(codename) { + ifelse( + codename %in% names(UBUNTU_CODENAMES), + UBUNTU_CODENAMES[codename], + ifelse( + grepl("^rhel[0-9]+$", codename), + sub("^rhel", "redhat-", codename), + ifelse( + grepl("^alpine[0-9]+$", codename), + sub("^alpine([0-9])([0-9]+)$", "alpine-\\1\\2", codename), + NA_character_ + ) + ) + ) +} + +# Structural lines only, so a codename mentioned in a comment is not a finding. +# Three shapes carry a slot: a matrix row (`- OS: alpine-323`), a continuation +# key (`CODENAME: alpine323`), and a dropdown option (`- alpine-323-amd64`). +slots_in_pipeline <- function(path) { + lines <- readLines(path, warn = FALSE) + hits <- character(0) + row <- regmatches( + lines, + regexec("^\\s*-?\\s*(OS|CODENAME|OS_ID):\\s*(\\S+)\\s*$", lines) + ) + hits <- c( + hits, + vapply(row, function(m) if (length(m) == 3L) m[[3L]] else NA_character_, "") + ) + opt <- regmatches( + lines, + regexec("^\\s*-\\s*([a-z0-9]+-[a-z0-9]+)-(amd64|arm64)\\s*$", lines) + ) + hits <- c( + hits, + vapply(opt, function(m) if (length(m) == 4L) m[[2L]] else NA_character_, "") + ) + hits <- hits[!is.na(hits)] + # `OS: alpine` (build-all-versions' bare image name) is not a slot. + unique(hits[grepl("^(alpine|redhat|ubuntu)-?[0-9]", hits)]) +} + +codenames <- published_codenames() +published_os <- unname(as_pipeline_os(codenames)) +if (anyNA(published_os)) { + stop( + sprintf( + "Unrecognised codename(s) in cdn.tf: %s", + paste(codenames[is.na(published_os)], collapse = ", ") + ), + call. = FALSE + ) +} +# A slot is written either way depending on the file, so accept both spellings. +published <- unique(c(published_os, codenames)) + +cat(sprintf( + "Published slots (cdn.tf): %s\n\n", + paste(sort(published_os), collapse = ", ") +)) + +files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE)) +findings <- list() +for (f in files) { + found <- slots_in_pipeline(f) + stale <- setdiff(found, published) + if (length(stale) > 0L) { + findings[[f]] <- stale + cat(sprintf( + "%-45s targets unpublished: %s\n", + f, + paste(sort(stale), collapse = ", ") + )) + } +} + +covered <- unique(unlist(lapply(files, slots_in_pipeline))) +uncovered <- setdiff(published_os, covered) +if (length(uncovered) > 0L) { + cat(sprintf( + "\nNote: %s appear(s) in no pipeline at all. That is a gap, not necessarily a bug.\n", + paste(sort(uncovered), collapse = ", ") + )) +} + +if (length(findings) == 0L) { + cat("All pipeline slots are published and routed.\n") + quit(status = 0) +} + +cat(sprintf( + "\n%d file(s) target a slot the edge does not route. Retire it with:\n just retire-slot \n", + length(findings) +)) +quit(status = 1) diff --git a/local/retire-slot.R b/local/retire-slot.R new file mode 100644 index 0000000..6c22157 --- /dev/null +++ b/local/retire-slot.R @@ -0,0 +1,267 @@ +### Retire a slot: strip it from the pipelines and delete its objects from S3. +# +# Usage: +# Rscript local/retire-slot.R [--apply] [--config-only|--assets-only] +# +# Rscript local/retire-slot.R alpine322 # dry run, both halves +# Rscript local/retire-slot.R alpine322 --config-only # edit .crow/, no S3 +# Rscript local/retire-slot.R alpine322 --assets-only --apply +# +# Retiring a distro retires both architectures, because `local.rpkgs_slots` in +# `cdn.tf` is a setproduct over `["amd64", "arm64"]` and there is no way to +# publish one without the other. +# +# Three things have to happen and nothing ties them together on its own: +# +# 1. the slot leaves `local.rpkgs_slots` in `cdn.tf` (and `DistroId` in the +# website's `src/src/data/repos.ts`), which is what stops the edge routing +# it. Do this first and by hand: it is the decision, the rest is cleanup. +# 2. the slot leaves every `.crow/*.yaml` matrix, which is what stops the +# fleet building, indexing and verifying it. This script does that. +# 3. the objects leave the bucket. This script does that too. +# +# A fourth step cannot be scripted from here: the slot's cron jobs live on the +# Crow server, not in this repository, and removing a matrix row does not remove +# the cron that targets it. The script prints the ones it finds by name at the +# end, with the API call to delete them. +# +# Dry run by default. Nothing is edited or deleted without --apply. +# +# Order matters. Deleting the objects while a pipeline still targets the slot +# means the next nightly rebuilds them. +# +# Only structural YAML is edited: matrix rows, dropdown options and the +# `OS_VERSION` tag. Prose that happens to mention the slot (a usage example in +# a header comment, a dropdown `description:`) is left alone, because rewriting +# free text automatically is guesswork. Grep for the name afterwards. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +args <- commandArgs(trailingOnly = TRUE) +apply_changes <- "--apply" %in% args +config_only <- "--config-only" %in% args +assets_only <- "--assets-only" %in% args +args <- args[!args %in% c("--apply", "--config-only", "--assets-only")] +if (length(args) != 1L || (config_only && assets_only)) { + stop( + "usage: retire-slot.R [--apply] [--config-only|--assets-only]", + call. = FALSE + ) +} +codename <- args[[1L]] + +# `alpine322` is how the bucket, the edge and `CODENAME:` spell it; `alpine-322` +# is how `OS:` and the dropdown options spell it. Both have to go. +pipeline_os <- if (grepl("^rhel[0-9]+$", codename)) { + sub("^rhel", "redhat-", codename) +} else if (grepl("^alpine[0-9]+$", codename)) { + sub("^alpine", "alpine-", codename) +} else { + # Single bracket, so an unknown codename yields NA and the check below + # reports it rather than raising "subscript out of bounds". + c(resolute = "ubuntu-2604", noble = "ubuntu-2404", jammy = "ubuntu-2204")[ + codename + ] +} +if (is.na(pipeline_os) || length(pipeline_os) == 0L) { + stop(sprintf("Unrecognised codename '%s'.", codename), call. = FALSE) +} + +# The `OS_VERSION` dropdowns are keyed on the image tag, not the slot name. +image_tag <- if (grepl("^alpine-", pipeline_os)) { + sub("^alpine-([0-9])([0-9]+)$", "\\1.\\2", pipeline_os) +} else if (grepl("^redhat-", pipeline_os)) { + sub("^redhat-", "", pipeline_os) +} else { + codename +} + +# Refuse to retire a slot that is still routed. Without this the script is a +# fast way to break a live slot: the edge would keep sending clients to indexes +# whose objects no longer exist, and every install would 404. +tf <- readLines("cdn.tf", warn = FALSE) +start <- grep("rpkgs_slots\\s*=", tf) +if (length(start) != 1L) { + stop( + "Could not find exactly one `rpkgs_slots` local in cdn.tf.", + call. = FALSE + ) +} +block <- paste(tf[start:min(start + 10L, length(tf))], collapse = "\n") +if (grepl(sprintf('"%s"', codename), block, fixed = TRUE)) { + stop( + sprintf( + "'%s' is still listed in `local.rpkgs_slots` in cdn.tf, so the edge still routes it. Remove it there (and from the website's DistroId union) and apply the OpenTofu change first.", # nolint + codename + ), + call. = FALSE + ) +} + +cat(sprintf( + "Retiring %s (pipelines call it '%s', images tag it '%s')\n\n", + codename, + pipeline_os, + image_tag +)) + +## 1. Pipeline definitions ---------------------------------------------------- + +# Drops a matrix row and everything indented under it, a dropdown option, and +# an `OS_VERSION` entry for the retired tag. Line-based on purpose: round +# tripping through a YAML parser would reflow all 18 files and bury the change. +strip_slot <- function(lines) { + keep <- rep(TRUE, length(lines)) + i <- 1L + while (i <= length(lines)) { + line <- lines[[i]] + row <- regexec( + sprintf( + "^(\\s*)-\\s*(OS|CODENAME):\\s*(%s|%s)\\s*$", + pipeline_os, + codename + ), + line + ) + m <- regmatches(line, row)[[1L]] + if (length(m) == 4L) { + indent <- nchar(m[[2L]]) + keep[[i]] <- FALSE + j <- i + 1L + while (j <= length(lines)) { + nxt <- lines[[j]] + if ( + !nzchar(trimws(nxt)) || nchar(sub("^(\\s*).*$", "\\1", nxt)) <= indent + ) { + break + } + keep[[j]] <- FALSE + j <- j + 1L + } + i <- j + next + } + if (grepl(sprintf("^\\s*-\\s*%s-(amd64|arm64)\\s*$", pipeline_os), line)) { + keep[[i]] <- FALSE + } + if (grepl(sprintf("^\\s*-\\s*'%s'\\s*$", image_tag), line)) { + keep[[i]] <- FALSE + } + i <- i + 1L + } + out <- lines[keep] + # A dropdown whose default was the retired tag would reject its own default, + # so move it to the first surviving option of the same block. + bad_default <- grepl(sprintf("^\\s*default:\\s*'%s'\\s*$", image_tag), out) + if (any(bad_default)) { + for (k in which(bad_default)) { + opts <- grep("^\\s*-\\s*'[^']+'\\s*$", out[seq_len(k - 1L)]) + if (length(opts) == 0L) { + next + } + # Walk back to the first option of this block rather than the last, so the + # default lands on the lowest surviving version instead of an unrelated one. + first <- max(opts) + while (first > 1L && grepl("^\\s*-\\s*'[^']+'\\s*$", out[[first - 1L]])) { + first <- first - 1L + } + replacement <- sub("^\\s*-\\s*'([^']+)'\\s*$", "\\1", out[[first]]) + out[[k]] <- sub("'[^']+'", sprintf("'%s'", replacement), out[[k]]) + cat(sprintf(" default moved to '%s'\n", replacement)) + } + } + out +} + +if (!assets_only) { + files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE)) + touched <- 0L + for (f in files) { + lines <- readLines(f, warn = FALSE) + out <- strip_slot(lines) + if (length(out) == length(lines)) { + next + } + touched <- touched + 1L + cat(sprintf( + " %s %-45s -%d lines\n", + if (apply_changes) "EDIT " else "would edit", + f, + length(lines) - length(out) + )) + if (apply_changes) { + writeLines(out, f) + } + } + if (touched == 0L) { + cat(" No pipeline references the slot.\n") + } else if (apply_changes) { + cat(sprintf( + " %d file(s) edited. Verify with `crow lint .crow/` and `Rscript local/check-slots.R`.\n", + touched + )) + } + cat("\n") +} + +## 2. Bucket objects ---------------------------------------------------------- + +if (!config_only) { + library(s3fs, quietly = TRUE) + if (!nzchar(Sys.getenv("B2_S3_ACCESS_KEY"))) { + stop( + "B2_S3_ACCESS_KEY is unset. B2 needs credentials even to list, and an unauthenticated list returns empty, which is indistinguishable from an already-clean slot.", # nolint + call. = FALSE + ) + } + bucket <- "devxy-rpkgs-binaries" + s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE + ) + + for (arch in c("amd64", "arm64")) { + prefix <- sprintf("%s/%s/%s", bucket, arch, codename) + if (!s3fs::s3_dir_exists(prefix)) { + cat(sprintf(" %s/%s: nothing in the bucket.\n", arch, codename)) + next + } + objects <- s3fs::s3_dir_ls(prefix, recurse = TRUE) + cat(sprintf( + " %s %s/%s: %d object(s)\n", + if (apply_changes) "DELETE" else "would delete", + arch, + codename, + length(objects) + )) + if (apply_changes && length(objects) > 0L) { + s3fs::s3_dir_delete(prefix) + cat(sprintf(" deleted s3://%s\n", prefix)) + } + } + cat("\n") +} + +## 3. What is left for a human ------------------------------------------------ + +if (!apply_changes) { + cat("Dry run: nothing was edited or deleted. Re-run with --apply.\n\n") +} + +cat("Crons still to remove on the Crow server (they are not in this repo):\n") +cat(sprintf( + " crow-at ci.devxy.io repo cron ls devxy/build-cran-binaries | grep %s\n", + pipeline_os +)) +cat( + " # `crow repo cron rm` 404s on this server version; use the API, whose base is /api/v1\n" +) +cat(" eval \"$(crow-at --export ci.devxy.io)\"\n") +cat(" curl -s -X DELETE -H \"Authorization: Bearer $CROW_TOKEN\" \\\n") +cat(" https://ci.devxy.io/api/v1/repos/5/cron/\n") -- 2.54.0 From cc9d7b60464111ce4912337f03f2b486e2c71933 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 11 Sep 2026 08:21:23 +0000 Subject: [PATCH 2/2] refactor(slots): cut the comments down to the essentials Addresses review on #214. This change will: - Trim the header comments of `local/check-slots.R`, `local/retire-slot.R` and `.crow/retire-slot.yaml` to what a reader needs, dropping the narrative about how Alpine 3.22 got into that state. - Reduce the README section to the ordered steps and the two constraints that matter. Behaviour is unchanged: against the tree at 523424b the script still reproduces #213's structural changes, differing only in the same prose-only file, and both guards still fire. --- .crow/retire-slot.yaml | 23 +++-------- README.md | 34 +++++----------- local/check-slots.R | 89 +++++++++++++----------------------------- local/retire-slot.R | 65 +++++++++--------------------- 4 files changed, 63 insertions(+), 148 deletions(-) diff --git a/.crow/retire-slot.yaml b/.crow/retire-slot.yaml index 323a00e..1b62ba7 100644 --- a/.crow/retire-slot.yaml +++ b/.crow/retire-slot.yaml @@ -1,27 +1,16 @@ # Delete a retired slot's objects from the bucket. # -# The config half of a retirement (`.crow/*.yaml` matrices) is a commit, so it -# happens on a workstation: +# The config half is a commit, so it runs on a workstation: +# just retire-slot --apply # -# Rscript local/retire-slot.R --config-only --apply +# This half needs the B2 credentials, which only exist in CI. +# `local/retire-slot.R` refuses any slot still listed in `local.rpkgs_slots`, +# so this cannot delete a live slot. Both architectures go together. # -# The asset half needs the B2 credentials, which only exist in CI, so it -# happens here. `local/retire-slot.R` refuses any slot still listed in -# `local.rpkgs_slots` in cdn.tf, so this pipeline cannot delete a live slot -# however it is invoked. -# -# Dry run unless RETIRE_APPLY is 'yes'. The dry run reports the object count, -# which is the number worth looking at before deleting a five-figure prefix. +# Dry run unless RETIRE_APPLY is 'yes': # # crow pipeline create devxy/build-cran-binaries --branch main \ # --var retire_slot=alpine322 --var RETIRE_APPLY=yes -# -# Retiring a distro retires both architectures: `local.rpkgs_slots` is a -# setproduct over both, so there is no way to publish one without the other, -# and the script deletes `amd64//` and `arm64//` together. -# -# The slot's cron jobs are not in this repository and are not removed here. The -# script prints them by name, with the API call, at the end of the run. variables: # A manual pipeline creation instantiates every file in .crow/, so the diff --git a/README.md b/README.md index 1b63e59..d137728 100644 --- a/README.md +++ b/README.md @@ -200,39 +200,25 @@ Technically, no date-based snapshots are planned, so this component from the Pos ## Retiring a slot -A slot is only reachable by clients if it appears in `local.rpkgs_slots` in `cdn.tf`. -That list feeds the `UNION_SLOTS` variable and `edge/rpkgs-router.ts` gates all per-R-minor index rewriting on it. -A slot missing from it is not merely unadvertised, it is broken: every request falls through to the flat index, which holds a single build per package, so clients on any R minor other than the one the slot was built against die at `dyn.load()`. +A slot is only reachable if it is listed in `local.rpkgs_slots` in `cdn.tf`, which feeds `UNION_SLOTS` in `edge/rpkgs-router.ts`. +An unrouted slot serves the flat index to every client, so its binaries fail at `dyn.load()` under every R minor but one. -Alpine 3.22 sat in that state for months, built nightly and never routed. -Retire a slot in this order: +In order: 1. Remove it from `local.rpkgs_slots` in `cdn.tf` and from `DistroId` in the website's `src/src/data/repos.ts`, then `tofu apply`. - This is the decision; everything below is cleanup. -2. Strip it from the pipeline matrices and commit the result: - - ```sh - # dry run - just retire-slot alpine322 - # write the changes - just retire-slot alpine322 --apply - ``` - -3. Delete its cron jobs on the Crow server. - They are not in this repository, and removing a matrix row does not remove the cron that targets it. - `local/retire-slot.R` prints the ones it finds, with the API call. -4. Delete its objects from the bucket, which needs the B2 credentials and therefore runs in CI: +2. `just retire-slot --apply`, then commit. +3. Delete its cron jobs on the Crow server; `local/retire-slot.R` prints them with the API call. +4. Delete its objects: ```sh crow pipeline create devxy/build-cran-binaries --branch main \ - --var retire_slot=alpine322 --var RETIRE_APPLY=yes + --var retire_slot= --var RETIRE_APPLY=yes ``` -`local/retire-slot.R` refuses any slot still listed in `cdn.tf`, so steps 2 and 4 cannot run against a live slot. -Doing step 4 before step 2 is the one ordering that actively breaks things: the next nightly would rebuild everything just deleted. +Steps 2 and 4 refuse any slot still listed in `cdn.tf`. +Running step 4 before step 2 means the next nightly rebuilds everything just deleted. -`just check-slots` fails when any pipeline still targets a slot the edge does not route, which is the drift that put Alpine 3.22 where it was. -It is one-directional on purpose: a published slot missing from _some_ pipeline is only reported, because several pipelines cover a subset deliberately. +`just check-slots` fails when a pipeline still targets an unrouted slot. ## Helpers diff --git a/local/check-slots.R b/local/check-slots.R index 22e3a9a..5345da9 100644 --- a/local/check-slots.R +++ b/local/check-slots.R @@ -1,47 +1,20 @@ -### Fail when a pipeline still targets a slot the edge does not route. +### Fail when a pipeline targets a slot that is not in `local.rpkgs_slots`. # -# Usage: # Rscript local/check-slots.R # -# A slot is only reachable by clients if it appears in `local.rpkgs_slots` in -# `cdn.tf`. That list feeds the `UNION_SLOTS` variable, and -# `edge/rpkgs-router.ts` gates all per-R-minor index rewriting on it. A slot -# missing from it is not merely unadvertised: every request falls through to -# the flat index, which holds a single build per package, so clients on any R -# minor other than the one the slot was built against die at dyn.load(). +# That list feeds `UNION_SLOTS`, which `edge/rpkgs-router.ts` gates per-R-minor +# routing on. An unrouted slot serves the flat index to every client, so its +# binaries fail at dyn.load() under every R minor but one. # -# Alpine 3.22 sat in that state for months. It was built nightly, its per-minor -# indexes were correct, and it was never routed. `verify_routing=all` found it -# only because the run happened to include it: 1056 of 1056 checks passed on -# the 16 published slots and every failure in the run was alpine322, on both -# architectures, identical across all three URL forms. -# -# The check is deliberately one-directional. A pipeline targeting an unpublished -# slot is an error, because it spends build capacity on binaries nobody can -# reach and turns `verify_routing=all` permanently red for a reason that is not -# a defect. A published slot missing from *some* pipeline is only reported: -# several pipelines cover a subset on purpose (archive-missed-packages has no -# ubuntu rows, weekly-rebuild-missing has no alpine-324 rows), and the coverage -# a pipeline needs is its own business. -# -# The website's `DistroId` union in `src/src/data/repos.ts` must agree with -# `cdn.tf` too, but it lives in another repository so it cannot be checked here. +# One-directional: a published slot missing from some pipeline is only +# reported, since several pipelines cover a subset on purpose. options(error = function() { cat("ERROR:", geterrmessage(), "\n", file = stdout()) q(status = 1) }) -# `cdn.tf` is the source of truth, so read it rather than restating the list. -# The shape this depends on is the second string array inside the -# `rpkgs_slots` local, i.e. the distro list of the setproduct: -# -# rpkgs_slots = [ -# for pair in setproduct( -# ["amd64", "arm64"], -# ["resolute", "noble", ...] -# ) : "${pair[0]}/${pair[1]}" -# ] +# Parses the distro list out of the setproduct's second array. published_codenames <- function(path = "cdn.tf") { if (!file.exists(path)) { stop( @@ -59,27 +32,23 @@ published_codenames <- function(path = "cdn.tf") { } block <- paste(lines[start:min(start + 10L, length(lines))], collapse = "\n") arrays <- regmatches(block, gregexpr("\\[[^]]*\\]", block))[[1L]] - # The first array is the setproduct's arch list, the second is the distros. if (length(arrays) < 2L) { stop( - "The `rpkgs_slots` local no longer has the two-array setproduct shape this parser expects. Update local/check-slots.R alongside cdn.tf.", # nolint + "`rpkgs_slots` no longer has the two-array setproduct shape this parses.", call. = FALSE ) } - codenames <- regmatches(arrays[[2L]], gregexpr('"[^"]+"', arrays[[2L]]))[[1L]] - codenames <- gsub('"', "", codenames) + codenames <- gsub( + '"', + "", + regmatches(arrays[[2L]], gregexpr('"[^"]+"', arrays[[2L]]))[[1L]] + ) if (length(codenames) == 0L) { - stop( - "Parsed an empty distro list from cdn.tf; refusing to report every pipeline slot as unpublished.", - call. = FALSE - ) + stop("Parsed an empty distro list from cdn.tf.", call. = FALSE) } codenames } -# Pipelines name a slot `-`, the edge and the bucket name it by -# codename. The ubuntu half of this mapping is also in `edge/rpkgs-router.ts` -# (UBUNTU_CODENAMES) and `local/build-all.R`. UBUNTU_CODENAMES <- c( resolute = "ubuntu-2604", noble = "ubuntu-2404", @@ -102,26 +71,23 @@ as_pipeline_os <- function(codename) { ) } -# Structural lines only, so a codename mentioned in a comment is not a finding. -# Three shapes carry a slot: a matrix row (`- OS: alpine-323`), a continuation -# key (`CODENAME: alpine323`), and a dropdown option (`- alpine-323-amd64`). +# Structural lines only, so a codename in a comment is not a finding. slots_in_pipeline <- function(path) { lines <- readLines(path, warn = FALSE) - hits <- character(0) row <- regmatches( lines, regexec("^\\s*-?\\s*(OS|CODENAME|OS_ID):\\s*(\\S+)\\s*$", lines) ) - hits <- c( - hits, - vapply(row, function(m) if (length(m) == 3L) m[[3L]] else NA_character_, "") - ) opt <- regmatches( lines, regexec("^\\s*-\\s*([a-z0-9]+-[a-z0-9]+)-(amd64|arm64)\\s*$", lines) ) hits <- c( - hits, + vapply( + row, + function(m) if (length(m) == 3L) m[[3L]] else NA_character_, + "" + ), vapply(opt, function(m) if (length(m) == 4L) m[[2L]] else NA_character_, "") ) hits <- hits[!is.na(hits)] @@ -140,7 +106,7 @@ if (anyNA(published_os)) { call. = FALSE ) } -# A slot is written either way depending on the file, so accept both spellings. +# Files spell a slot either way, so accept both. published <- unique(c(published_os, codenames)) cat(sprintf( @@ -151,8 +117,7 @@ cat(sprintf( files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE)) findings <- list() for (f in files) { - found <- slots_in_pipeline(f) - stale <- setdiff(found, published) + stale <- setdiff(slots_in_pipeline(f), published) if (length(stale) > 0L) { findings[[f]] <- stale cat(sprintf( @@ -163,11 +128,13 @@ for (f in files) { } } -covered <- unique(unlist(lapply(files, slots_in_pipeline))) -uncovered <- setdiff(published_os, covered) +uncovered <- setdiff( + published_os, + unique(unlist(lapply(files, slots_in_pipeline))) +) if (length(uncovered) > 0L) { cat(sprintf( - "\nNote: %s appear(s) in no pipeline at all. That is a gap, not necessarily a bug.\n", + "\nNote: %s appear(s) in no pipeline at all.\n", paste(sort(uncovered), collapse = ", ") )) } @@ -178,7 +145,7 @@ if (length(findings) == 0L) { } cat(sprintf( - "\n%d file(s) target a slot the edge does not route. Retire it with:\n just retire-slot \n", + "\n%d file(s) target an unrouted slot. Retire it with:\n just retire-slot \n", length(findings) )) quit(status = 1) diff --git a/local/retire-slot.R b/local/retire-slot.R index 6c22157..383fd2c 100644 --- a/local/retire-slot.R +++ b/local/retire-slot.R @@ -1,39 +1,20 @@ ### Retire a slot: strip it from the pipelines and delete its objects from S3. # -# Usage: # Rscript local/retire-slot.R [--apply] [--config-only|--assets-only] # -# Rscript local/retire-slot.R alpine322 # dry run, both halves -# Rscript local/retire-slot.R alpine322 --config-only # edit .crow/, no S3 -# Rscript local/retire-slot.R alpine322 --assets-only --apply -# # Retiring a distro retires both architectures, because `local.rpkgs_slots` in -# `cdn.tf` is a setproduct over `["amd64", "arm64"]` and there is no way to -# publish one without the other. +# `cdn.tf` is a setproduct over them. # -# Three things have to happen and nothing ties them together on its own: +# Order: remove the slot from `local.rpkgs_slots` (and the website's +# `DistroId`) and apply that first, then run this. Deleting objects while a +# pipeline still targets the slot just means the next nightly rebuilds them. # -# 1. the slot leaves `local.rpkgs_slots` in `cdn.tf` (and `DistroId` in the -# website's `src/src/data/repos.ts`), which is what stops the edge routing -# it. Do this first and by hand: it is the decision, the rest is cleanup. -# 2. the slot leaves every `.crow/*.yaml` matrix, which is what stops the -# fleet building, indexing and verifying it. This script does that. -# 3. the objects leave the bucket. This script does that too. +# Cron jobs live on the Crow server, not here, and are printed at the end. # -# A fourth step cannot be scripted from here: the slot's cron jobs live on the -# Crow server, not in this repository, and removing a matrix row does not remove -# the cron that targets it. The script prints the ones it finds by name at the -# end, with the API call to delete them. +# Only structural YAML is edited. Prose mentioning the slot is left alone, so +# grep for the name afterwards. # -# Dry run by default. Nothing is edited or deleted without --apply. -# -# Order matters. Deleting the objects while a pipeline still targets the slot -# means the next nightly rebuilds them. -# -# Only structural YAML is edited: matrix rows, dropdown options and the -# `OS_VERSION` tag. Prose that happens to mention the slot (a usage example in -# a header comment, a dropdown `description:`) is left alone, because rewriting -# free text automatically is guesswork. Grep for the name afterwards. +# Dry run by default. options(error = function() { cat("ERROR:", geterrmessage(), "\n", file = stdout()) @@ -53,15 +34,14 @@ if (length(args) != 1L || (config_only && assets_only)) { } codename <- args[[1L]] -# `alpine322` is how the bucket, the edge and `CODENAME:` spell it; `alpine-322` -# is how `OS:` and the dropdown options spell it. Both have to go. +# The bucket and `CODENAME:` say `alpine322`; `OS:` and the options say +# `alpine-322`. Both have to go. pipeline_os <- if (grepl("^rhel[0-9]+$", codename)) { sub("^rhel", "redhat-", codename) } else if (grepl("^alpine[0-9]+$", codename)) { sub("^alpine", "alpine-", codename) } else { - # Single bracket, so an unknown codename yields NA and the check below - # reports it rather than raising "subscript out of bounds". + # Single bracket so an unknown codename yields NA, not an error. c(resolute = "ubuntu-2604", noble = "ubuntu-2404", jammy = "ubuntu-2204")[ codename ] @@ -79,9 +59,8 @@ image_tag <- if (grepl("^alpine-", pipeline_os)) { codename } -# Refuse to retire a slot that is still routed. Without this the script is a -# fast way to break a live slot: the edge would keep sending clients to indexes -# whose objects no longer exist, and every install would 404. +# Refuse a slot that is still routed: deleting its objects would leave the edge +# serving indexes that name files which no longer exist. tf <- readLines("cdn.tf", warn = FALSE) start <- grep("rpkgs_slots\\s*=", tf) if (length(start) != 1L) { @@ -110,9 +89,7 @@ cat(sprintf( ## 1. Pipeline definitions ---------------------------------------------------- -# Drops a matrix row and everything indented under it, a dropdown option, and -# an `OS_VERSION` entry for the retired tag. Line-based on purpose: round -# tripping through a YAML parser would reflow all 18 files and bury the change. +# Line-based on purpose: a YAML round trip would reflow all 18 files. strip_slot <- function(lines) { keep <- rep(TRUE, length(lines)) i <- 1L @@ -153,8 +130,7 @@ strip_slot <- function(lines) { i <- i + 1L } out <- lines[keep] - # A dropdown whose default was the retired tag would reject its own default, - # so move it to the first surviving option of the same block. + # A dropdown cannot default to an option that no longer exists. bad_default <- grepl(sprintf("^\\s*default:\\s*'%s'\\s*$", image_tag), out) if (any(bad_default)) { for (k in which(bad_default)) { @@ -162,8 +138,7 @@ strip_slot <- function(lines) { if (length(opts) == 0L) { next } - # Walk back to the first option of this block rather than the last, so the - # default lands on the lowest surviving version instead of an unrelated one. + # Walk back to this block's first option, not the nearest one. first <- max(opts) while (first > 1L && grepl("^\\s*-\\s*'[^']+'\\s*$", out[[first - 1L]])) { first <- first - 1L @@ -213,7 +188,7 @@ if (!config_only) { library(s3fs, quietly = TRUE) if (!nzchar(Sys.getenv("B2_S3_ACCESS_KEY"))) { stop( - "B2_S3_ACCESS_KEY is unset. B2 needs credentials even to list, and an unauthenticated list returns empty, which is indistinguishable from an already-clean slot.", # nolint + "B2_S3_ACCESS_KEY is unset. B2 lists nothing without credentials, which is indistinguishable from an already-clean slot.", # nolint call. = FALSE ) } @@ -248,7 +223,7 @@ if (!config_only) { cat("\n") } -## 3. What is left for a human ------------------------------------------------ +## 3. Crons ------------------------------------------------------------------ if (!apply_changes) { cat("Dry run: nothing was edited or deleted. Re-run with --apply.\n\n") @@ -259,9 +234,7 @@ cat(sprintf( " crow-at ci.devxy.io repo cron ls devxy/build-cran-binaries | grep %s\n", pipeline_os )) -cat( - " # `crow repo cron rm` 404s on this server version; use the API, whose base is /api/v1\n" -) +cat(" # `crow repo cron rm` 404s here; the API base is /api/v1\n") cat(" eval \"$(crow-at --export ci.devxy.io)\"\n") cat(" curl -s -X DELETE -H \"Authorization: Bearer $CROW_TOKEN\" \\\n") cat(" https://ci.devxy.io/api/v1/repos/5/cron/\n") -- 2.54.0