### Retire a slot: strip it from the pipelines and delete its objects from S3. # # Usage: # Rscript local/retire-slot.R [--apply] [--config-only|--assets-only] # # Rscript local/retire-slot.R alpine322 # dry run, both halves # Rscript local/retire-slot.R alpine322 --config-only # edit .crow/, no S3 # Rscript local/retire-slot.R alpine322 --assets-only --apply # # Retiring a distro retires both architectures, because `local.rpkgs_slots` in # `cdn.tf` is a setproduct over `["amd64", "arm64"]` and there is no way to # publish one without the other. # # Three things have to happen and nothing ties them together on its own: # # 1. the slot leaves `local.rpkgs_slots` in `cdn.tf` (and `DistroId` in the # website's `src/src/data/repos.ts`), which is what stops the edge routing # it. Do this first and by hand: it is the decision, the rest is cleanup. # 2. the slot leaves every `.crow/*.yaml` matrix, which is what stops the # fleet building, indexing and verifying it. This script does that. # 3. the objects leave the bucket. This script does that too. # # A fourth step cannot be scripted from here: the slot's cron jobs live on the # Crow server, not in this repository, and removing a matrix row does not remove # the cron that targets it. The script prints the ones it finds by name at the # end, with the API call to delete them. # # Dry run by default. Nothing is edited or deleted without --apply. # # Order matters. Deleting the objects while a pipeline still targets the slot # means the next nightly rebuilds them. # # Only structural YAML is edited: matrix rows, dropdown options and the # `OS_VERSION` tag. Prose that happens to mention the slot (a usage example in # a header comment, a dropdown `description:`) is left alone, because rewriting # free text automatically is guesswork. Grep for the name afterwards. options(error = function() { cat("ERROR:", geterrmessage(), "\n", file = stdout()) q(status = 1) }) args <- commandArgs(trailingOnly = TRUE) apply_changes <- "--apply" %in% args config_only <- "--config-only" %in% args assets_only <- "--assets-only" %in% args args <- args[!args %in% c("--apply", "--config-only", "--assets-only")] if (length(args) != 1L || (config_only && assets_only)) { stop( "usage: retire-slot.R [--apply] [--config-only|--assets-only]", call. = FALSE ) } codename <- args[[1L]] # `alpine322` is how the bucket, the edge and `CODENAME:` spell it; `alpine-322` # is how `OS:` and the dropdown options spell it. Both have to go. pipeline_os <- if (grepl("^rhel[0-9]+$", codename)) { sub("^rhel", "redhat-", codename) } else if (grepl("^alpine[0-9]+$", codename)) { sub("^alpine", "alpine-", codename) } else { # Single bracket, so an unknown codename yields NA and the check below # reports it rather than raising "subscript out of bounds". c(resolute = "ubuntu-2604", noble = "ubuntu-2404", jammy = "ubuntu-2204")[ codename ] } if (is.na(pipeline_os) || length(pipeline_os) == 0L) { stop(sprintf("Unrecognised codename '%s'.", codename), call. = FALSE) } # The `OS_VERSION` dropdowns are keyed on the image tag, not the slot name. image_tag <- if (grepl("^alpine-", pipeline_os)) { sub("^alpine-([0-9])([0-9]+)$", "\\1.\\2", pipeline_os) } else if (grepl("^redhat-", pipeline_os)) { sub("^redhat-", "", pipeline_os) } else { codename } # Refuse to retire a slot that is still routed. Without this the script is a # fast way to break a live slot: the edge would keep sending clients to indexes # whose objects no longer exist, and every install would 404. tf <- readLines("cdn.tf", warn = FALSE) start <- grep("rpkgs_slots\\s*=", tf) if (length(start) != 1L) { stop( "Could not find exactly one `rpkgs_slots` local in cdn.tf.", call. = FALSE ) } block <- paste(tf[start:min(start + 10L, length(tf))], collapse = "\n") if (grepl(sprintf('"%s"', codename), block, fixed = TRUE)) { stop( sprintf( "'%s' is still listed in `local.rpkgs_slots` in cdn.tf, so the edge still routes it. Remove it there (and from the website's DistroId union) and apply the OpenTofu change first.", # nolint codename ), call. = FALSE ) } cat(sprintf( "Retiring %s (pipelines call it '%s', images tag it '%s')\n\n", codename, pipeline_os, image_tag )) ## 1. Pipeline definitions ---------------------------------------------------- # Drops a matrix row and everything indented under it, a dropdown option, and # an `OS_VERSION` entry for the retired tag. Line-based on purpose: round # tripping through a YAML parser would reflow all 18 files and bury the change. strip_slot <- function(lines) { keep <- rep(TRUE, length(lines)) i <- 1L while (i <= length(lines)) { line <- lines[[i]] row <- regexec( sprintf( "^(\\s*)-\\s*(OS|CODENAME):\\s*(%s|%s)\\s*$", pipeline_os, codename ), line ) m <- regmatches(line, row)[[1L]] if (length(m) == 4L) { indent <- nchar(m[[2L]]) keep[[i]] <- FALSE j <- i + 1L while (j <= length(lines)) { nxt <- lines[[j]] if ( !nzchar(trimws(nxt)) || nchar(sub("^(\\s*).*$", "\\1", nxt)) <= indent ) { break } keep[[j]] <- FALSE j <- j + 1L } i <- j next } if (grepl(sprintf("^\\s*-\\s*%s-(amd64|arm64)\\s*$", pipeline_os), line)) { keep[[i]] <- FALSE } if (grepl(sprintf("^\\s*-\\s*'%s'\\s*$", image_tag), line)) { keep[[i]] <- FALSE } i <- i + 1L } out <- lines[keep] # A dropdown whose default was the retired tag would reject its own default, # so move it to the first surviving option of the same block. bad_default <- grepl(sprintf("^\\s*default:\\s*'%s'\\s*$", image_tag), out) if (any(bad_default)) { for (k in which(bad_default)) { opts <- grep("^\\s*-\\s*'[^']+'\\s*$", out[seq_len(k - 1L)]) if (length(opts) == 0L) { next } # Walk back to the first option of this block rather than the last, so the # default lands on the lowest surviving version instead of an unrelated one. first <- max(opts) while (first > 1L && grepl("^\\s*-\\s*'[^']+'\\s*$", out[[first - 1L]])) { first <- first - 1L } replacement <- sub("^\\s*-\\s*'([^']+)'\\s*$", "\\1", out[[first]]) out[[k]] <- sub("'[^']+'", sprintf("'%s'", replacement), out[[k]]) cat(sprintf(" default moved to '%s'\n", replacement)) } } out } if (!assets_only) { files <- sort(list.files(".crow", pattern = "\\.yaml$", full.names = TRUE)) touched <- 0L for (f in files) { lines <- readLines(f, warn = FALSE) out <- strip_slot(lines) if (length(out) == length(lines)) { next } touched <- touched + 1L cat(sprintf( " %s %-45s -%d lines\n", if (apply_changes) "EDIT " else "would edit", f, length(lines) - length(out) )) if (apply_changes) { writeLines(out, f) } } if (touched == 0L) { cat(" No pipeline references the slot.\n") } else if (apply_changes) { cat(sprintf( " %d file(s) edited. Verify with `crow lint .crow/` and `Rscript local/check-slots.R`.\n", touched )) } cat("\n") } ## 2. Bucket objects ---------------------------------------------------------- if (!config_only) { library(s3fs, quietly = TRUE) if (!nzchar(Sys.getenv("B2_S3_ACCESS_KEY"))) { stop( "B2_S3_ACCESS_KEY is unset. B2 needs credentials even to list, and an unauthenticated list returns empty, which is indistinguishable from an already-clean slot.", # nolint call. = FALSE ) } bucket <- "devxy-rpkgs-binaries" s3fs::s3_file_system( aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), endpoint = "https://s3.eu-central-003.backblazeb2.com", region_name = "eu-central-003", refresh = TRUE ) for (arch in c("amd64", "arm64")) { prefix <- sprintf("%s/%s/%s", bucket, arch, codename) if (!s3fs::s3_dir_exists(prefix)) { cat(sprintf(" %s/%s: nothing in the bucket.\n", arch, codename)) next } objects <- s3fs::s3_dir_ls(prefix, recurse = TRUE) cat(sprintf( " %s %s/%s: %d object(s)\n", if (apply_changes) "DELETE" else "would delete", arch, codename, length(objects) )) if (apply_changes && length(objects) > 0L) { s3fs::s3_dir_delete(prefix) cat(sprintf(" deleted s3://%s\n", prefix)) } } cat("\n") } ## 3. What is left for a human ------------------------------------------------ if (!apply_changes) { cat("Dry run: nothing was edited or deleted. Re-run with --apply.\n\n") } cat("Crons still to remove on the Crow server (they are not in this repo):\n") cat(sprintf( " crow-at ci.devxy.io repo cron ls devxy/build-cran-binaries | grep %s\n", pipeline_os )) cat( " # `crow repo cron rm` 404s on this server version; use the API, whose base is /api/v1\n" ) cat(" eval \"$(crow-at --export ci.devxy.io)\"\n") cat(" curl -s -X DELETE -H \"Authorization: Bearer $CROW_TOKEN\" \\\n") cat(" https://ci.devxy.io/api/v1/repos/5/cron/\n")