diff --git a/.crow/prune-stale-flat-binaries.yaml b/.crow/prune-stale-flat-binaries.yaml new file mode 100644 index 0000000..a759daf --- /dev/null +++ b/.crow/prune-stale-flat-binaries.yaml @@ -0,0 +1,245 @@ +# Remove flat-slot binaries belonging to ABI-risky packages, then re-index. +# +# `upload_single_binary()` writes an ABI-risky package only to the per-minor +# slot, so a risky package should have no flat binary at all. Packages built +# before they were classified risky still do, and the flat slot is what every +# client that cannot be routed by R minor receives. That binary was built under +# whichever R was primary at the time, so it dies at load under any minor that +# dropped a symbol it uses: +# +# base64enc.so: undefined symbol: SETLENGTH +# +# Deleting the flat object does not create a hole. The source fallback +# republishes the CRAN source under the same key, so the package stays +# installable everywhere and compiles against the client's own R, while routed +# clients keep getting the per-minor binary. +# +# Dry run unless PRUNE_APPLY is 'yes'. Start with an explicit package list: +# +# crow pipeline create devxy/build-cran-binaries \ +# --var prune_flat=ubuntu-2604-amd64 \ +# --var PRUNE_PACKAGES="base64enc lobstr" \ +# --var PRUNE_APPLY=yes + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the default + # must match no matrix row. + prune_flat: + description: "Prune target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + PRUNE_PACKAGES: + description: 'Space-separated package names. Empty means every package the classifier cache calls risky.' + default: '' + PRUNE_APPLY: + description: "Set to 'yes' to actually delete. Anything else is a dry run." + options: + - 'no' + - 'yes' + default: 'no' + +when: + - event: manual + evaluate: 'prune_flat == "all" || prune_flat == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + CODENAME: alpine322 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + CODENAME: alpine322 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + CODENAME: alpine323 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + CODENAME: alpine323 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + CODENAME: alpine324 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + CODENAME: alpine324 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + CODENAME: rhel8 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + CODENAME: rhel8 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + CODENAME: rhel9 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + CODENAME: rhel9 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + CODENAME: rhel10 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + CODENAME: rhel10 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + CODENAME: jammy + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + CODENAME: jammy + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + CODENAME: noble + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + CODENAME: noble + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + CODENAME: resolute + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + CODENAME: resolute + IMG: ubuntu:resolute + +steps: + - name: 'Prune stale flat binaries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /tmp/R-pkgs + # Pipeline variables are substituted into the YAML, not exported, so they + # have to be bound here to be readable as env vars in the script below. + PRUNE_ARCH: ${ARCH} + PRUNE_CODENAME: ${CODENAME} + PRUNE_PACKAGES: ${PRUNE_PACKAGES} + PRUNE_APPLY: ${PRUNE_APPLY} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /tmp/R-pkgs + - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R + - | + set -eu + APPLY="" + if [ "$$PRUNE_APPLY" = "yes" ]; then + APPLY="--apply" + fi + /opt/R/4.5.3/bin/Rscript local/prune-stale-flat-binaries.R \ + "$$PRUNE_ARCH" "$$PRUNE_CODENAME" $$APPLY $$PRUNE_PACKAGES + backend_options: + docker: + resources: + requests: + memory: 3Gi + cpu: 1000m + + # Deleting objects without re-indexing is worse than leaving them: the + # published index keeps naming files that no longer exist, so every client + # gets a 404 instead of a working source fallback. The re-index is what + # republishes them as CRAN source, so it belongs in the same run. + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + R_LIBS_USER: /tmp/R-pkgs + PRUNE_APPLY: ${PRUNE_APPLY} + commands: + - | + set -eu + if [ "$$PRUNE_APPLY" != "yes" ]; then + echo "Dry run: nothing was deleted, so there is nothing to re-index." + exit 0 + fi + /opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + backend_options: + docker: + resources: + requests: + memory: 3Gi + cpu: 1000m + + # The edge caches indexes for ~370 days, so without this the deletion is + # invisible to clients and they keep being handed the removed binary. + - name: 'Purge CDN cache' + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + # Same zones as reindex.yaml: the production pull zone id, the alliance + # hostname and the staging zone, which shares the B2 origin. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' + PRUNE_APPLY: ${PRUNE_APPLY} + commands: + - | + set -eu + if [ "$$PRUNE_APPLY" != "yes" ]; then + echo "Dry run: nothing changed at the origin, so no purge is needed." + exit 0 + fi + apk add --no-cache -q bash curl jq + bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES diff --git a/local/prune-stale-flat-binaries.R b/local/prune-stale-flat-binaries.R new file mode 100644 index 0000000..dbb0ab1 --- /dev/null +++ b/local/prune-stale-flat-binaries.R @@ -0,0 +1,136 @@ +### Remove flat-slot binaries of packages that are ABI-risky. +# +# Usage: +# Rscript local/prune-stale-flat-binaries.R [--apply] [pkg ...] +# +# `upload_single_binary()` writes an ABI-risky package only to the per-minor +# slot, never to the flat one, so a risky package should have no flat binary at +# all. Packages built before they were classified risky still do, and the flat +# slot is what every client that cannot be routed by R minor receives. That +# binary was built under whichever R was primary at the time, so it fails at +# load under any minor that dropped a symbol it uses: +# +# base64enc.so: undefined symbol: SETLENGTH +# +# Deleting the flat object does not create a hole: bincraft's source fallback +# publishes the CRAN source under the same key, so the package stays installable +# everywhere and compiles against the client's own R. Clients that are routed +# keep getting the per-minor binary. +# +# Dry run by default. Nothing is deleted without --apply. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +args <- commandArgs(trailingOnly = TRUE) +apply_changes <- "--apply" %in% args +args <- args[args != "--apply"] +if (length(args) < 2L) { + stop( + "usage: prune-stale-flat-binaries.R [--apply] [pkg ...]", + call. = FALSE + ) +} +arch <- args[[1L]] +codename <- args[[2L]] +explicit <- args[-c(1L, 2L)] + +bucket <- "devxy-rpkgs-binaries" +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) + +contrib <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename) +objects <- s3fs::s3_dir_ls(contrib, recurse = TRUE) +relative <- sub(sprintf("^(s3://)?%s/", contrib), "", objects) + +is_tarball <- grepl("\\.tar\\.gz$", relative) +# Flat objects only: no `/` prefix and not under Archive/. +is_flat <- is_tarball & !grepl("/", relative, fixed = TRUE) +flat_files <- relative[is_flat] +flat_pkg <- sub("_.*$", "", flat_files) + +if (length(explicit) > 0L) { + risky <- explicit + cat(sprintf("Restricted to %d named package(s)\n", length(risky))) +} else { + risky <- bincraft:::abi_cache_risky_packages() + cat(sprintf("Classifier cache reports %d risky package(s)\n", length(risky))) + if (length(risky) == 0L) { + stop( + "The classifier cache returned nothing. Refusing to proceed: an empty risky set here is indistinguishable from an unreachable cache, and would silently prune nothing while reporting success.", # nolint + call. = FALSE + ) + } +} + +doomed <- flat_files[flat_pkg %in% risky] +cat(sprintf( + "%s/%s: %d flat binaries, %d belong to ABI-risky packages\n", + arch, + codename, + length(flat_files), + length(doomed) +)) + +# Never remove the last artifact of a package. The source fallback republishes +# the CRAN source under this key, but only for packages CRAN still ships, so +# confirm the source exists before deleting the binary. +cran <- tryCatch( + { + con <- gzcon(url( + "https://cloud.r-project.org/src/contrib/PACKAGES.gz", + open = "rb" + )) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = "Package")[, "Package"] + }, + error = function(e) character(0) +) +if (length(cran) == 0L) { + stop("Could not read CRAN's index; refusing to prune blind.", call. = FALSE) +} + +orphaned <- doomed[!(sub("_.*$", "", doomed) %in% cran)] +if (length(orphaned) > 0L) { + cat(sprintf( + "Keeping %d object(s) whose package is no longer on CRAN, so there is no source to fall back to: %s\n", + length(orphaned), + paste(utils::head(orphaned, 5L), collapse = ", ") + )) + doomed <- setdiff(doomed, orphaned) +} + +if (length(doomed) == 0L) { + cat("Nothing to prune.\n") + quit(status = 0) +} + +for (f in doomed) { + cat(sprintf(" %s %s\n", if (apply_changes) "DELETE" else "would delete", f)) +} + +if (!apply_changes) { + cat(sprintf( + "\nDry run: %d object(s) left in place. Re-run with --apply to delete.\n", + length(doomed) + )) + quit(status = 0) +} + +for (f in doomed) { + s3fs::s3_file_delete(file.path(contrib, f)) +} +cat(sprintf( + "Deleted %d flat binary object(s). Re-index the slot so the source fallback republishes them.\n", + length(doomed) +))