From f9ebb3dba489a73c0d7c73c372ce14fbb0834231 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 4 Sep 2026 10:54:09 +0000 Subject: [PATCH] fix(abi): prune flat-slot binaries of ABI-risky packages (#200) ## Why `base64enc` still fails to install under R 4.6 even after it was built into the 4.6 slot: ``` base64enc.so: undefined symbol: SETLENGTH ``` The per-minor binary is correct and serves correctly to any client the router can identify. But the **flat slot still holds a binary built under R 4.4.3**, and that is what every client receives when it cannot be routed by R minor. That object should not exist. `upload_single_binary()` writes an ABI-risky package only to the per-minor path: ```r if (is_r_minor_sensitive) { remote_tarball_path <- file.path(remote_bin_path, minor_version, tarball_name) } else { remote_tarball_path <- file.path(remote_bin_path, tarball_name) } ``` `base64enc`'s flat binary is a legacy artifact from before it was classified risky. Removing it makes the outcome correct regardless of routing, user agent, downloader or renv configuration, which is the part I could not otherwise guarantee. ## What it does `local/prune-stale-flat-binaries.R` lists a slot's flat objects, intersects them with the risky set, and deletes them. Deleting is safe here: the source fallback republishes the CRAN source under the same key, so the package stays installable everywhere and compiles against the client's own R, while routed clients keep getting the per-minor binary. Safety posture, given this deletes production objects: - **dry run by default**; nothing is removed without `--apply` - takes an **explicit package list**, so the first runs need not trust the classifier cache at all - **refuses to proceed on an empty risky set**, which is indistinguishable from an unreachable cache and would otherwise report success having pruned nothing - **keeps objects whose package CRAN no longer ships**, since there would be no source to fall back to and the package would vanish - **re-indexes and purges in the same run**. This is not optional: a published index naming deleted files serves 404s, which is worse than the binary it replaced. ## Use ```sh # dry run first crow pipeline create devxy/build-cran-binaries \ --var prune_flat=ubuntu-2604-amd64 --var PRUNE_PACKAGES="base64enc lobstr" # then apply crow pipeline create devxy/build-cran-binaries \ --var prune_flat=ubuntu-2604-amd64 --var PRUNE_PACKAGES="base64enc lobstr" \ --var PRUNE_APPLY=yes ``` `crow lint` passes. ## Sequencing Run with an explicit list first, on one slot, and confirm the flat URL then serves source while the per-minor URLs still serve binaries. Only widen to the cache-derived set once the backfill has finished and its output looks sane: the first 250 classifications came back 232 r-minor-sensitive, and a rate that high suggests the source-grep classifier is broader than the set that actually breaks, which is worth checking before pruning thousands of flat binaries. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/200 --- .crow/prune-stale-flat-binaries.yaml | 245 +++++++++++++++++++++++++++ local/prune-stale-flat-binaries.R | 136 +++++++++++++++ 2 files changed, 381 insertions(+) create mode 100644 .crow/prune-stale-flat-binaries.yaml create mode 100644 local/prune-stale-flat-binaries.R diff --git a/.crow/prune-stale-flat-binaries.yaml b/.crow/prune-stale-flat-binaries.yaml new file mode 100644 index 0000000..a759daf --- /dev/null +++ b/.crow/prune-stale-flat-binaries.yaml @@ -0,0 +1,245 @@ +# Remove flat-slot binaries belonging to ABI-risky packages, then re-index. +# +# `upload_single_binary()` writes an ABI-risky package only to the per-minor +# slot, so a risky package should have no flat binary at all. Packages built +# before they were classified risky still do, and the flat slot is what every +# client that cannot be routed by R minor receives. That binary was built under +# whichever R was primary at the time, so it dies at load under any minor that +# dropped a symbol it uses: +# +# base64enc.so: undefined symbol: SETLENGTH +# +# Deleting the flat object does not create a hole. The source fallback +# republishes the CRAN source under the same key, so the package stays +# installable everywhere and compiles against the client's own R, while routed +# clients keep getting the per-minor binary. +# +# Dry run unless PRUNE_APPLY is 'yes'. Start with an explicit package list: +# +# crow pipeline create devxy/build-cran-binaries \ +# --var prune_flat=ubuntu-2604-amd64 \ +# --var PRUNE_PACKAGES="base64enc lobstr" \ +# --var PRUNE_APPLY=yes + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the default + # must match no matrix row. + prune_flat: + description: "Prune target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + PRUNE_PACKAGES: + description: 'Space-separated package names. Empty means every package the classifier cache calls risky.' + default: '' + PRUNE_APPLY: + description: "Set to 'yes' to actually delete. Anything else is a dry run." + options: + - 'no' + - 'yes' + default: 'no' + +when: + - event: manual + evaluate: 'prune_flat == "all" || prune_flat == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + CODENAME: alpine322 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + CODENAME: alpine322 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + CODENAME: alpine323 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + CODENAME: alpine323 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + CODENAME: alpine324 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + CODENAME: alpine324 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + CODENAME: rhel8 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + CODENAME: rhel8 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + CODENAME: rhel9 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + CODENAME: rhel9 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + CODENAME: rhel10 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + CODENAME: rhel10 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + CODENAME: jammy + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + CODENAME: jammy + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + CODENAME: noble + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + CODENAME: noble + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + CODENAME: resolute + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + CODENAME: resolute + IMG: ubuntu:resolute + +steps: + - name: 'Prune stale flat binaries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /tmp/R-pkgs + # Pipeline variables are substituted into the YAML, not exported, so they + # have to be bound here to be readable as env vars in the script below. + PRUNE_ARCH: ${ARCH} + PRUNE_CODENAME: ${CODENAME} + PRUNE_PACKAGES: ${PRUNE_PACKAGES} + PRUNE_APPLY: ${PRUNE_APPLY} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /tmp/R-pkgs + - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R + - | + set -eu + APPLY="" + if [ "$$PRUNE_APPLY" = "yes" ]; then + APPLY="--apply" + fi + /opt/R/4.5.3/bin/Rscript local/prune-stale-flat-binaries.R \ + "$$PRUNE_ARCH" "$$PRUNE_CODENAME" $$APPLY $$PRUNE_PACKAGES + backend_options: + docker: + resources: + requests: + memory: 3Gi + cpu: 1000m + + # Deleting objects without re-indexing is worse than leaving them: the + # published index keeps naming files that no longer exist, so every client + # gets a 404 instead of a working source fallback. The re-index is what + # republishes them as CRAN source, so it belongs in the same run. + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + R_LIBS_USER: /tmp/R-pkgs + PRUNE_APPLY: ${PRUNE_APPLY} + commands: + - | + set -eu + if [ "$$PRUNE_APPLY" != "yes" ]; then + echo "Dry run: nothing was deleted, so there is nothing to re-index." + exit 0 + fi + /opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + backend_options: + docker: + resources: + requests: + memory: 3Gi + cpu: 1000m + + # The edge caches indexes for ~370 days, so without this the deletion is + # invisible to clients and they keep being handed the removed binary. + - name: 'Purge CDN cache' + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + # Same zones as reindex.yaml: the production pull zone id, the alliance + # hostname and the staging zone, which shares the B2 origin. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' + PRUNE_APPLY: ${PRUNE_APPLY} + commands: + - | + set -eu + if [ "$$PRUNE_APPLY" != "yes" ]; then + echo "Dry run: nothing changed at the origin, so no purge is needed." + exit 0 + fi + apk add --no-cache -q bash curl jq + bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES diff --git a/local/prune-stale-flat-binaries.R b/local/prune-stale-flat-binaries.R new file mode 100644 index 0000000..dbb0ab1 --- /dev/null +++ b/local/prune-stale-flat-binaries.R @@ -0,0 +1,136 @@ +### Remove flat-slot binaries of packages that are ABI-risky. +# +# Usage: +# Rscript local/prune-stale-flat-binaries.R [--apply] [pkg ...] +# +# `upload_single_binary()` writes an ABI-risky package only to the per-minor +# slot, never to the flat one, so a risky package should have no flat binary at +# all. Packages built before they were classified risky still do, and the flat +# slot is what every client that cannot be routed by R minor receives. That +# binary was built under whichever R was primary at the time, so it fails at +# load under any minor that dropped a symbol it uses: +# +# base64enc.so: undefined symbol: SETLENGTH +# +# Deleting the flat object does not create a hole: bincraft's source fallback +# publishes the CRAN source under the same key, so the package stays installable +# everywhere and compiles against the client's own R. Clients that are routed +# keep getting the per-minor binary. +# +# Dry run by default. Nothing is deleted without --apply. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +args <- commandArgs(trailingOnly = TRUE) +apply_changes <- "--apply" %in% args +args <- args[args != "--apply"] +if (length(args) < 2L) { + stop( + "usage: prune-stale-flat-binaries.R [--apply] [pkg ...]", + call. = FALSE + ) +} +arch <- args[[1L]] +codename <- args[[2L]] +explicit <- args[-c(1L, 2L)] + +bucket <- "devxy-rpkgs-binaries" +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) + +contrib <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename) +objects <- s3fs::s3_dir_ls(contrib, recurse = TRUE) +relative <- sub(sprintf("^(s3://)?%s/", contrib), "", objects) + +is_tarball <- grepl("\\.tar\\.gz$", relative) +# Flat objects only: no `/` prefix and not under Archive/. +is_flat <- is_tarball & !grepl("/", relative, fixed = TRUE) +flat_files <- relative[is_flat] +flat_pkg <- sub("_.*$", "", flat_files) + +if (length(explicit) > 0L) { + risky <- explicit + cat(sprintf("Restricted to %d named package(s)\n", length(risky))) +} else { + risky <- bincraft:::abi_cache_risky_packages() + cat(sprintf("Classifier cache reports %d risky package(s)\n", length(risky))) + if (length(risky) == 0L) { + stop( + "The classifier cache returned nothing. Refusing to proceed: an empty risky set here is indistinguishable from an unreachable cache, and would silently prune nothing while reporting success.", # nolint + call. = FALSE + ) + } +} + +doomed <- flat_files[flat_pkg %in% risky] +cat(sprintf( + "%s/%s: %d flat binaries, %d belong to ABI-risky packages\n", + arch, + codename, + length(flat_files), + length(doomed) +)) + +# Never remove the last artifact of a package. The source fallback republishes +# the CRAN source under this key, but only for packages CRAN still ships, so +# confirm the source exists before deleting the binary. +cran <- tryCatch( + { + con <- gzcon(url( + "https://cloud.r-project.org/src/contrib/PACKAGES.gz", + open = "rb" + )) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = "Package")[, "Package"] + }, + error = function(e) character(0) +) +if (length(cran) == 0L) { + stop("Could not read CRAN's index; refusing to prune blind.", call. = FALSE) +} + +orphaned <- doomed[!(sub("_.*$", "", doomed) %in% cran)] +if (length(orphaned) > 0L) { + cat(sprintf( + "Keeping %d object(s) whose package is no longer on CRAN, so there is no source to fall back to: %s\n", + length(orphaned), + paste(utils::head(orphaned, 5L), collapse = ", ") + )) + doomed <- setdiff(doomed, orphaned) +} + +if (length(doomed) == 0L) { + cat("Nothing to prune.\n") + quit(status = 0) +} + +for (f in doomed) { + cat(sprintf(" %s %s\n", if (apply_changes) "DELETE" else "would delete", f)) +} + +if (!apply_changes) { + cat(sprintf( + "\nDry run: %d object(s) left in place. Re-run with --apply to delete.\n", + length(doomed) + )) + quit(status = 0) +} + +for (f in doomed) { + s3fs::s3_file_delete(file.path(contrib, f)) +} +cat(sprintf( + "Deleted %d flat binary object(s). Re-index the slot so the source fallback republishes them.\n", + length(doomed) +))