2026-09-04 10:54:10 +00:00
2 changed files with 381 additions and 0 deletions
--- a/.crow/prune-stale-flat-binaries.yaml
+++ b/.crow/prune-stale-flat-binaries.yaml
@ -0,0 +1,245 @@
+# Remove flat-slot binaries belonging to ABI-risky packages, then re-index.
+#
+# `upload_single_binary()` writes an ABI-risky package only to the per-minor
+# slot, so a risky package should have no flat binary at all. Packages built
+# before they were classified risky still do, and the flat slot is what every
+# client that cannot be routed by R minor receives. That binary was built under
+# whichever R was primary at the time, so it dies at load under any minor that
+# dropped a symbol it uses:
+#
+#   base64enc.so: undefined symbol: SETLENGTH
+#
+# Deleting the flat object does not create a hole. The source fallback
+# republishes the CRAN source under the same key, so the package stays
+# installable everywhere and compiles against the client's own R, while routed
+# clients keep getting the per-minor binary.
+#
+# Dry run unless PRUNE_APPLY is 'yes'. Start with an explicit package list:
+#
+#   crow pipeline create devxy/build-cran-binaries \
+#     --var prune_flat=ubuntu-2604-amd64 \
+#     --var PRUNE_PACKAGES="base64enc lobstr" \
+#     --var PRUNE_APPLY=yes
+
+variables:
+  # A manual pipeline creation instantiates every file in .crow/, so the default
+  # must match no matrix row.
+  prune_flat:
+    description: "Prune target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
+    options:
+      - none
+      - all
+      - alpine-322-amd64
+      - alpine-322-arm64
+      - alpine-323-amd64
+      - alpine-323-arm64
+      - alpine-324-amd64
+      - alpine-324-arm64
+      - redhat-8-amd64
+      - redhat-8-arm64
+      - redhat-9-amd64
+      - redhat-9-arm64
+      - redhat-10-amd64
+      - redhat-10-arm64
+      - ubuntu-2204-amd64
+      - ubuntu-2204-arm64
+      - ubuntu-2404-amd64
+      - ubuntu-2404-arm64
+      - ubuntu-2604-amd64
+      - ubuntu-2604-arm64
+    default: none
+  PRUNE_PACKAGES:
+    description: 'Space-separated package names. Empty means every package the classifier cache calls risky.'
+    default: ''
+  PRUNE_APPLY:
+    description: "Set to 'yes' to actually delete. Anything else is a dry run."
+    options:
+      - 'no'
+      - 'yes'
+    default: 'no'
+
+when:
+  - event: manual
+    evaluate: 'prune_flat == "all" || prune_flat == "${OS}-${ARCH}"'
+
+skip_clone: true
+
+labels:
+  group: rpkgs-${ARCH}
+
+matrix:
+  include:
+    - OS: alpine-322
+      ARCH: amd64
+      CODENAME: alpine322
+      IMG: alpine:3.22
+    - OS: alpine-322
+      ARCH: arm64
+      CODENAME: alpine322
+      IMG: alpine:3.22
+    - OS: alpine-323
+      ARCH: amd64
+      CODENAME: alpine323
+      IMG: alpine:3.23
+    - OS: alpine-323
+      ARCH: arm64
+      CODENAME: alpine323
+      IMG: alpine:3.23
+    - OS: alpine-324
+      ARCH: amd64
+      CODENAME: alpine324
+      IMG: alpine:3.24
+    - OS: alpine-324
+      ARCH: arm64
+      CODENAME: alpine324
+      IMG: alpine:3.24
+    - OS: redhat-8
+      ARCH: amd64
+      CODENAME: rhel8
+      IMG: redhat:8
+    - OS: redhat-8
+      ARCH: arm64
+      CODENAME: rhel8
+      IMG: redhat:8
+    - OS: redhat-9
+      ARCH: amd64
+      CODENAME: rhel9
+      IMG: redhat:9
+    - OS: redhat-9
+      ARCH: arm64
+      CODENAME: rhel9
+      IMG: redhat:9
+    - OS: redhat-10
+      ARCH: amd64
+      CODENAME: rhel10
+      IMG: redhat:10
+    - OS: redhat-10
+      ARCH: arm64
+      CODENAME: rhel10
+      IMG: redhat:10
+    - OS: ubuntu-2204
+      ARCH: amd64
+      CODENAME: jammy
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2204
+      ARCH: arm64
+      CODENAME: jammy
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2404
+      ARCH: amd64
+      CODENAME: noble
+      IMG: ubuntu:noble
+    - OS: ubuntu-2404
+      ARCH: arm64
+      CODENAME: noble
+      IMG: ubuntu:noble
+    - OS: ubuntu-2604
+      ARCH: amd64
+      CODENAME: resolute
+      IMG: ubuntu:resolute
+    - OS: ubuntu-2604
+      ARCH: arm64
+      CODENAME: resolute
+      IMG: ubuntu:resolute
+
+steps:
+  - name: 'Prune stale flat binaries'
+    image: reg.devxy.io/rpkgs/build-env-${IMG}
+    pull: true
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GIT_USER: pat-s
+      R_LIBS_USER: /tmp/R-pkgs
+      # Pipeline variables are substituted into the YAML, not exported, so they
+      # have to be bound here to be readable as env vars in the script below.
+      PRUNE_ARCH: ${ARCH}
+      PRUNE_CODENAME: ${CODENAME}
+      PRUNE_PACKAGES: ${PRUNE_PACKAGES}
+      PRUNE_APPLY: ${PRUNE_APPLY}
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - mkdir -p /tmp/R-pkgs
+      - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R
+      - |
+        set -eu
+        APPLY=""
+        if [ "$$PRUNE_APPLY" = "yes" ]; then
+          APPLY="--apply"
+        fi
+        /opt/R/4.5.3/bin/Rscript local/prune-stale-flat-binaries.R \
+          "$$PRUNE_ARCH" "$$PRUNE_CODENAME" $$APPLY $$PRUNE_PACKAGES
+    backend_options:
+      docker:
+        resources:
+          requests:
+            memory: 3Gi
+            cpu: 1000m
+
+  # Deleting objects without re-indexing is worse than leaving them: the
+  # published index keeps naming files that no longer exist, so every client
+  # gets a 404 instead of a working source fallback. The re-index is what
+  # republishes them as CRAN source, so it belongs in the same run.
+  - name: 'Re-index the slot'
+    image: reg.devxy.io/rpkgs/build-env-${IMG}
+    pull: true
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      R_LIBS_USER: /tmp/R-pkgs
+      PRUNE_APPLY: ${PRUNE_APPLY}
+    commands:
+      - |
+        set -eu
+        if [ "$$PRUNE_APPLY" != "yes" ]; then
+          echo "Dry run: nothing was deleted, so there is nothing to re-index."
+          exit 0
+        fi
+        /opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
+        for RBIN in /opt/R/[0-9]*/bin/R; do
+          RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
+          /opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
+        done
+    backend_options:
+      docker:
+        resources:
+          requests:
+            memory: 3Gi
+            cpu: 1000m
+
+  # The edge caches indexes for ~370 days, so without this the deletion is
+  # invisible to clients and they keep being handed the removed binary.
+  - name: 'Purge CDN cache'
+    image: reg.devxy.io/docker.io/library/alpine:3.24
+    environment:
+      BUNNYNET_API_KEY:
+        from_secret: BUNNYNET_API_KEY
+      # Same zones as reindex.yaml: the production pull zone id, the alliance
+      # hostname and the staging zone, which shares the B2 origin.
+      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
+      PRUNE_APPLY: ${PRUNE_APPLY}
+    commands:
+      - |
+        set -eu
+        if [ "$$PRUNE_APPLY" != "yes" ]; then
+          echo "Dry run: nothing changed at the origin, so no purge is needed."
+          exit 0
+        fi
+        apk add --no-cache -q bash curl jq
+        bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
--- a/local/prune-stale-flat-binaries.R
+++ b/local/prune-stale-flat-binaries.R
@ -0,0 +1,136 @@
+### Remove flat-slot binaries of packages that are ABI-risky.
+#
+# Usage:
+#   Rscript local/prune-stale-flat-binaries.R <arch> <codename> [--apply] [pkg ...]
+#
+# `upload_single_binary()` writes an ABI-risky package only to the per-minor
+# slot, never to the flat one, so a risky package should have no flat binary at
+# all. Packages built before they were classified risky still do, and the flat
+# slot is what every client that cannot be routed by R minor receives. That
+# binary was built under whichever R was primary at the time, so it fails at
+# load under any minor that dropped a symbol it uses:
+#
+#   base64enc.so: undefined symbol: SETLENGTH
+#
+# Deleting the flat object does not create a hole: bincraft's source fallback
+# publishes the CRAN source under the same key, so the package stays installable
+# everywhere and compiles against the client's own R. Clients that are routed
+# keep getting the per-minor binary.
+#
+# Dry run by default. Nothing is deleted without --apply.
+
+options(error = function() {
+  cat("ERROR:", geterrmessage(), "\n", file = stdout())
+  traceback(2)
+  q(status = 1)
+})
+
+library(bincraft, quietly = TRUE)
+
+args <- commandArgs(trailingOnly = TRUE)
+apply_changes <- "--apply" %in% args
+args <- args[args != "--apply"]
+if (length(args) < 2L) {
+  stop(
+    "usage: prune-stale-flat-binaries.R <arch> <codename> [--apply] [pkg ...]",
+    call. = FALSE
+  )
+}
+arch <- args[[1L]]
+codename <- args[[2L]]
+explicit <- args[-c(1L, 2L)]
+
+bucket <- "devxy-rpkgs-binaries"
+s3fs::s3_file_system(
+  aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
+  aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
+  endpoint = "https://s3.eu-central-003.backblazeb2.com",
+  region_name = "eu-central-003",
+  refresh = TRUE
+)
+
+contrib <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename)
+objects <- s3fs::s3_dir_ls(contrib, recurse = TRUE)
+relative <- sub(sprintf("^(s3://)?%s/", contrib), "", objects)
+
+is_tarball <- grepl("\\.tar\\.gz$", relative)
+# Flat objects only: no `<minor>/` prefix and not under Archive/.
+is_flat <- is_tarball & !grepl("/", relative, fixed = TRUE)
+flat_files <- relative[is_flat]
+flat_pkg <- sub("_.*$", "", flat_files)
+
+if (length(explicit) > 0L) {
+  risky <- explicit
+  cat(sprintf("Restricted to %d named package(s)\n", length(risky)))
+} else {
+  risky <- bincraft:::abi_cache_risky_packages()
+  cat(sprintf("Classifier cache reports %d risky package(s)\n", length(risky)))
+  if (length(risky) == 0L) {
+    stop(
+      "The classifier cache returned nothing. Refusing to proceed: an empty risky set here is indistinguishable from an unreachable cache, and would silently prune nothing while reporting success.", # nolint
+      call. = FALSE
+    )
+  }
+}
+
+doomed <- flat_files[flat_pkg %in% risky]
+cat(sprintf(
+  "%s/%s: %d flat binaries, %d belong to ABI-risky packages\n",
+  arch,
+  codename,
+  length(flat_files),
+  length(doomed)
+))
+
+# Never remove the last artifact of a package. The source fallback republishes
+# the CRAN source under this key, but only for packages CRAN still ships, so
+# confirm the source exists before deleting the binary.
+cran <- tryCatch(
+  {
+    con <- gzcon(url(
+      "https://cloud.r-project.org/src/contrib/PACKAGES.gz",
+      open = "rb"
+    ))
+    on.exit(close(con), add = TRUE)
+    read.dcf(con, fields = "Package")[, "Package"]
+  },
+  error = function(e) character(0)
+)
+if (length(cran) == 0L) {
+  stop("Could not read CRAN's index; refusing to prune blind.", call. = FALSE)
+}
+
+orphaned <- doomed[!(sub("_.*$", "", doomed) %in% cran)]
+if (length(orphaned) > 0L) {
+  cat(sprintf(
+    "Keeping %d object(s) whose package is no longer on CRAN, so there is no source to fall back to: %s\n",
+    length(orphaned),
+    paste(utils::head(orphaned, 5L), collapse = ", ")
+  ))
+  doomed <- setdiff(doomed, orphaned)
+}
+
+if (length(doomed) == 0L) {
+  cat("Nothing to prune.\n")
+  quit(status = 0)
+}
+
+for (f in doomed) {
+  cat(sprintf("  %s %s\n", if (apply_changes) "DELETE" else "would delete", f))
+}
+
+if (!apply_changes) {
+  cat(sprintf(
+    "\nDry run: %d object(s) left in place. Re-run with --apply to delete.\n",
+    length(doomed)
+  ))
+  quit(status = 0)
+}
+
+for (f in doomed) {
+  s3fs::s3_file_delete(file.path(contrib, f))
+}
+cat(sprintf(
+  "Deleted %d flat binary object(s). Re-index the slot so the source fallback republishes them.\n",
+  length(doomed)
+))