feat(abi): backfill per-object flat-slot safety verdicts

union_index_records() keeps a carried flat record when that object is recorded
safe and drops it otherwise. Objects uploaded before that record existed have no
verdict, so they are dropped. A full re-index removed roughly 2000 records per
per-minor index -- 16398 across the 4.4 indexes, 20076 across 4.5, 15532 across
4.6 -- each resolving to CRAN source for clients the routing serves correctly.

This inspects the objects already in the bucket and writes the verdict, which is
what restores that coverage.

The question is decided by symbols rather than heuristics: a binary fails at
dyn.load() under minor M exactly when it references a symbol M does not export,
and the volatile set is derived by diffing the R builds installed in the image.

This change will:

- add local/backfill-flat-safety.R, inspecting each compiled flat object and
  recording whether it loads under every installed minor
- refuse to run where fewer than two R minors are installed, since every verdict
  would then read "safe" for want of anything to compare
- skip packages with no compiled code, which cannot fail this way
- add a manual .crow workflow over the same slot matrix as reindex.yaml

.crow/backfill-flat-safety.yaml

@@ -0,0 +1,178 @@
+# Record, per flat-slot object, whether it loads under every supported R minor.
+#
+# `union_index_records()` keeps a carried flat record when that object is
+# recorded safe and drops it otherwise. Objects uploaded before that record
+# existed have no verdict, so they are dropped: a full re-index removed roughly
+# 2000 records per per-minor index (16398 across the 4.4 indexes, 20076 across
+# 4.5, 15532 across 4.6), each one resolving to CRAN source for clients the
+# routing serves correctly.
+#
+# This inspects what is already in the bucket and writes the verdict, which is
+# what restores that coverage. Decided by symbols rather than heuristics: a
+# binary fails at dyn.load() under minor M exactly when it references a symbol M
+# does not export, and the volatile set is derived by diffing the R builds in
+# the image.
+#
+# Re-running is cheap; recorded objects are skipped.
+#
+#   crow pipeline create devxy/build-cran-binaries --var flat_safety=all
+
+variables:
+  # A manual pipeline creation instantiates every file in .crow/, so the default
+  # must match no matrix row.
+  flat_safety:
+    description: "Inspect target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
+    options:
+      - none
+      - all
+      - alpine-322-amd64
+      - alpine-322-arm64
+      - alpine-323-amd64
+      - alpine-323-arm64
+      - alpine-324-amd64
+      - alpine-324-arm64
+      - redhat-8-amd64
+      - redhat-8-arm64
+      - redhat-9-amd64
+      - redhat-9-arm64
+      - redhat-10-amd64
+      - redhat-10-arm64
+      - ubuntu-2204-amd64
+      - ubuntu-2204-arm64
+      - ubuntu-2404-amd64
+      - ubuntu-2404-arm64
+      - ubuntu-2604-amd64
+      - ubuntu-2604-arm64
+    default: none
+  FLAT_SAFETY_LIMIT:
+    description: 'Inspect at most this many objects per slot (0 = no limit).'
+    default: '0'
+
+when:
+  - event: manual
+    evaluate: 'flat_safety == "all" || flat_safety == "${OS}-${ARCH}"'
+
+skip_clone: true
+
+labels:
+  group: rpkgs-${ARCH}
+
+matrix:
+  include:
+    - OS: alpine-322
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+    - OS: alpine-322
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+    - OS: alpine-323
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+    - OS: alpine-323
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+    - OS: alpine-324
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: alpine-324
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: redhat-8
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+    - OS: redhat-8
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+    - OS: redhat-9
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+    - OS: redhat-9
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+    - OS: redhat-10
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+    - OS: redhat-10
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+    - OS: ubuntu-2204
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2204
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2404
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+    - OS: ubuntu-2404
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+    - OS: ubuntu-2604
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+    - OS: ubuntu-2604
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+
+steps:
+  - name: 'Inspect flat objects and record safety'
+    image: reg.devxy.io/rpkgs/build-env-${IMG}
+    pull: true
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GIT_USER: pat-s
+      R_LIBS_USER: /tmp/R-pkgs
+      FLAT_SAFETY_ARCH: ${ARCH}
+      FLAT_SAFETY_LIMIT: ${FLAT_SAFETY_LIMIT}
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - mkdir -p /tmp/R-pkgs
+      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
+      - |
+        set -eu
+        LIMIT=""
+        if [ "$$FLAT_SAFETY_LIMIT" != "0" ]; then
+          LIMIT="$$FLAT_SAFETY_LIMIT"
+        fi
+        # The codename comes from the image's /etc/os-release, exactly as the
+        # re-index step derives it, so it cannot drift from the matrix.
+        CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))')
+        echo "slot: $$FLAT_SAFETY_ARCH/$$CODENAME"
+        /opt/R/$R_VERSION/bin/Rscript local/backfill-flat-safety.R \
+          "$$FLAT_SAFETY_ARCH" "$$CODENAME" $$LIMIT
+    backend_options:
+      docker:
+        resources:
+          requests:
+            memory: 4Gi
+            cpu: 1000m

local/backfill-flat-safety.R

@@ -0,0 +1,148 @@
+### Record, per flat-slot object, whether it loads under every supported R minor.
+#
+# Usage: Rscript local/backfill-flat-safety.R <arch> <codename> [limit]
+#
+# `union_index_records()` keeps a carried flat record when that object is
+# recorded safe, and drops it otherwise. Objects uploaded before that record
+# existed have no verdict, so they are dropped: a full re-index removed ~2000
+# records per per-minor index, all resolving to CRAN source for clients the
+# routing serves correctly.
+#
+# This inspects the objects already in the bucket and writes the verdict, which
+# is what restores that coverage. The question is decided by symbols, not
+# heuristics: a binary fails at dyn.load() under minor M exactly when it
+# references a symbol M does not export, and the volatile set is derived by
+# diffing the R builds installed in this image.
+#
+# Only compiled packages are inspected. A package with no shared object cannot
+# fail this way and is never dropped, so proving it would be wasted downloads.
+#
+# Resumable: objects already recorded are skipped, so a re-run costs one query.
+
+options(error = function() {
+  cat("ERROR:", geterrmessage(), "\n", file = stdout())
+  traceback(2)
+  q(status = 1)
+})
+
+library(bincraft, quietly = TRUE)
+
+args <- commandArgs(trailingOnly = TRUE)
+if (length(args) < 2L) {
+  stop("usage: backfill-flat-safety.R <arch> <codename> [limit]", call. = FALSE)
+}
+arch <- args[[1L]]
+codename <- args[[2L]]
+limit <- if (length(args) >= 3L) as.integer(args[[3L]]) else NA_integer_
+
+sets <- bincraft:::installed_r_symbol_sets()
+if (is.null(sets)) {
+  stop(
+    "Fewer than two R minors are installed here, so nothing can be compared. Refusing to record verdicts that would all read 'safe'.", # nolint
+    call. = FALSE
+  )
+}
+cat(sprintf(
+  "Comparing R minors: %s (%d symbols vary across them)\n",
+  paste(names(sets), collapse = ", "),
+  length(bincraft:::cross_minor_volatile_symbols(sets))
+))
+
+read_index <- function(index_url) {
+  con <- gzcon(url(index_url, open = "rb"))
+  on.exit(close(con), add = TRUE)
+  read.dcf(con, fields = c("Package", "Version", "Built", "NeedsCompilation"))
+}
+
+flat <- read_index(sprintf(
+  "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz",
+  arch,
+  codename
+))
+# Source fallbacks carry no binary, so there is nothing to inspect.
+flat <- flat[!is.na(flat[, "Built"]), , drop = FALSE]
+compiled <- flat[
+  !is.na(flat[, "NeedsCompilation"]) & flat[, "NeedsCompilation"] == "yes",
+  ,
+  drop = FALSE
+]
+cat(sprintf(
+  "%s/%s: %d flat binaries, %d compiled and worth inspecting\n",
+  arch,
+  codename,
+  nrow(flat),
+  nrow(compiled)
+))
+if (!is.na(limit) && limit > 0L && limit < nrow(compiled)) {
+  compiled <- compiled[seq_len(limit), , drop = FALSE]
+  cat(sprintf("Limited to the first %d\n", limit))
+}
+
+base <- sprintf(
+  "https://cran.rpkgs.com/%s/%s/latest/src/contrib",
+  arch,
+  codename
+)
+safe <- 0L
+unsafe <- 0L
+skipped <- 0L
+for (i in seq_len(nrow(compiled))) {
+  pkg <- compiled[i, "Package"]
+  ver <- compiled[i, "Version"]
+  tarball <- file.path(tempdir(), sprintf("%s_%s.tar.gz", pkg, ver))
+  ok <- tryCatch(
+    {
+      utils::download.file(
+        sprintf("%s/%s_%s.tar.gz", base, pkg, ver),
+        tarball,
+        quiet = TRUE,
+        mode = "wb"
+      )
+      TRUE
+    },
+    error = function(e) FALSE,
+    warning = function(w) FALSE
+  )
+  if (!ok) {
+    skipped <- skipped + 1L
+    next
+  }
+  verdict <- bincraft:::tarball_abi_verdict(tarball, sets)
+  unlink(tarball)
+  if (!isTRUE(verdict$inspected)) {
+    skipped <- skipped + 1L
+    next
+  }
+  bincraft:::flat_safety_store(
+    pkg,
+    ver,
+    codename,
+    arch,
+    safe = !isTRUE(verdict$sensitive),
+    unsupported = verdict$unsupported
+  )
+  if (isTRUE(verdict$sensitive)) {
+    unsafe <- unsafe + 1L
+  } else {
+    safe <- safe + 1L
+  }
+  if (i %% 250L == 0L) {
+    cat(sprintf(
+      "  inspected %d/%d (%d safe, %d unsafe, %d skipped)\n",
+      i,
+      nrow(compiled),
+      safe,
+      unsafe,
+      skipped
+    ))
+    utils::flush.console()
+  }
+}
+
+cat(sprintf(
+  "Done: %d inspected, %d safe for every minor, %d unsafe, %d skipped\n",
+  nrow(compiled),
+  safe,
+  unsafe,
+  skipped
+))