feat(abi): repair from the recorded verdicts rather than an ad-hoc scan

The earlier repair worked from package lists produced by a one-off local symbol
scan. Those lists were derived from a candidate set seeded on one slot, so they
were a lower bound by construction, and they do not survive the machine that
made them.

abi_flat_safety now holds the authoritative answer for every slot: the
flat-safety backfill inspected all 95418 objects and recorded which cannot load
under which minor. 4077 are unsafe, against the 93% of compiled packages the
source classifier suspects.

Each object is rebuilt under every installed R minor with no --sensitive-only
gate. Since bincraft v5.3.0 the destination is decided by the built binary's
symbols, so a package needing a per-minor slot lands there and a portable one
lands in the flat slot. Gating on the source classifier is what let Rcpp slip
through and break every R 4.4 client on the slot.

This change will:

- add local/list-unsafe-flat-objects.R, printing the recorded unsafe objects
- fail loudly when a slot has no verdicts at all, since an empty repair list
  would otherwise read as "nothing to do"
- add a manual workflow rebuilding them under every installed minor

.crow/repair-unsafe-flat-objects.yaml

@@ -0,0 +1,183 @@
+# Rebuild every flat object recorded as unable to load under some R minor.
+#
+# The list comes from `abi_flat_safety`, written by the flat-safety backfill
+# from the binaries themselves, so it names exactly the objects that fail at
+# dyn.load() somewhere rather than everything a source heuristic suspects. The
+# heuristic calls 93% of compiled packages risky; the binaries say 4.3%.
+#
+# Each object is rebuilt under every R minor installed in the image, with no
+# `--sensitive-only` gate. Since bincraft v5.3.0 the destination is decided by
+# the built binary's symbols in `upload_single_binary()`, so a package needing a
+# per-minor slot lands there and a portable one lands in the flat slot. Gating
+# on the source classifier is what let Rcpp slip through and break every R 4.4
+# client on the slot.
+#
+#   crow pipeline create devxy/build-cran-binaries --var repair_unsafe=all
+
+variables:
+  # A manual pipeline creation instantiates every file in .crow/, so the default
+  # must match no matrix row.
+  repair_unsafe:
+    description: "Repair target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
+    options:
+      - none
+      - all
+      - alpine-322-amd64
+      - alpine-322-arm64
+      - alpine-323-amd64
+      - alpine-323-arm64
+      - alpine-324-amd64
+      - alpine-324-arm64
+      - redhat-8-amd64
+      - redhat-8-arm64
+      - redhat-9-amd64
+      - redhat-9-arm64
+      - redhat-10-amd64
+      - redhat-10-arm64
+      - ubuntu-2204-amd64
+      - ubuntu-2204-arm64
+      - ubuntu-2404-amd64
+      - ubuntu-2404-arm64
+      - ubuntu-2604-amd64
+      - ubuntu-2604-arm64
+    default: none
+
+when:
+  - event: manual
+    evaluate: 'repair_unsafe == "all" || repair_unsafe == "${OS}-${ARCH}"'
+
+skip_clone: true
+
+labels:
+  group: rpkgs-${ARCH}
+
+matrix:
+  include:
+    - OS: alpine-322
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+    - OS: alpine-322
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+    - OS: alpine-323
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+    - OS: alpine-323
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+    - OS: alpine-324
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: alpine-324
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: redhat-8
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+    - OS: redhat-8
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+    - OS: redhat-9
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+    - OS: redhat-9
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+    - OS: redhat-10
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+    - OS: redhat-10
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+    - OS: ubuntu-2204
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2204
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2404
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+    - OS: ubuntu-2404
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+    - OS: ubuntu-2604
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+    - OS: ubuntu-2604
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+
+steps:
+  - name: 'Rebuild objects recorded unsafe'
+    image: reg.devxy.io/rpkgs/build-env-${IMG}
+    pull: true
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GITHUB_PAT:
+        from_secret: GITHUB_PAT
+      GIT_USER: pat-s
+      R_LIBS_USER: /tmp/R-pkgs
+      NCPUS: 2
+      REPAIR_ARCH: ${ARCH}
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - mkdir -p /tmp/R-pkgs
+      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
+      - |
+        set -eu
+        CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))')
+        /opt/R/$R_VERSION/bin/Rscript local/list-unsafe-flat-objects.R \
+          "$$REPAIR_ARCH" "$$CODENAME" > /tmp/unsafe.txt
+        echo "objects to rebuild: $(wc -l < /tmp/unsafe.txt)"
+        rc=0
+        for RBIN in /opt/R/[0-9]*/bin/R; do
+          RV=$(basename "$(dirname "$(dirname "$$RBIN")")")
+          RMINOR=$(echo "$$RV" | cut -d. -f1-2)
+          echo "=== rebuilding under R $$RV ==="
+          LIB="/tmp/R-pkgs-$$RMINOR"
+          mkdir -p "$$LIB"
+          R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" local/install-bincraft.R || true
+          while read -r pkg ver; do
+            [ -z "$$pkg" ] && continue
+            R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" \
+              local/build-one.R "$$pkg" "$$ver" || rc=1
+          done < /tmp/unsafe.txt
+        done
+        exit $$rc
+    backend_options:
+      docker:
+        resources:
+          requests:
+            memory: 5Gi
+            cpu: 1000m

local/list-unsafe-flat-objects.R

@@ -0,0 +1,80 @@
+### Print the flat-slot objects recorded as unable to load under some R minor.
+#
+# Usage: Rscript local/list-unsafe-flat-objects.R <arch> <codename>
+#
+# Prints one `<package> <version>` per line, for a caller that rebuilds each
+# under every installed R minor.
+#
+# The verdicts come from `abi_flat_safety`, written by the flat-safety backfill
+# from the binaries themselves: an object is unsafe when it references a symbol
+# some supported minor does not export. That is the authoritative list of what
+# needs a per-minor build, and it replaces the ad-hoc symbol scans that produced
+# earlier repair lists.
+
+options(error = function() {
+  cat("ERROR:", geterrmessage(), "\n", file = stdout())
+  q(status = 1)
+})
+
+args <- commandArgs(trailingOnly = TRUE)
+if (length(args) < 2L) {
+  stop("usage: list-unsafe-flat-objects.R <arch> <codename>", call. = FALSE)
+}
+arch <- args[[1L]]
+codename <- args[[2L]]
+
+con <- DBI::dbConnect(
+  RPostgres::Postgres(),
+  dbname = "build_metadata",
+  host = "r-binaries.devxy.io",
+  port = 15432L,
+  user = "rpkgs",
+  password = Sys.getenv("PGPASS"),
+  sslmode = "require"
+)
+on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE)
+
+rows <- DBI::dbGetQuery(
+  con,
+  paste0(
+    "SELECT package, version FROM abi_flat_safety ",
+    "WHERE NOT safe AND codename = $1 AND arch = $2 ",
+    "ORDER BY package, version"
+  ),
+  params = list(codename, arch)
+)
+
+# An empty result is ambiguous: either nothing is unsafe, or the backfill never
+# ran for this slot. Say which, rather than let the caller read silence as done.
+if (nrow(rows) == 0L) {
+  total <- DBI::dbGetQuery(
+    con,
+    "SELECT count(*) AS n FROM abi_flat_safety WHERE codename = $1 AND arch = $2",
+    params = list(codename, arch)
+  )$n[[1L]]
+  if (total == 0L) {
+    stop(
+      sprintf(
+        "No verdicts recorded for %s/%s at all. Run the flat-safety backfill first; an empty repair list here would look like success.", # nolint
+        arch,
+        codename
+      ),
+      call. = FALSE
+    )
+  }
+  message(sprintf(
+    "%s/%s: %d objects recorded, none unsafe. Nothing to repair.",
+    arch,
+    codename,
+    total
+  ))
+  quit(status = 0)
+}
+
+message(sprintf(
+  "%s/%s: %d objects need a per-minor build",
+  arch,
+  codename,
+  nrow(rows)
+))
+cat(sprintf("%s %s", rows$package, rows$version), sep = "\n")