.crow/backfill-abi-classification.yaml

@@ -0,0 +1,90 @@
+# Populate the ABI classifier cache for packages that predate the per-minor scheme.
+#
+# bincraft decides whether a flat binary may be carried into a per-minor index
+# from its risky-package set. That set is now the union of the per-minor tarball
+# scan and the `abi_classification` cache, but the cache only holds verdicts for
+# packages that passed through the build pipeline after the scheme existed.
+# Everything older has no verdict, and no verdict means "carry it" - which is how
+# a base64enc built under R 4.4.3 reached R 4.6 clients and died at dyn.load()
+# on SETLENGTH.
+#
+# The verdict is computed from the package source, so it does not depend on OS,
+# arch or R minor: one run populates the cache for every slot. The arch/codename
+# below only choose which flat index to enumerate.
+#
+# Re-running is cheap. Cached verdicts are returned without touching the network,
+# so a second run costs one query per package and downloads only what is still
+# unclassified.
+#
+#   crow pipeline create devxy/build-cran-binaries --var backfill_abi=run
+
+variables:
+  # A manual pipeline creation instantiates every file in .crow/, so the default
+  # must be a value the `when` clause rejects.
+  backfill_abi:
+    description: "Set to 'run' to classify the flat index into the ABI cache."
+    options:
+      - none
+      - run
+    default: none
+  BACKFILL_ARCH:
+    description: 'Arch of the flat index to enumerate.'
+    options:
+      - amd64
+      - arm64
+    default: amd64
+  BACKFILL_CODENAME:
+    description: 'Codename of the flat index to enumerate.'
+    default: resolute
+  BACKFILL_LIMIT:
+    description: 'Classify at most this many packages (0 = no limit).'
+    default: '0'
+
+when:
+  - event: manual
+    evaluate: 'backfill_abi == "run"'
+
+skip_clone: true
+
+labels:
+  group: rpkgs-amd64
+
+steps:
+  - name: 'Classify the flat index into the ABI cache'
+    image: reg.devxy.io/rpkgs/build-env-ubuntu:resolute
+    pull: true
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GITHUB_PAT:
+        from_secret: GITHUB_PAT
+      GIT_USER: pat-s
+      # Pipeline variables are substituted into the YAML, not exported, so they
+      # have to be bound here to be readable as env vars in the script below.
+      BACKFILL_ARCH: ${BACKFILL_ARCH}
+      BACKFILL_CODENAME: ${BACKFILL_CODENAME}
+      BACKFILL_LIMIT: ${BACKFILL_LIMIT}
+      R_LIBS_USER: /tmp/R-pkgs
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - mkdir -p /tmp/R-pkgs
+      - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R
+      - |
+        set -eu
+        LIMIT=""
+        if [ "$$BACKFILL_LIMIT" != "0" ]; then
+          LIMIT="$$BACKFILL_LIMIT"
+        fi
+        /opt/R/4.5.3/bin/Rscript local/backfill-abi-classification.R \
+          "$$BACKFILL_ARCH" "$$BACKFILL_CODENAME" $$LIMIT
+    backend_options:
+      docker:
+        resources:
+          requests:
+            memory: 4Gi
+            cpu: 1000m

local/backfill-abi-classification.R

@@ -0,0 +1,129 @@
+### Classify every compiled package in a slot's flat index, into the ABI cache.
+#
+# Usage: Rscript local/backfill-abi-classification.R <arch> <codename> [limit]
+#
+# The per-minor scheme only ever classified packages that passed through the
+# build pipeline after it existed. Everything built before that, and not updated
+# on CRAN since, has no verdict at all -- and bincraft's risky set is what
+# decides whether a flat binary may be carried into a per-minor index. With no
+# verdict the answer defaults to "carry it", which is how a base64enc built
+# under R 4.4.3 reached R 4.6 clients and died at dyn.load() on SETLENGTH.
+#
+# The verdict is a property of the source, so it is independent of OS, arch and
+# R minor: one slot's backfill populates the cache for every slot. The slot
+# argument only selects which flat index to enumerate.
+#
+# Only packages with NeedsCompilation=yes are classified. A package with no
+# compiled code has no symbols to bind and cannot fail this way, so downloading
+# its source to prove that would be wasted work.
+#
+# Resumable: `classify_r_minor_sensitive()` returns cached verdicts without
+# touching the network, so a re-run costs one query per package and only the
+# unclassified remainder is downloaded.
+
+options(error = function() {
+  cat("ERROR:", geterrmessage(), "\n", file = stdout())
+  traceback(2)
+  q(status = 1)
+})
+
+library(bincraft, quietly = TRUE)
+
+args <- commandArgs(trailingOnly = TRUE)
+if (length(args) < 2L) {
+  stop(
+    "usage: backfill-abi-classification.R <arch> <codename> [limit]",
+    call. = FALSE
+  )
+}
+arch <- args[[1L]]
+codename <- args[[2L]]
+limit <- if (length(args) >= 3L) as.integer(args[[3L]]) else NA_integer_
+
+db <- list(
+  metadata_db_type = "postgres",
+  metadata_db_host = "r-binaries.devxy.io",
+  metadata_db_name = "build_metadata",
+  metadata_db_port = 15432L,
+  metadata_db_user = "rpkgs",
+  metadata_db_password = Sys.getenv("PGPASS"),
+  metadata_db_sslmode = "require"
+)
+
+read_index <- function(index_url) {
+  con <- gzcon(url(index_url, open = "rb"))
+  on.exit(close(con), add = TRUE)
+  read.dcf(con, fields = c("Package", "Version", "Built"))
+}
+
+flat <- read_index(sprintf(
+  "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz",
+  arch,
+  codename
+))
+# Source fallbacks have no binary to be unsafe, so skip them.
+flat <- flat[!is.na(flat[, "Built"]), , drop = FALSE]
+
+cran <- read.dcf(
+  gzcon(url(
+    "https://cloud.r-project.org/src/contrib/PACKAGES.gz",
+    open = "rb"
+  )),
+  fields = c("Package", "NeedsCompilation")
+)
+compiled <- cran[
+  !is.na(cran[, "NeedsCompilation"]) & cran[, "NeedsCompilation"] == "yes",
+  "Package"
+]
+
+todo <- flat[flat[, "Package"] %in% compiled, , drop = FALSE]
+cat(sprintf(
+  "%s/%s: %d binaries in the flat index, %d compiled and due classification\n",
+  arch,
+  codename,
+  nrow(flat),
+  nrow(todo)
+))
+if (!is.na(limit) && limit > 0L && limit < nrow(todo)) {
+  todo <- todo[seq_len(limit), , drop = FALSE]
+  cat(sprintf("Limited to the first %d\n", limit))
+}
+
+sensitive <- 0L
+failed <- 0L
+for (i in seq_len(nrow(todo))) {
+  pkg <- todo[i, "Package"]
+  ver <- todo[i, "Version"]
+  verdict <- tryCatch(
+    do.call(
+      bincraft::classify_r_minor_sensitive,
+      c(list(package_name = pkg, tag = ver), db)
+    ),
+    error = function(e) {
+      cat(sprintf("  %s %s: %s\n", pkg, ver, conditionMessage(e)))
+      NA
+    }
+  )
+  if (is.na(verdict)) {
+    failed <- failed + 1L
+  } else if (isTRUE(verdict)) {
+    sensitive <- sensitive + 1L
+  }
+  if (i %% 250L == 0L) {
+    cat(sprintf(
+      "  classified %d/%d (%d r-minor-sensitive, %d failed)\n",
+      i,
+      nrow(todo),
+      sensitive,
+      failed
+    ))
+    utils::flush.console()
+  }
+}
+
+cat(sprintf(
+  "Done: %d classified, %d r-minor-sensitive, %d failed\n",
+  nrow(todo),
+  sensitive,
+  failed
+))