.crow/backfill-flat-safety.yaml

@@ -0,0 +1,178 @@
+# Record, per flat-slot object, whether it loads under every supported R minor.
+#
+# `union_index_records()` keeps a carried flat record when that object is
+# recorded safe and drops it otherwise. Objects uploaded before that record
+# existed have no verdict, so they are dropped: a full re-index removed roughly
+# 2000 records per per-minor index (16398 across the 4.4 indexes, 20076 across
+# 4.5, 15532 across 4.6), each one resolving to CRAN source for clients the
+# routing serves correctly.
+#
+# This inspects what is already in the bucket and writes the verdict, which is
+# what restores that coverage. Decided by symbols rather than heuristics: a
+# binary fails at dyn.load() under minor M exactly when it references a symbol M
+# does not export, and the volatile set is derived by diffing the R builds in
+# the image.
+#
+# Re-running is cheap; recorded objects are skipped.
+#
+#   crow pipeline create devxy/build-cran-binaries --var flat_safety=all
+
+variables:
+  # A manual pipeline creation instantiates every file in .crow/, so the default
+  # must match no matrix row.
+  flat_safety:
+    description: "Inspect target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
+    options:
+      - none
+      - all
+      - alpine-322-amd64
+      - alpine-322-arm64
+      - alpine-323-amd64
+      - alpine-323-arm64
+      - alpine-324-amd64
+      - alpine-324-arm64
+      - redhat-8-amd64
+      - redhat-8-arm64
+      - redhat-9-amd64
+      - redhat-9-arm64
+      - redhat-10-amd64
+      - redhat-10-arm64
+      - ubuntu-2204-amd64
+      - ubuntu-2204-arm64
+      - ubuntu-2404-amd64
+      - ubuntu-2404-arm64
+      - ubuntu-2604-amd64
+      - ubuntu-2604-arm64
+    default: none
+  FLAT_SAFETY_LIMIT:
+    description: 'Inspect at most this many objects per slot (0 = no limit).'
+    default: '0'
+
+when:
+  - event: manual
+    evaluate: 'flat_safety == "all" || flat_safety == "${OS}-${ARCH}"'
+
+skip_clone: true
+
+labels:
+  group: rpkgs-${ARCH}
+
+matrix:
+  include:
+    - OS: alpine-322
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+    - OS: alpine-322
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.22
+    - OS: alpine-323
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+    - OS: alpine-323
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.23
+    - OS: alpine-324
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: alpine-324
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: redhat-8
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+    - OS: redhat-8
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+    - OS: redhat-9
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+    - OS: redhat-9
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+    - OS: redhat-10
+      ARCH: amd64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+    - OS: redhat-10
+      ARCH: arm64
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+    - OS: ubuntu-2204
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2204
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2404
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+    - OS: ubuntu-2404
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+    - OS: ubuntu-2604
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+    - OS: ubuntu-2604
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+
+steps:
+  - name: 'Inspect flat objects and record safety'
+    image: reg.devxy.io/rpkgs/build-env-${IMG}
+    pull: true
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GIT_USER: pat-s
+      R_LIBS_USER: /tmp/R-pkgs
+      FLAT_SAFETY_ARCH: ${ARCH}
+      FLAT_SAFETY_LIMIT: ${FLAT_SAFETY_LIMIT}
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - mkdir -p /tmp/R-pkgs
+      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
+      - |
+        set -eu
+        LIMIT=""
+        if [ "$$FLAT_SAFETY_LIMIT" != "0" ]; then
+          LIMIT="$$FLAT_SAFETY_LIMIT"
+        fi
+        # The codename comes from the image's /etc/os-release, exactly as the
+        # re-index step derives it, so it cannot drift from the matrix.
+        CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))')
+        echo "slot: $$FLAT_SAFETY_ARCH/$$CODENAME"
+        /opt/R/$R_VERSION/bin/Rscript local/backfill-flat-safety.R \
+          "$$FLAT_SAFETY_ARCH" "$$CODENAME" $$LIMIT
+    backend_options:
+      docker:
+        resources:
+          requests:
+            memory: 4Gi
+            cpu: 1000m

local/backfill-flat-safety.R

@@ -0,0 +1,148 @@
+### Record, per flat-slot object, whether it loads under every supported R minor.
+#
+# Usage: Rscript local/backfill-flat-safety.R <arch> <codename> [limit]
+#
+# `union_index_records()` keeps a carried flat record when that object is
+# recorded safe, and drops it otherwise. Objects uploaded before that record
+# existed have no verdict, so they are dropped: a full re-index removed ~2000
+# records per per-minor index, all resolving to CRAN source for clients the
+# routing serves correctly.
+#
+# This inspects the objects already in the bucket and writes the verdict, which
+# is what restores that coverage. The question is decided by symbols, not
+# heuristics: a binary fails at dyn.load() under minor M exactly when it
+# references a symbol M does not export, and the volatile set is derived by
+# diffing the R builds installed in this image.
+#
+# Only compiled packages are inspected. A package with no shared object cannot
+# fail this way and is never dropped, so proving it would be wasted downloads.
+#
+# Resumable: objects already recorded are skipped, so a re-run costs one query.
+
+options(error = function() {
+  cat("ERROR:", geterrmessage(), "\n", file = stdout())
+  traceback(2)
+  q(status = 1)
+})
+
+library(bincraft, quietly = TRUE)
+
+args <- commandArgs(trailingOnly = TRUE)
+if (length(args) < 2L) {
+  stop("usage: backfill-flat-safety.R <arch> <codename> [limit]", call. = FALSE)
+}
+arch <- args[[1L]]
+codename <- args[[2L]]
+limit <- if (length(args) >= 3L) as.integer(args[[3L]]) else NA_integer_
+
+sets <- bincraft:::installed_r_symbol_sets()
+if (is.null(sets)) {
+  stop(
+    "Fewer than two R minors are installed here, so nothing can be compared. Refusing to record verdicts that would all read 'safe'.", # nolint
+    call. = FALSE
+  )
+}
+cat(sprintf(
+  "Comparing R minors: %s (%d symbols vary across them)\n",
+  paste(names(sets), collapse = ", "),
+  length(bincraft:::cross_minor_volatile_symbols(sets))
+))
+
+read_index <- function(index_url) {
+  con <- gzcon(url(index_url, open = "rb"))
+  on.exit(close(con), add = TRUE)
+  read.dcf(con, fields = c("Package", "Version", "Built", "NeedsCompilation"))
+}
+
+flat <- read_index(sprintf(
+  "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz",
+  arch,
+  codename
+))
+# Source fallbacks carry no binary, so there is nothing to inspect.
+flat <- flat[!is.na(flat[, "Built"]), , drop = FALSE]
+compiled <- flat[
+  !is.na(flat[, "NeedsCompilation"]) & flat[, "NeedsCompilation"] == "yes",
+  ,
+  drop = FALSE
+]
+cat(sprintf(
+  "%s/%s: %d flat binaries, %d compiled and worth inspecting\n",
+  arch,
+  codename,
+  nrow(flat),
+  nrow(compiled)
+))
+if (!is.na(limit) && limit > 0L && limit < nrow(compiled)) {
+  compiled <- compiled[seq_len(limit), , drop = FALSE]
+  cat(sprintf("Limited to the first %d\n", limit))
+}
+
+base <- sprintf(
+  "https://cran.rpkgs.com/%s/%s/latest/src/contrib",
+  arch,
+  codename
+)
+safe <- 0L
+unsafe <- 0L
+skipped <- 0L
+for (i in seq_len(nrow(compiled))) {
+  pkg <- compiled[i, "Package"]
+  ver <- compiled[i, "Version"]
+  tarball <- file.path(tempdir(), sprintf("%s_%s.tar.gz", pkg, ver))
+  ok <- tryCatch(
+    {
+      utils::download.file(
+        sprintf("%s/%s_%s.tar.gz", base, pkg, ver),
+        tarball,
+        quiet = TRUE,
+        mode = "wb"
+      )
+      TRUE
+    },
+    error = function(e) FALSE,
+    warning = function(w) FALSE
+  )
+  if (!ok) {
+    skipped <- skipped + 1L
+    next
+  }
+  verdict <- bincraft:::tarball_abi_verdict(tarball, sets)
+  unlink(tarball)
+  if (!isTRUE(verdict$inspected)) {
+    skipped <- skipped + 1L
+    next
+  }
+  bincraft:::flat_safety_store(
+    pkg,
+    ver,
+    codename,
+    arch,
+    safe = !isTRUE(verdict$sensitive),
+    unsupported = verdict$unsupported
+  )
+  if (isTRUE(verdict$sensitive)) {
+    unsafe <- unsafe + 1L
+  } else {
+    safe <- safe + 1L
+  }
+  if (i %% 250L == 0L) {
+    cat(sprintf(
+      "  inspected %d/%d (%d safe, %d unsafe, %d skipped)\n",
+      i,
+      nrow(compiled),
+      safe,
+      unsafe,
+      skipped
+    ))
+    utils::flush.console()
+  }
+}
+
+cat(sprintf(
+  "Done: %d inspected, %d safe for every minor, %d unsafe, %d skipped\n",
+  nrow(compiled),
+  safe,
+  unsafe,
+  skipped
+))