diff --git a/.crow/backfill-flat-safety.yaml b/.crow/backfill-flat-safety.yaml new file mode 100644 index 0000000..e2f850d --- /dev/null +++ b/.crow/backfill-flat-safety.yaml @@ -0,0 +1,178 @@ +# Record, per flat-slot object, whether it loads under every supported R minor. +# +# `union_index_records()` keeps a carried flat record when that object is +# recorded safe and drops it otherwise. Objects uploaded before that record +# existed have no verdict, so they are dropped: a full re-index removed roughly +# 2000 records per per-minor index (16398 across the 4.4 indexes, 20076 across +# 4.5, 15532 across 4.6), each one resolving to CRAN source for clients the +# routing serves correctly. +# +# This inspects what is already in the bucket and writes the verdict, which is +# what restores that coverage. Decided by symbols rather than heuristics: a +# binary fails at dyn.load() under minor M exactly when it references a symbol M +# does not export, and the volatile set is derived by diffing the R builds in +# the image. +# +# Re-running is cheap; recorded objects are skipped. +# +# crow pipeline create devxy/build-cran-binaries --var flat_safety=all + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the default + # must match no matrix row. + flat_safety: + description: "Inspect target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + FLAT_SAFETY_LIMIT: + description: 'Inspect at most this many objects per slot (0 = no limit).' + default: '0' + +when: + - event: manual + evaluate: 'flat_safety == "all" || flat_safety == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Inspect flat objects and record safety' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /tmp/R-pkgs + FLAT_SAFETY_ARCH: ${ARCH} + FLAT_SAFETY_LIMIT: ${FLAT_SAFETY_LIMIT} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /tmp/R-pkgs + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - | + set -eu + LIMIT="" + if [ "$$FLAT_SAFETY_LIMIT" != "0" ]; then + LIMIT="$$FLAT_SAFETY_LIMIT" + fi + # The codename comes from the image's /etc/os-release, exactly as the + # re-index step derives it, so it cannot drift from the matrix. + CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))') + echo "slot: $$FLAT_SAFETY_ARCH/$$CODENAME" + /opt/R/$R_VERSION/bin/Rscript local/backfill-flat-safety.R \ + "$$FLAT_SAFETY_ARCH" "$$CODENAME" $$LIMIT + backend_options: + docker: + resources: + requests: + memory: 4Gi + cpu: 1000m diff --git a/local/backfill-flat-safety.R b/local/backfill-flat-safety.R new file mode 100644 index 0000000..fe23735 --- /dev/null +++ b/local/backfill-flat-safety.R @@ -0,0 +1,148 @@ +### Record, per flat-slot object, whether it loads under every supported R minor. +# +# Usage: Rscript local/backfill-flat-safety.R [limit] +# +# `union_index_records()` keeps a carried flat record when that object is +# recorded safe, and drops it otherwise. Objects uploaded before that record +# existed have no verdict, so they are dropped: a full re-index removed ~2000 +# records per per-minor index, all resolving to CRAN source for clients the +# routing serves correctly. +# +# This inspects the objects already in the bucket and writes the verdict, which +# is what restores that coverage. The question is decided by symbols, not +# heuristics: a binary fails at dyn.load() under minor M exactly when it +# references a symbol M does not export, and the volatile set is derived by +# diffing the R builds installed in this image. +# +# Only compiled packages are inspected. A package with no shared object cannot +# fail this way and is never dropped, so proving it would be wasted downloads. +# +# Resumable: objects already recorded are skipped, so a re-run costs one query. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2L) { + stop("usage: backfill-flat-safety.R [limit]", call. = FALSE) +} +arch <- args[[1L]] +codename <- args[[2L]] +limit <- if (length(args) >= 3L) as.integer(args[[3L]]) else NA_integer_ + +sets <- bincraft:::installed_r_symbol_sets() +if (is.null(sets)) { + stop( + "Fewer than two R minors are installed here, so nothing can be compared. Refusing to record verdicts that would all read 'safe'.", # nolint + call. = FALSE + ) +} +cat(sprintf( + "Comparing R minors: %s (%d symbols vary across them)\n", + paste(names(sets), collapse = ", "), + length(bincraft:::cross_minor_volatile_symbols(sets)) +)) + +read_index <- function(index_url) { + con <- gzcon(url(index_url, open = "rb")) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "Built", "NeedsCompilation")) +} + +flat <- read_index(sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", + arch, + codename +)) +# Source fallbacks carry no binary, so there is nothing to inspect. +flat <- flat[!is.na(flat[, "Built"]), , drop = FALSE] +compiled <- flat[ + !is.na(flat[, "NeedsCompilation"]) & flat[, "NeedsCompilation"] == "yes", + , + drop = FALSE +] +cat(sprintf( + "%s/%s: %d flat binaries, %d compiled and worth inspecting\n", + arch, + codename, + nrow(flat), + nrow(compiled) +)) +if (!is.na(limit) && limit > 0L && limit < nrow(compiled)) { + compiled <- compiled[seq_len(limit), , drop = FALSE] + cat(sprintf("Limited to the first %d\n", limit)) +} + +base <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib", + arch, + codename +) +safe <- 0L +unsafe <- 0L +skipped <- 0L +for (i in seq_len(nrow(compiled))) { + pkg <- compiled[i, "Package"] + ver <- compiled[i, "Version"] + tarball <- file.path(tempdir(), sprintf("%s_%s.tar.gz", pkg, ver)) + ok <- tryCatch( + { + utils::download.file( + sprintf("%s/%s_%s.tar.gz", base, pkg, ver), + tarball, + quiet = TRUE, + mode = "wb" + ) + TRUE + }, + error = function(e) FALSE, + warning = function(w) FALSE + ) + if (!ok) { + skipped <- skipped + 1L + next + } + verdict <- bincraft:::tarball_abi_verdict(tarball, sets) + unlink(tarball) + if (!isTRUE(verdict$inspected)) { + skipped <- skipped + 1L + next + } + bincraft:::flat_safety_store( + pkg, + ver, + codename, + arch, + safe = !isTRUE(verdict$sensitive), + unsupported = verdict$unsupported + ) + if (isTRUE(verdict$sensitive)) { + unsafe <- unsafe + 1L + } else { + safe <- safe + 1L + } + if (i %% 250L == 0L) { + cat(sprintf( + " inspected %d/%d (%d safe, %d unsafe, %d skipped)\n", + i, + nrow(compiled), + safe, + unsafe, + skipped + )) + utils::flush.console() + } +} + +cat(sprintf( + "Done: %d inspected, %d safe for every minor, %d unsafe, %d skipped\n", + nrow(compiled), + safe, + unsafe, + skipped +))