diff --git a/.crow/repair-unsafe-flat-objects.yaml b/.crow/repair-unsafe-flat-objects.yaml new file mode 100644 index 0000000..9a57107 --- /dev/null +++ b/.crow/repair-unsafe-flat-objects.yaml @@ -0,0 +1,183 @@ +# Rebuild every flat object recorded as unable to load under some R minor. +# +# The list comes from `abi_flat_safety`, written by the flat-safety backfill +# from the binaries themselves, so it names exactly the objects that fail at +# dyn.load() somewhere rather than everything a source heuristic suspects. The +# heuristic calls 93% of compiled packages risky; the binaries say 4.3%. +# +# Each object is rebuilt under every R minor installed in the image, with no +# `--sensitive-only` gate. Since bincraft v5.3.0 the destination is decided by +# the built binary's symbols in `upload_single_binary()`, so a package needing a +# per-minor slot lands there and a portable one lands in the flat slot. Gating +# on the source classifier is what let Rcpp slip through and break every R 4.4 +# client on the slot. +# +# crow pipeline create devxy/build-cran-binaries --var repair_unsafe=all + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the default + # must match no matrix row. + repair_unsafe: + description: "Repair target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: manual + evaluate: 'repair_unsafe == "all" || repair_unsafe == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Rebuild objects recorded unsafe' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + GIT_USER: pat-s + R_LIBS_USER: /tmp/R-pkgs + NCPUS: 2 + REPAIR_ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /tmp/R-pkgs + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - | + set -eu + CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))') + /opt/R/$R_VERSION/bin/Rscript local/list-unsafe-flat-objects.R \ + "$$REPAIR_ARCH" "$$CODENAME" > /tmp/unsafe.txt + echo "objects to rebuild: $(wc -l < /tmp/unsafe.txt)" + rc=0 + for RBIN in /opt/R/[0-9]*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$$RBIN")")") + RMINOR=$(echo "$$RV" | cut -d. -f1-2) + echo "=== rebuilding under R $$RV ===" + LIB="/tmp/R-pkgs-$$RMINOR" + mkdir -p "$$LIB" + R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" local/install-bincraft.R || true + while read -r pkg ver; do + [ -z "$$pkg" ] && continue + R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" \ + local/build-one.R "$$pkg" "$$ver" || rc=1 + done < /tmp/unsafe.txt + done + exit $$rc + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 1000m diff --git a/local/list-unsafe-flat-objects.R b/local/list-unsafe-flat-objects.R new file mode 100644 index 0000000..a495e83 --- /dev/null +++ b/local/list-unsafe-flat-objects.R @@ -0,0 +1,80 @@ +### Print the flat-slot objects recorded as unable to load under some R minor. +# +# Usage: Rscript local/list-unsafe-flat-objects.R +# +# Prints one ` ` per line, for a caller that rebuilds each +# under every installed R minor. +# +# The verdicts come from `abi_flat_safety`, written by the flat-safety backfill +# from the binaries themselves: an object is unsafe when it references a symbol +# some supported minor does not export. That is the authoritative list of what +# needs a per-minor build, and it replaces the ad-hoc symbol scans that produced +# earlier repair lists. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2L) { + stop("usage: list-unsafe-flat-objects.R ", call. = FALSE) +} +arch <- args[[1L]] +codename <- args[[2L]] + +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432L, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE) + +rows <- DBI::dbGetQuery( + con, + paste0( + "SELECT package, version FROM abi_flat_safety ", + "WHERE NOT safe AND codename = $1 AND arch = $2 ", + "ORDER BY package, version" + ), + params = list(codename, arch) +) + +# An empty result is ambiguous: either nothing is unsafe, or the backfill never +# ran for this slot. Say which, rather than let the caller read silence as done. +if (nrow(rows) == 0L) { + total <- DBI::dbGetQuery( + con, + "SELECT count(*) AS n FROM abi_flat_safety WHERE codename = $1 AND arch = $2", + params = list(codename, arch) + )$n[[1L]] + if (total == 0L) { + stop( + sprintf( + "No verdicts recorded for %s/%s at all. Run the flat-safety backfill first; an empty repair list here would look like success.", # nolint + arch, + codename + ), + call. = FALSE + ) + } + message(sprintf( + "%s/%s: %d objects recorded, none unsafe. Nothing to repair.", + arch, + codename, + total + )) + quit(status = 0) +} + +message(sprintf( + "%s/%s: %d objects need a per-minor build", + arch, + codename, + nrow(rows) +)) +cat(sprintf("%s %s", rows$package, rows$version), sep = "\n")