From 26406a3e86f62f7df71e71696f816fc2c76cbe29 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 4 Sep 2026 10:24:47 +0000 Subject: [PATCH] feat(ci): add a targeted per-R-minor rebuild workflow (#198) ## Why A user hit this on amd64/resolute under R 4.6.1: ``` base64enc.so: undefined symbol: SETLENGTH ``` R 4.6 no longer exports `SETLENGTH`: ``` R 4.4: missing -> R_ClosureEnv R_ClosureFormals R_MakeMissingBinding R_PromiseEnv R_PromiseValue R_mkClosure R 4.5: missing -> R_MakeMissingBinding R_PromiseEnv R_PromiseValue R 4.6: missing -> R_PromiseEnv R_PromiseValue SETLENGTH ``` `base64enc` calls `SETLENGTH` (`base64.c:206`, `253`), its flat binary was built under R 4.4.3, and it has no per-minor copy at all, so the union index hands that binary to a 4.6 client. The safety net missed it because the risky set is defined circularly: ```r risky_packages_across_minors <- function(...) { # a package is "risky" iff it already has a tarball in some per-minor slot ``` A package counts as ABI-risky only if it has *already* been built per-minor. `base64enc` never was, and having no new CRAN version it never enters the build list to be classified. So the per-minor protection only covers packages that passed through the pipeline after the scheme existed. ## Scope Scanning all 4,585 compiled binaries in amd64/resolute's flat index for symbols R 4.6 does not export found 10 referencing `SETLENGTH`. Most already have per-minor builds and are served correctly. Two do not: ``` base64enc_0.1-6 flat=200 4.4=404 4.5=404 4.6=404 lobstr_1.2.1 flat=200 4.4=404 4.5=200 4.6=404 ``` (`iotools` lacks a 4.5 copy but 4.5 still exports `SETLENGTH`, so it does not break.) ## What this adds Rebuilding one package into one minor was only possible with `just rebuild` from a workstation holding the S3 and Postgres credentials. This adds the CI path: ```sh crow pipeline create devxy/build-cran-binaries \ --var rebuild_abi_gap=ubuntu-2604-amd64 \ --var ABI_PACKAGES="base64enc@0.1-6 lobstr@1.2.1" \ --var ABI_R_MINOR=4.6 ``` - reuses `reindex.yaml`'s slot matrix verbatim, so the slot-to-image mapping cannot drift - runs `build-one.R --sensitive-only`, which reclassifies each version and skips anything not ABI-risky rather than writing it to the wrong slot - mounts no shared cache volume, so it cannot disturb the build pipeline's per-agent caches - attempts every package, then exits non-zero if any failed `crow lint` passes. ## Not in this PR This is the remediation path, not the fix. The circular `risky_packages_across_minors()` definition is still in place, so new instances of this class can still appear. That is the follow-up. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/198 --- .crow/rebuild-abi-gap.yaml | 218 +++++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 .crow/rebuild-abi-gap.yaml diff --git a/.crow/rebuild-abi-gap.yaml b/.crow/rebuild-abi-gap.yaml new file mode 100644 index 0000000..d346adf --- /dev/null +++ b/.crow/rebuild-abi-gap.yaml @@ -0,0 +1,218 @@ +# Rebuild specific packages into a per-R-minor slot. +# +# The per-minor scheme only ever covered packages that passed through the build +# pipeline after it existed. A package built into the flat slot before that, and +# never updated on CRAN since, is never reclassified and never gets a per-minor +# copy - so the union index serves its flat binary to every minor. When that +# binary references a symbol the client's R no longer exports, the install fails +# at dyn.load(): +# +# base64enc.so: undefined symbol: SETLENGTH +# +# base64enc calls SETLENGTH, its flat binary was built under R 4.4.3, and R 4.6 +# does not export SETLENGTH. There was no way to rebuild one package into one +# minor from CI, only `just rebuild` from a workstation with the S3 and Postgres +# credentials to hand. This adds that path. +# +# `build-one.R --sensitive-only` re-runs the ABI classifier per version, so a +# package that turns out not to be risky is skipped rather than written into the +# wrong slot. +# +# Trigger with the slot and an explicit package list: +# +# crow pipeline create devxy/build-cran-binaries \ +# --var rebuild_abi_gap=ubuntu-2604-amd64 \ +# --var ABI_PACKAGES="base64enc@0.1-6 lobstr@1.2.1" \ +# --var ABI_R_MINOR=4.6 + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the + # default must match no matrix row. + rebuild_abi_gap: + description: "Rebuild target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + ABI_PACKAGES: + description: 'Space-separated package@version list, e.g. "base64enc@0.1-6 lobstr@1.2.1".' + default: '' + ABI_R_MINOR: + description: 'The R minor to build into.' + options: + - '4.4' + - '4.5' + - '4.6' + default: '4.6' + +when: + - event: manual + evaluate: 'rebuild_abi_gap == "all" || rebuild_abi_gap == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Rebuild packages into the per-minor slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + GIT_USER: pat-s + # Pipeline variables are substituted into the YAML, not exported, so they + # have to be bound here to be readable as env vars in the script below. + ABI_PACKAGES: ${ABI_PACKAGES} + ABI_R_MINOR: ${ABI_R_MINOR} + UVR_CACHE_DIR: /tmp/uvr/cache + UVR_PACKAGES_DIR: /tmp/uvr/packages + CCACHE_DIR: /tmp/ccache + NCPUS: 2 + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p $UVR_CACHE_DIR $UVR_PACKAGES_DIR $CCACHE_DIR + - | + set -eu + if [ -z "$$ABI_PACKAGES" ]; then + echo "ABI_PACKAGES is empty; nothing to rebuild." >&2 + exit 1 + fi + RBIN=$(ls -d /opt/R/$${ABI_R_MINOR}*/bin 2>/dev/null | head -1) + if [ -z "$$RBIN" ]; then + echo "R $$ABI_R_MINOR is not installed in this image." >&2 + exit 1 + fi + # Container-local library: this workflow mounts no shared cache volume, + # so it cannot disturb the build pipeline's per-agent caches. + LIB="/tmp/R-pkgs-$$ABI_R_MINOR" + mkdir -p "$$LIB" + export R_LIBS_USER="$$LIB" + rm -rf "$$LIB"/00LOCK-* + "$$RBIN/Rscript" local/install-bincraft.R + rc=0 + for spec in $$ABI_PACKAGES; do + pkg=$${spec%@*} + ver=$${spec#*@} + if [ "$$pkg" = "$$ver" ]; then + echo "malformed spec '$$spec'; expected package@version" >&2 + rc=1 + continue + fi + echo "=== rebuilding $$pkg $$ver into $$ABI_R_MINOR ===" + "$$RBIN/Rscript" local/build-one.R --sensitive-only "$$pkg" "$$ver" || rc=1 + done + exit $$rc + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 1000m