From ca336c2720486d48fcc0fb76e84b6e1ebdb033c4 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 29 Jun 2026 12:11:03 +0000 Subject: [PATCH 1/8] fix(ci): detect existing R versions via public object HEAD probe (#8) The existing-version check listed the bucket with an anonymous curl, but Backblaze B2 requires authentication for the list-bucket API, so it returned AccessDenied and r-versions-existing.txt was always empty -> every version was rebuilt despite already existing in s3. Signing the request is not workable here: the B2 credentials are scoped to the plugin-s3 image in crow and are not injected into this plain alpine step (curl saw --user ":"). Individual artifacts are public-read, so probe each expected object URL with an anonymous HEAD instead of listing the bucket. The filename mirrors what nfpm produces (version/release pinned to 1): r-_1_.{apk,deb} for alpine/ubuntu and R--1-1..rpm for el*. Verified against the live bucket for all three packager families. Reviewed-on: https://git.devxy.io/devxy/r-builds/pulls/8 --- .crow/build.yaml | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index b3b6c0e..c980b4b 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -158,28 +158,27 @@ steps: - name: Check which R versions already exist in s3 image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true - environment: - AWS_ACCESS_KEY_ID: - from_secret: B2_S3_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: - from_secret: B2_S3_SECRET_KEY commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done - | - # Backblaze B2 requires authentication for the list-bucket API (anonymous - # GET works only for individual public-read objects), so the request must be - # SigV4-signed with the same credentials used for the upload step. Without - # this the listing returns AccessDenied, r-versions-existing.txt stays empty, - # and every version is rebuilt even though it already exists. - LISTING=$(curl -s --aws-sigv4 "aws:amz:eu-central-003:s3" \ - --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ - "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ - grep -oE '[^<]+' | sed 's/<[^>]*>//g') + # Backblaze B2's list-bucket API requires authentication, but the upload + # credentials are scoped to the plugin-s3 image and aren't available here. + # Individual objects are public-read, so probe each expected artifact URL + # with an anonymous HEAD instead of listing the bucket. The filename mirrors + # what nfpm produces in builder/package.${PLATFORM} (version/release pinned + # to 1): r-_1_.{apk,deb} for alpine/ubuntu, R--1-1..rpm + # for el*. + case "${PLATFORM}" in + alpine-*) FILE="r-VER_1_${ARCH_ID}.apk" ;; + ubuntu-*) FILE="r-VER_1_${ARCH_ID}.deb" ;; + *) FILE="R-VER-1-1.${ARCH_ID}.rpm" ;; + esac + BASE="https://s3.eu-central-003.backblazeb2.com/devxy-r-builds/${PLATFORM_ID}" : > r-versions-existing.txt for VERSION in $(cat r-versions-to-build.txt); do - if printf '%s\n' "$LISTING" | \ - awk -v arch="${ARCH_ID}" -v ver="$VERSION" 'index($0, arch) && index($0, ver) { found=1 } END { exit !found }'; then + URL="${BASE}/$(echo "$FILE" | sed "s/VER/${VERSION}/")" + if [ "$(curl -s -o /dev/null -w '%{http_code}' -I "$URL")" = "200" ]; then echo "$VERSION" >> r-versions-existing.txt fi done From 12aa310b8d4f2e403c75fbb442456387c649be8e Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 29 Jun 2026 13:00:26 +0200 Subject: [PATCH 2/8] fix(ci): detect existing R versions via public object HEAD probe The existing-version check listed the bucket with an anonymous curl, but Backblaze B2 requires authentication for the list-bucket API, so it returned AccessDenied and r-versions-existing.txt was always empty -> every version was rebuilt despite already existing in s3. Signing the request is not workable here: the B2 credentials are scoped to the plugin-s3 image in crow and are not injected into this plain alpine step. Individual artifacts are public-read, so probe each expected object URL with an anonymous HEAD instead. The filename mirrors what nfpm produces (version and release pinned to 1): r-_1_.{apk,deb} for alpine/ubuntu and R--1-1..rpm for el*. Verified against the live bucket for all three packager families. Shell variables use bare $name rather than ${name}: crow runs its own ${VAR} substitution over the commands before the shell executes, so a ${VAR} naming a shell var (BASE, VERSION) was blanked to empty -- only matrix vars resolve at that stage. Build the filename with printf to avoid the ${VERSION}_1 brace requirement entirely. --- .crow/build.yaml | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index b3b6c0e..18f126f 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -158,28 +158,29 @@ steps: - name: Check which R versions already exist in s3 image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true - environment: - AWS_ACCESS_KEY_ID: - from_secret: B2_S3_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: - from_secret: B2_S3_SECRET_KEY commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done - | - # Backblaze B2 requires authentication for the list-bucket API (anonymous - # GET works only for individual public-read objects), so the request must be - # SigV4-signed with the same credentials used for the upload step. Without - # this the listing returns AccessDenied, r-versions-existing.txt stays empty, - # and every version is rebuilt even though it already exists. - LISTING=$(curl -s --aws-sigv4 "aws:amz:eu-central-003:s3" \ - --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ - "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ - grep -oE '[^<]+' | sed 's/<[^>]*>//g') + # Backblaze B2's list-bucket API requires authentication, but the upload + # credentials are scoped to the plugin-s3 image and aren't available here. + # Individual objects are public-read, so probe each expected artifact URL + # with an anonymous HEAD instead of listing the bucket. The filename mirrors + # what nfpm produces in builder/package.${PLATFORM} (version/release pinned + # to 1): r-_1_.{apk,deb} for alpine/ubuntu, R--1-1..rpm + # for el*. Shell variables must use bare $name, not ${name}: crow performs + # its own ${VAR} substitution on these commands before the shell runs, so + # ${VAR} for a shell var would be blanked out (only matrix vars resolve there). + case "${PLATFORM}" in + alpine-*) FMT="r-%s_1_%s.apk" ;; + ubuntu-*) FMT="r-%s_1_%s.deb" ;; + *) FMT="R-%s-1-1.%s.rpm" ;; + esac + BASE="https://s3.eu-central-003.backblazeb2.com/devxy-r-builds/${PLATFORM_ID}" : > r-versions-existing.txt for VERSION in $(cat r-versions-to-build.txt); do - if printf '%s\n' "$LISTING" | \ - awk -v arch="${ARCH_ID}" -v ver="$VERSION" 'index($0, arch) && index($0, ver) { found=1 } END { exit !found }'; then + URL="$BASE/$(printf "$FMT" "$VERSION" "${ARCH_ID}")" + if [ "$(curl -s -o /dev/null -w '%{http_code}' -I "$URL")" = "200" ]; then echo "$VERSION" >> r-versions-existing.txt fi done From 8eacc0ab47fc45a93e0658d2d8193966fbd36e0c Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 29 Jun 2026 14:15:55 +0200 Subject: [PATCH 3/8] fix(ci): use bare shell vars in existing-version check (crow ${VAR} substitution) The merged HEAD-probe check still rebuilt everything: crow runs its own ${VAR} substitution over the commands before the shell executes, so the shell variables ${BASE} and ${VERSION} were blanked to empty (only matrix vars resolve at that stage). Every probed URL was therefore malformed and returned non-200, leaving r-versions-existing.txt empty. Use bare $name for shell variables and build the filename with printf, which avoids the ${VERSION}_1 brace requirement entirely. Matrix vars (${PLATFORM}, ${PLATFORM_ID}, ${ARCH_ID}) keep braces since crow resolves those correctly. Verified URL construction against the live bucket for apk, deb and rpm. --- .crow/build.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index c980b4b..18f126f 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -168,16 +168,18 @@ steps: # with an anonymous HEAD instead of listing the bucket. The filename mirrors # what nfpm produces in builder/package.${PLATFORM} (version/release pinned # to 1): r-_1_.{apk,deb} for alpine/ubuntu, R--1-1..rpm - # for el*. + # for el*. Shell variables must use bare $name, not ${name}: crow performs + # its own ${VAR} substitution on these commands before the shell runs, so + # ${VAR} for a shell var would be blanked out (only matrix vars resolve there). case "${PLATFORM}" in - alpine-*) FILE="r-VER_1_${ARCH_ID}.apk" ;; - ubuntu-*) FILE="r-VER_1_${ARCH_ID}.deb" ;; - *) FILE="R-VER-1-1.${ARCH_ID}.rpm" ;; + alpine-*) FMT="r-%s_1_%s.apk" ;; + ubuntu-*) FMT="r-%s_1_%s.deb" ;; + *) FMT="R-%s-1-1.%s.rpm" ;; esac BASE="https://s3.eu-central-003.backblazeb2.com/devxy-r-builds/${PLATFORM_ID}" : > r-versions-existing.txt for VERSION in $(cat r-versions-to-build.txt); do - URL="${BASE}/$(echo "$FILE" | sed "s/VER/${VERSION}/")" + URL="$BASE/$(printf "$FMT" "$VERSION" "${ARCH_ID}")" if [ "$(curl -s -o /dev/null -w '%{http_code}' -I "$URL")" = "200" ]; then echo "$VERSION" >> r-versions-existing.txt fi From 56d26ef59c4077e7f5222816b0a95a43ab3df4bc Mon Sep 17 00:00:00 2001 From: pat-s Date: Thu, 2 Jul 2026 08:43:18 +0200 Subject: [PATCH 4/8] docs: add CLAUDE.md with B2 storage gotchas and CI workflow conventions for agents --- CLAUDE.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..911139b --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,13 @@ +# CLAUDE.md + +CI pipelines (Crow, in `.crow/`) that build R binaries and upload them to Backblaze B2. + +## Conventions + +- **PRs:** the remote is Forgejo on `git.devxy.io`; use `fj -H git.devxy.io` (not `gh`). +- **Storage:** Backblaze B2 bucket `devxy-r-builds` (endpoint configured in `.crow/build.yaml`). + +## Gotchas + +- **B2 requires authentication for the list-bucket API.** Anonymous GET only works for individual public-read objects — an empty bucket listing usually means missing credentials, not an empty bucket. This silently broke rebuild dedup once. +- When a pipeline fails, fetch the Crow logs yourself instead of asking the user to paste them, and monitor reruns in the background until green. From 1362e0bd3a7d8eaf2b90ece71d34293b024bb28f Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 18 Sep 2026 07:04:22 +0000 Subject: [PATCH 5/8] fix(alpine): build R with cairo, png, jpeg and tiff graphics devices (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation R on the Alpine images was built without graphics support: `capabilities()` reports `cairo`, `png`, `jpeg` and `tiff` as `FALSE`, while Ubuntu reports all four `TRUE`. The images install `R-dev` but never the graphics headers, and R's `configure` treats those as optional and does not fail when they are absent, so the build succeeds and ships an R that cannot draw. Reproduced in a clean `alpine:3.22` container with the old package set: ``` Additional capabilities: PNG, NLS, ICU Capabilities skipped: JPEG, TIFF, cairo ``` One correction to the original diagnosis: `--with-x=no` was not the cause, and libpng was in fact found. R's bitmap and `pangocairo` checks run independently of `R_X11` (`configure.ac:2604` gates on `want_cairo`, not `use_X11`). `capabilities("png")` needs a device *backend*, and with X11 off and cairo absent there was none, which is why all four read `FALSE` despite configure listing PNG. So the fix is purely the missing `-dev` packages. X11 stays off, which is correct for headless images, and cairo covers `png()` / `jpeg()` / `tiff()` / `svg()`. ## Changes - `builder/Dockerfile.alpine-{320,321,322,323,324}` — install `pkgconf cairo-dev pango-dev fontconfig-dev freetype-dev libpng-dev jpeg-dev tiff-dev zlib-dev`; pass `--with-cairo --with-libpng --with-jpeglib --with-libtiff` explicitly so the intent sits next to `--with-x=no`; set `REQUIRED_CAPABILITIES`. - `builder/build.sh` — add `verify_capabilities()` between `compile_r` and `package_r`. A build that loses a graphics device now fails instead of being packaged and uploaded. Opt-in via `REQUIRED_CAPABILITIES`, so every non-Alpine platform is untouched. - `.crow/build.yaml` — add a `FORCE_REBUILD` escape hatch to the dedup step. Artifact filenames carry only the R version, so the HEAD probe would otherwise keep skipping every Alpine version and serving the stale broken binaries. - `CLAUDE.md` — record the three gotchas. ## Verification Compiled R 4.5.1 in the patched `alpine-322` image, through the real `build.sh`: ``` jpeg png tiff tcltk X11 aqua TRUE TRUE TRUE TRUE FALSE FALSE NLS Rprof profmem cairo ICU long.double TRUE TRUE TRUE TRUE TRUE TRUE All required capabilities are present. ``` The devices actually render, not just report `TRUE` (`default bitmapType: cairo`): ``` png -> t.png (2154 bytes) jpeg -> t.jpeg (4489 bytes) tiff -> t.tiff (120160 bytes) svg -> t.svg (9381 bytes) ``` The guard was exercised in both directions against that real build: it passes on the four graphics capabilities, and returns non-zero naming the missing entry when additionally asked for `X11`. `crow lint .crow/` passes and `bash -n builder/build.sh` is clean. ## Behaviour change Alpine R binaries gain the cairo, png, jpeg and tiff devices, and link against `libcairo`, `libpangocairo`, `libpng`, `libjpeg` and `libtiff`. The `.apk` metadata already declared those runtime dependencies, so consumer images need no change. Any Alpine build that loses a graphics device now fails the pipeline rather than uploading silently. **Needs a follow-up after merge:** the first run must have `FORCE_REBUILD` set, otherwise dedup skips every existing Alpine version and the broken artifacts stay in the bucket. Reviewed-on: https://git.devxy.io/devxy/r-builds/pulls/9 --- .crow/build.yaml | 22 +++++++++++++----- CLAUDE.md | 7 ++++++ builder/Dockerfile.alpine-320 | 14 +++++++++++- builder/Dockerfile.alpine-321 | 14 +++++++++++- builder/Dockerfile.alpine-322 | 14 +++++++++++- builder/Dockerfile.alpine-323 | 14 +++++++++++- builder/Dockerfile.alpine-324 | 14 +++++++++++- builder/build.sh | 42 +++++++++++++++++++++++++++++++++++ 8 files changed, 130 insertions(+), 11 deletions(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index 18f126f..acdb7a3 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -178,12 +178,22 @@ steps: esac BASE="https://s3.eu-central-003.backblazeb2.com/devxy-r-builds/${PLATFORM_ID}" : > r-versions-existing.txt - for VERSION in $(cat r-versions-to-build.txt); do - URL="$BASE/$(printf "$FMT" "$VERSION" "${ARCH_ID}")" - if [ "$(curl -s -o /dev/null -w '%{http_code}' -I "$URL")" = "200" ]; then - echo "$VERSION" >> r-versions-existing.txt - fi - done + # Escape hatch for when the builder image itself changes in a way that + # alters the resulting R binary (a newly enabled graphics device, a + # different BLAS). The artifact filename carries only the R version, so + # dedup would otherwise keep the stale binaries forever. Set FORCE_REBUILD + # to a non-empty value as a pipeline variable on a manual run, or hardcode + # it here temporarily; uploads overwrite in place. + if [ -n "$FORCE_REBUILD" ]; then + echo "FORCE_REBUILD is set, ignoring artifacts already in the bucket" + else + for VERSION in $(cat r-versions-to-build.txt); do + URL="$BASE/$(printf "$FMT" "$VERSION" "${ARCH_ID}")" + if [ "$(curl -s -o /dev/null -w '%{http_code}' -I "$URL")" = "200" ]; then + echo "$VERSION" >> r-versions-existing.txt + fi + done + fi - echo "Already present in s3:"; cat r-versions-existing.txt backend_options: kubernetes: diff --git a/CLAUDE.md b/CLAUDE.md index 911139b..9d6c279 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,4 +10,11 @@ CI pipelines (Crow, in `.crow/`) that build R binaries and upload them to Backbl ## Gotchas - **B2 requires authentication for the list-bucket API.** Anonymous GET only works for individual public-read objects — an empty bucket listing usually means missing credentials, not an empty bucket. This silently broke rebuild dedup once. +- **R's configure silently disables graphics devices when headers are missing.** +It never fails the build, so a missing `-dev` package yields a working R whose `capabilities()` reports `cairo`, `png`, `jpeg` and `tiff` as `FALSE`. +Installing the runtime libraries on the target system afterwards cannot bring compiled-out devices back; the binary has to be rebuilt. +`builder/build.sh` therefore asserts `REQUIRED_CAPABILITIES` (set per Dockerfile) after `make install`. +- **The `--with-x=no` on the Alpine images does not disable cairo.** R's bitmap and pangocairo checks run independently of `R_X11`, so `png()` and `svg()` work headless via cairo. +- **Artifact filenames carry only the R version, not the builder image.** +When a builder change alters the resulting binary, dedup would keep serving the stale artifact forever; set `FORCE_REBUILD` in `.crow/build.yaml` to rebuild and overwrite in place. - When a pipeline fails, fetch the Crow logs yourself instead of asking the user to paste them, and monitor reruns in the background until green. diff --git a/builder/Dockerfile.alpine-320 b/builder/Dockerfile.alpine-320 index b83ef43..726fa04 100644 --- a/builder/Dockerfile.alpine-320 +++ b/builder/Dockerfile.alpine-320 @@ -5,6 +5,13 @@ ENV OS_IDENTIFIER alpine-320 RUN set -x \ && apk add -q R-dev curl ca-certificates bash g++ tzdata openjdk17 texmf-dist texlive-full tar sed patch tcl tk tk-dev xvfb libdeflate libdeflate-dev +# Graphics devices. R's configure only enables the cairo/png/jpeg/tiff devices +# when the corresponding headers are present at compile time, and pkgconf must +# exist for the pangocairo check to run at all. Without these, capabilities() +# reports cairo/png/jpeg/tiff as FALSE, and installing the runtime libraries +# afterwards cannot bring the compiled-out devices back. +RUN apk add -q pkgconf cairo-dev pango-dev fontconfig-dev freetype-dev libpng-dev jpeg-dev tiff-dev zlib-dev + # Install s5cmd for S3 uploads (much faster than AWS CLI) RUN ARCH=$(uname -m); if [ "$ARCH" = "x86_64" ]; then S5CMD_ARCH="64bit"; else S5CMD_ARCH="arm64"; fi && \ curl -sL "https://github.com/peak/s5cmd/releases/download/v2.3.0/s5cmd_2.3.0_Linux-${S5CMD_ARCH}.tar.gz" | tar xz -C /usr/local/bin s5cmd @@ -18,7 +25,12 @@ RUN chmod 0777 /opt # Override the default pager used by R ENV PAGER /usr/bin/pager -ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-blas --with-lapack" +# X11 stays off: these are headless images, and the cairo/bitmap devices are +# detected independently of X11, so png()/svg() still work via cairo. +ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-cairo --with-libpng --with-jpeglib --with-libtiff --with-blas --with-lapack" + +# Fail the build if a graphics device silently drops out again. +ENV REQUIRED_CAPABILITIES "cairo,png,jpeg,tiff" COPY package.alpine-320 /package.sh COPY build.sh . diff --git a/builder/Dockerfile.alpine-321 b/builder/Dockerfile.alpine-321 index 5d86190..e2298ef 100644 --- a/builder/Dockerfile.alpine-321 +++ b/builder/Dockerfile.alpine-321 @@ -5,6 +5,13 @@ ENV OS_IDENTIFIER alpine-321 RUN set -x \ && apk add -q R-dev curl ca-certificates bash g++ tzdata openjdk17 texmf-dist texlive-full tar sed patch tcl tk tk-dev xvfb libdeflate libdeflate-dev +# Graphics devices. R's configure only enables the cairo/png/jpeg/tiff devices +# when the corresponding headers are present at compile time, and pkgconf must +# exist for the pangocairo check to run at all. Without these, capabilities() +# reports cairo/png/jpeg/tiff as FALSE, and installing the runtime libraries +# afterwards cannot bring the compiled-out devices back. +RUN apk add -q pkgconf cairo-dev pango-dev fontconfig-dev freetype-dev libpng-dev jpeg-dev tiff-dev zlib-dev + # Install s5cmd for S3 uploads (much faster than AWS CLI) RUN ARCH=$(uname -m); if [ "$ARCH" = "x86_64" ]; then S5CMD_ARCH="64bit"; else S5CMD_ARCH="arm64"; fi && \ curl -sL "https://github.com/peak/s5cmd/releases/download/v2.3.0/s5cmd_2.3.0_Linux-${S5CMD_ARCH}.tar.gz" | tar xz -C /usr/local/bin s5cmd @@ -18,7 +25,12 @@ RUN chmod 0777 /opt # Override the default pager used by R ENV PAGER /usr/bin/pager -ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-blas --with-lapack" +# X11 stays off: these are headless images, and the cairo/bitmap devices are +# detected independently of X11, so png()/svg() still work via cairo. +ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-cairo --with-libpng --with-jpeglib --with-libtiff --with-blas --with-lapack" + +# Fail the build if a graphics device silently drops out again. +ENV REQUIRED_CAPABILITIES "cairo,png,jpeg,tiff" COPY package.alpine-321 /package.sh COPY build.sh . diff --git a/builder/Dockerfile.alpine-322 b/builder/Dockerfile.alpine-322 index 0a3695a..23a7793 100644 --- a/builder/Dockerfile.alpine-322 +++ b/builder/Dockerfile.alpine-322 @@ -5,6 +5,13 @@ ENV OS_IDENTIFIER alpine-322 RUN set -x \ && apk add -q R-dev curl ca-certificates bash g++ tzdata openjdk17 texmf-dist texlive-full tar sed patch tcl tk tk-dev xvfb libdeflate libdeflate-dev +# Graphics devices. R's configure only enables the cairo/png/jpeg/tiff devices +# when the corresponding headers are present at compile time, and pkgconf must +# exist for the pangocairo check to run at all. Without these, capabilities() +# reports cairo/png/jpeg/tiff as FALSE, and installing the runtime libraries +# afterwards cannot bring the compiled-out devices back. +RUN apk add -q pkgconf cairo-dev pango-dev fontconfig-dev freetype-dev libpng-dev jpeg-dev tiff-dev zlib-dev + # Install s5cmd for S3 uploads (much faster than AWS CLI) RUN ARCH=$(uname -m); if [ "$ARCH" = "x86_64" ]; then S5CMD_ARCH="64bit"; else S5CMD_ARCH="arm64"; fi && \ curl -sL "https://github.com/peak/s5cmd/releases/download/v2.3.0/s5cmd_2.3.0_Linux-${S5CMD_ARCH}.tar.gz" | tar xz -C /usr/local/bin s5cmd @@ -18,7 +25,12 @@ RUN chmod 0777 /opt # Override the default pager used by R ENV PAGER /usr/bin/pager -ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-blas --with-lapack" +# X11 stays off: these are headless images, and the cairo/bitmap devices are +# detected independently of X11, so png()/svg() still work via cairo. +ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-cairo --with-libpng --with-jpeglib --with-libtiff --with-blas --with-lapack" + +# Fail the build if a graphics device silently drops out again. +ENV REQUIRED_CAPABILITIES "cairo,png,jpeg,tiff" COPY package.alpine-322 /package.sh COPY build.sh . diff --git a/builder/Dockerfile.alpine-323 b/builder/Dockerfile.alpine-323 index 9e7fa0b..82b8c5a 100644 --- a/builder/Dockerfile.alpine-323 +++ b/builder/Dockerfile.alpine-323 @@ -5,6 +5,13 @@ ENV OS_IDENTIFIER alpine-323 RUN set -x \ && apk add -q R-dev curl ca-certificates bash g++ tzdata openjdk17 texmf-dist texlive-full tar sed patch tcl tk tk-dev xvfb libdeflate libdeflate-dev +# Graphics devices. R's configure only enables the cairo/png/jpeg/tiff devices +# when the corresponding headers are present at compile time, and pkgconf must +# exist for the pangocairo check to run at all. Without these, capabilities() +# reports cairo/png/jpeg/tiff as FALSE, and installing the runtime libraries +# afterwards cannot bring the compiled-out devices back. +RUN apk add -q pkgconf cairo-dev pango-dev fontconfig-dev freetype-dev libpng-dev jpeg-dev tiff-dev zlib-dev + # Install s5cmd for S3 uploads (much faster than AWS CLI) RUN ARCH=$(uname -m); if [ "$ARCH" = "x86_64" ]; then S5CMD_ARCH="64bit"; else S5CMD_ARCH="arm64"; fi && \ curl -sL "https://github.com/peak/s5cmd/releases/download/v2.3.0/s5cmd_2.3.0_Linux-${S5CMD_ARCH}.tar.gz" | tar xz -C /usr/local/bin s5cmd @@ -18,7 +25,12 @@ RUN chmod 0777 /opt # Override the default pager used by R ENV PAGER /usr/bin/pager -ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-blas --with-lapack" +# X11 stays off: these are headless images, and the cairo/bitmap devices are +# detected independently of X11, so png()/svg() still work via cairo. +ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-cairo --with-libpng --with-jpeglib --with-libtiff --with-blas --with-lapack" + +# Fail the build if a graphics device silently drops out again. +ENV REQUIRED_CAPABILITIES "cairo,png,jpeg,tiff" COPY package.alpine-322 /package.sh COPY build.sh . diff --git a/builder/Dockerfile.alpine-324 b/builder/Dockerfile.alpine-324 index b3daa01..c95ce40 100644 --- a/builder/Dockerfile.alpine-324 +++ b/builder/Dockerfile.alpine-324 @@ -5,6 +5,13 @@ ENV OS_IDENTIFIER alpine-324 RUN set -x \ && apk add -q R-dev curl ca-certificates bash g++ tzdata openjdk17 texmf-dist texlive-full tar sed patch tcl tk tk-dev xvfb libdeflate libdeflate-dev +# Graphics devices. R's configure only enables the cairo/png/jpeg/tiff devices +# when the corresponding headers are present at compile time, and pkgconf must +# exist for the pangocairo check to run at all. Without these, capabilities() +# reports cairo/png/jpeg/tiff as FALSE, and installing the runtime libraries +# afterwards cannot bring the compiled-out devices back. +RUN apk add -q pkgconf cairo-dev pango-dev fontconfig-dev freetype-dev libpng-dev jpeg-dev tiff-dev zlib-dev + # Install s5cmd for S3 uploads (much faster than AWS CLI) RUN ARCH=$(uname -m); if [ "$ARCH" = "x86_64" ]; then S5CMD_ARCH="64bit"; else S5CMD_ARCH="arm64"; fi && \ curl -sL "https://github.com/peak/s5cmd/releases/download/v2.3.0/s5cmd_2.3.0_Linux-${S5CMD_ARCH}.tar.gz" | tar xz -C /usr/local/bin s5cmd @@ -18,7 +25,12 @@ RUN chmod 0777 /opt # Override the default pager used by R ENV PAGER /usr/bin/pager -ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-blas --with-lapack" +# X11 stays off: these are headless images, and the cairo/bitmap devices are +# detected independently of X11, so png()/svg() still work via cairo. +ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-cairo --with-libpng --with-jpeglib --with-libtiff --with-blas --with-lapack" + +# Fail the build if a graphics device silently drops out again. +ENV REQUIRED_CAPABILITIES "cairo,png,jpeg,tiff" COPY package.alpine-324 /package.sh COPY build.sh . diff --git a/builder/build.sh b/builder/build.sh index 427da6e..7a8e59e 100755 --- a/builder/build.sh +++ b/builder/build.sh @@ -209,6 +209,47 @@ local({ EOF } +# verify_capabilities() +# Guard against a graphics device being silently compiled out. R's configure +# only enables the cairo/png/jpeg/tiff devices when the matching headers were +# present at compile time, and it does not fail when they are missing, so a +# missing -dev package produces a working R that simply cannot draw. Once that +# tarball is published, installing the runtime libraries on the target system +# cannot bring the devices back; the build has to be redone. +# +# Dockerfiles opt in by setting REQUIRED_CAPABILITIES to a comma-separated list +# of capabilities() names, e.g. "cairo,png,jpeg,tiff". +verify_capabilities() { + if [ -z "${REQUIRED_CAPABILITIES}" ]; then + return 0 + fi + + echo "Verifying required R capabilities: ${REQUIRED_CAPABILITIES}" + + # The full capabilities() table goes to stderr so it shows up in the build + # log; stdout carries only the space-separated list of missing entries. + local missing + missing=$("${R_INSTALL_PATH}/bin/Rscript" --vanilla -e ' + required <- trimws(strsplit(Sys.getenv("REQUIRED_CAPABILITIES"), ",")[[1]]) + required <- required[nzchar(required)] + caps <- capabilities() + writeLines(capture.output(print(caps)), con = stderr()) + ok <- (required %in% names(caps)) & (unname(caps[required]) %in% TRUE) + cat(paste(required[!ok], collapse = " "), "\n", sep = "") + ') || { + echo "ERROR: could not query capabilities() from the freshly built R" >&2 + return 1 + } + + if [ -n "${missing}" ]; then + echo "ERROR: R ${R_VERSION} for ${OS_IDENTIFIER} is missing required capabilities: ${missing}" >&2 + echo "ERROR: install the matching -dev packages in builder/Dockerfile.${OS_IDENTIFIER} and rebuild." >&2 + return 1 + fi + + echo "All required capabilities are present." +} + # check for packager script ## If it exists this build is ready for packaging with nFPM, so run the script ## else do nothing @@ -236,6 +277,7 @@ set_up_environment fetch_r_source $R_VERSION patch_r $R_VERSION compile_r $R_VERSION +verify_capabilities package_r $R_VERSION archive_r upload_r $R_VERSION From 3028a730f06937a525980868a5aea9bb9b40e18f Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 18 Sep 2026 07:14:17 +0000 Subject: [PATCH 6/8] fix(alpine): declare xz-dev so R can start after a clean install (#10) --- builder/package.alpine-320 | 5 ++++- builder/package.alpine-321 | 5 ++++- builder/package.alpine-322 | 5 ++++- builder/package.alpine-323 | 5 ++++- builder/package.alpine-324 | 5 ++++- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/builder/package.alpine-320 b/builder/package.alpine-320 index 2e0b28e..d8dacf6 100644 --- a/builder/package.alpine-320 +++ b/builder/package.alpine-320 @@ -42,7 +42,10 @@ ${deflate_libs} # - libgomp1 - icu-dev - jpeg-dev -# - liblzma-dev +# liblzma-dev is the Debian name and does not exist on Alpine; xz-dev is the +# equivalent and pulls xz-libs, which owns liblzma.so.5. Without it libR.so +# cannot relocate its lzma symbols and R fails to start at all. +- xz-dev - openblas-dev - pango # - libpangocairo-1.0-0 diff --git a/builder/package.alpine-321 b/builder/package.alpine-321 index 2e0b28e..d8dacf6 100644 --- a/builder/package.alpine-321 +++ b/builder/package.alpine-321 @@ -42,7 +42,10 @@ ${deflate_libs} # - libgomp1 - icu-dev - jpeg-dev -# - liblzma-dev +# liblzma-dev is the Debian name and does not exist on Alpine; xz-dev is the +# equivalent and pulls xz-libs, which owns liblzma.so.5. Without it libR.so +# cannot relocate its lzma symbols and R fails to start at all. +- xz-dev - openblas-dev - pango # - libpangocairo-1.0-0 diff --git a/builder/package.alpine-322 b/builder/package.alpine-322 index 2e0b28e..d8dacf6 100644 --- a/builder/package.alpine-322 +++ b/builder/package.alpine-322 @@ -42,7 +42,10 @@ ${deflate_libs} # - libgomp1 - icu-dev - jpeg-dev -# - liblzma-dev +# liblzma-dev is the Debian name and does not exist on Alpine; xz-dev is the +# equivalent and pulls xz-libs, which owns liblzma.so.5. Without it libR.so +# cannot relocate its lzma symbols and R fails to start at all. +- xz-dev - openblas-dev - pango # - libpangocairo-1.0-0 diff --git a/builder/package.alpine-323 b/builder/package.alpine-323 index 2e0b28e..d8dacf6 100644 --- a/builder/package.alpine-323 +++ b/builder/package.alpine-323 @@ -42,7 +42,10 @@ ${deflate_libs} # - libgomp1 - icu-dev - jpeg-dev -# - liblzma-dev +# liblzma-dev is the Debian name and does not exist on Alpine; xz-dev is the +# equivalent and pulls xz-libs, which owns liblzma.so.5. Without it libR.so +# cannot relocate its lzma symbols and R fails to start at all. +- xz-dev - openblas-dev - pango # - libpangocairo-1.0-0 diff --git a/builder/package.alpine-324 b/builder/package.alpine-324 index 2e0b28e..d8dacf6 100644 --- a/builder/package.alpine-324 +++ b/builder/package.alpine-324 @@ -42,7 +42,10 @@ ${deflate_libs} # - libgomp1 - icu-dev - jpeg-dev -# - liblzma-dev +# liblzma-dev is the Debian name and does not exist on Alpine; xz-dev is the +# equivalent and pulls xz-libs, which owns liblzma.so.5. Without it libR.so +# cannot relocate its lzma symbols and R fails to start at all. +- xz-dev - openblas-dev - pango # - libpangocairo-1.0-0 From 02dbe3ad5f61d3b121802385ed55d73eb50dcef3 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 18 Sep 2026 07:14:23 +0000 Subject: [PATCH 7/8] fix(ci): honour FORCE_REBUILD from both manual trigger forms (#11) --- .crow/build.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index acdb7a3..b10a930 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -181,10 +181,18 @@ steps: # Escape hatch for when the builder image itself changes in a way that # alters the resulting R binary (a newly enabled graphics device, a # different BLAS). The artifact filename carries only the R version, so - # dedup would otherwise keep the stale binaries forever. Set FORCE_REBUILD - # to a non-empty value as a pipeline variable on a manual run, or hardcode - # it here temporarily; uploads overwrite in place. - if [ -n "$FORCE_REBUILD" ]; then + # dedup would otherwise keep the stale binaries forever. Uploads + # overwrite in place. + # + # The two ways to trigger a manual run pass the flag differently: + # crow pipeline create --branch main --var FORCE_REBUILD=1 + # -> a crow substitution variable, read below as ${FORCE_REBUILD}, + # which crow blanks out when it was not supplied + # crow pipeline start -p FORCE_REBUILD=1 + # -> injected into the step environment, read as bare $FORCE_REBUILD + # Accept either, so the flag cannot be silently ignored. + FORCE_REBUILD_VAR="${FORCE_REBUILD}" + if [ -n "$FORCE_REBUILD_VAR" ] || [ -n "$FORCE_REBUILD" ]; then echo "FORCE_REBUILD is set, ignoring artifacts already in the bucket" else for VERSION in $(cat r-versions-to-build.txt); do From 8c2e3925875fe8d2bc0993764ae02749aec6a6d2 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 18 Sep 2026 07:18:08 +0000 Subject: [PATCH 8/8] fix(ci): declare FORCE_REBUILD as a UI variable and make it take effect (#12) --- .crow/build.yaml | 90 ++++++++++++++++++------------------------------ 1 file changed, 33 insertions(+), 57 deletions(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index b10a930..b970235 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -1,6 +1,18 @@ when: event: [cron, manual] +variables: + FORCE_REBUILD: + description: >- + Rebuild every targeted R version even when the artifact is already in the + bucket. Needed after a builder-image change that alters the resulting R + binary, because artifact filenames carry only the R version and would + otherwise dedup to the stale build forever. + options: + - "false" + - "true" + default: "false" + matrix: include: - PLATFORM: rhel-9 @@ -82,46 +94,6 @@ matrix: ARCH_ID: x86_64 steps: - # Ensures that we're always using the latest available upstream build chain - # - name: Pull upstream - # image: reg.devxy.io/docker.io/library/alpine:3.20 - # commands: - # - apk add -q --no-cache git - # - git remote add upstream https://github.com/cynkra/r-builds.git - # - git config --global user.email "you@example.com" - # - git config --global user.name "Your Name" - # - git config pull.ff only - # - git fetch upstream - # - git merge upstream/main --allow-unrelated-histories - #. - - - name: Net debug - image: reg.devxy.io/docker.io/library/alpine:3.23 - privileged: true - commands: - - ip link set dev eth0 mtu 1280 2>/dev/null || true - - echo "=== pod interface ===" - - cat /sys/class/net/eth0/mtu - - ip -o link show eth0 - - echo "=== route ===" - - ip route - - echo "=== resolv.conf ===" - - cat /etc/resolv.conf - - for i in 1 2 3; do apk add -q --no-cache iputils && break; sleep 5; done || echo "apk failed; falling back to busybox ping (no -M do)" - - | - echo "=== PMTUD probes (DF=1) to dl-cdn.alpinelinux.org ===" - for sz in 1472 1392 1352 1252 1200; do - echo "--- payload=$sz (wire ~$((sz+28))) ---" - ping -c 2 -W 3 -M do -s $sz dl-cdn.alpinelinux.org || echo "(size=$sz failed)" - done - - ping -c 3 -W 5 dl-cdn.alpinelinux.org || true - - wget --tries=1 --timeout=10 -O /dev/null -nv https://dl-cdn.alpinelinux.org/ 2>&1 || true - backend_options: - kubernetes: - nodeSelector: - kubernetes.io/arch: "${ARCH}" - - - name: Get R versions to build image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true @@ -178,22 +150,23 @@ steps: esac BASE="https://s3.eu-central-003.backblazeb2.com/devxy-r-builds/${PLATFORM_ID}" : > r-versions-existing.txt - # Escape hatch for when the builder image itself changes in a way that - # alters the resulting R binary (a newly enabled graphics device, a - # different BLAS). The artifact filename carries only the R version, so - # dedup would otherwise keep the stale binaries forever. Uploads - # overwrite in place. - # - # The two ways to trigger a manual run pass the flag differently: - # crow pipeline create --branch main --var FORCE_REBUILD=1 - # -> a crow substitution variable, read below as ${FORCE_REBUILD}, - # which crow blanks out when it was not supplied - # crow pipeline start -p FORCE_REBUILD=1 + # FORCE_REBUILD is declared as a top-level variable, so the UI offers it + # as a dropdown on a manual run. The two trigger paths deliver it + # differently, so accept either: + # UI, or crow pipeline create --var FORCE_REBUILD=true + # -> a substitution variable, expanded into ${FORCE_REBUILD} below + # crow pipeline start -p FORCE_REBUILD=true # -> injected into the step environment, read as bare $FORCE_REBUILD - # Accept either, so the flag cannot be silently ignored. - FORCE_REBUILD_VAR="${FORCE_REBUILD}" - if [ -n "$FORCE_REBUILD_VAR" ] || [ -n "$FORCE_REBUILD" ]; then - echo "FORCE_REBUILD is set, ignoring artifacts already in the bucket" + # The declared default is the string "false", so test the value rather + # than emptiness; an empty value means neither path supplied one. + FORCE_VALUE="${FORCE_REBUILD}" + [ -n "$FORCE_VALUE" ] || FORCE_VALUE="$FORCE_REBUILD" + case "$FORCE_VALUE" in + true|TRUE|True|1|yes|YES) FORCE_ON=1 ;; + *) FORCE_ON="" ;; + esac + if [ -n "$FORCE_ON" ]; then + echo "FORCE_REBUILD=$FORCE_VALUE, ignoring artifacts already in the bucket" else for VERSION in $(cat r-versions-to-build.txt); do URL="$BASE/$(printf "$FMT" "$VERSION" "${ARCH_ID}")" @@ -217,7 +190,6 @@ steps: DOCKER_HOST: tcp://docker:2375 commands: | ip link set dev eth0 mtu 1280 2>/dev/null || true - # docker info TO_BUILD="" for VERSION in $(cat r-versions-to-build.txt); do if grep -qxF "$VERSION" r-versions-existing.txt; then @@ -263,7 +235,11 @@ steps: from_secret: B2_S3_ACCESS_KEY_ID secret_key: from_secret: B2_S3_SECRET_KEY - overwrite: false + # Must stay true for FORCE_REBUILD to mean anything: with overwrite=false + # the plugin logs "File already exists, skipping upload" and still exits + # green, so a rebuild would be discarded. Normal runs are unaffected, + # since dedup means only not-yet-published versions are ever built. + overwrite: true volumes: - ci-tmp-rwx:/tmp # agent on k8s # - /tmp:/tmp # agent on VM