From d34684260f841db944569b5d7ffa01b15b757b49 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 12 Jun 2026 13:46:11 +0000 Subject: [PATCH 1/3] feat: add alpine-3.24 build config (#5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds an `alpine-3.24` build configuration, mirroring the existing `alpine-3.23` setup. - `builder/Dockerfile.alpine-324` — `FROM alpine:3.24`, `OS_IDENTIFIER=alpine-324`, copies `package.alpine-324` - `builder/package.alpine-324` — nfpm spec (identical to 323) - `builder/docker-compose.yml` — new `alpine-324` service - `Makefile` `PLATFORMS` and `Justfile` `build-r-alpine-324` target - `.crow/build.yaml` — alpine-324 matrix entries (arm64 + amd64), newest-first Existing alpine-322/323 matrix entries are left intact. Reviewed-on: https://git.devxy.io/devxy/r-builds/pulls/5 --- .crow/build.yaml | 8 ++++ Justfile | 5 +++ Makefile | 2 +- builder/Dockerfile.alpine-324 | 26 ++++++++++++ builder/docker-compose.yml | 13 ++++++ builder/package.alpine-324 | 74 +++++++++++++++++++++++++++++++++++ 6 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 builder/Dockerfile.alpine-324 create mode 100644 builder/package.alpine-324 diff --git a/.crow/build.yaml b/.crow/build.yaml index 57478f7..b573f23 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -56,6 +56,14 @@ matrix: # ARCH: arm64 # ARCH_ID: aarch64 # INSTANCE_TYPE: cax31 + - PLATFORM: alpine-324 + PLATFORM_ID: alpine324 + ARCH: arm64 + ARCH_ID: aarch64 + - PLATFORM: alpine-324 + PLATFORM_ID: alpine324 + ARCH: amd64 + ARCH_ID: x86_64 - PLATFORM: alpine-323 PLATFORM_ID: alpine323 ARCH: arm64 diff --git a/Justfile b/Justfile index 1a9d969..d402218 100644 --- a/Justfile +++ b/Justfile @@ -47,3 +47,8 @@ build-r-alpine-323 R_VERSION: export PLATFORM=alpine-323; \ export R_VERSION={{R_VERSION}}; \ make build-r-$PLATFORM + +build-r-alpine-324 R_VERSION: + export PLATFORM=alpine-324; \ + export R_VERSION={{R_VERSION}}; \ + make build-r-$PLATFORM diff --git a/Makefile b/Makefile index 4b3bde4..82bdcc0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PLATFORMS := ubuntu-2004 ubuntu-2204 ubuntu-2404 ubuntu-2604 debian-10 debian-11 debian-12 centos-7 centos-8 rhel-9 opensuse-155 fedora-38 fedora-39 fedora-40 alpine-321 alpine-322 alpine-323 rhel-10 +PLATFORMS := ubuntu-2004 ubuntu-2204 ubuntu-2404 ubuntu-2604 debian-10 debian-11 debian-12 centos-7 centos-8 rhel-9 opensuse-155 fedora-38 fedora-39 fedora-40 alpine-321 alpine-322 alpine-323 alpine-324 rhel-10 SLS_BINARY ?= ./node_modules/serverless/bin/serverless.js deps: diff --git a/builder/Dockerfile.alpine-324 b/builder/Dockerfile.alpine-324 new file mode 100644 index 0000000..b3daa01 --- /dev/null +++ b/builder/Dockerfile.alpine-324 @@ -0,0 +1,26 @@ +FROM reg.devxy.io/docker.io/library/alpine:3.24 + +ENV OS_IDENTIFIER alpine-324 + +RUN set -x \ + && apk add -q R-dev curl ca-certificates bash g++ tzdata openjdk17 texmf-dist texlive-full tar sed patch tcl tk tk-dev xvfb libdeflate libdeflate-dev + +# Install s5cmd for S3 uploads (much faster than AWS CLI) +RUN ARCH=$(uname -m); if [ "$ARCH" = "x86_64" ]; then S5CMD_ARCH="64bit"; else S5CMD_ARCH="arm64"; fi && \ + curl -sL "https://github.com/peak/s5cmd/releases/download/v2.3.0/s5cmd_2.3.0_Linux-${S5CMD_ARCH}.tar.gz" | tar xz -C /usr/local/bin s5cmd + +RUN curl -LO "https://github.com/goreleaser/nfpm/releases/download/v2.39.0/nfpm_2.39.0_$(arch).apk" && \ + apk add --allow-untrusted "./nfpm_2.39.0_$(arch).apk" && \ + rm "nfpm_2.39.0_$(arch).apk" + +RUN chmod 0777 /opt + +# Override the default pager used by R +ENV PAGER /usr/bin/pager + +ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-blas --with-lapack" + +COPY package.alpine-324 /package.sh +COPY build.sh . +COPY patches /patches +ENTRYPOINT ./build.sh diff --git a/builder/docker-compose.yml b/builder/docker-compose.yml index ee72ab7..a44093e 100644 --- a/builder/docker-compose.yml +++ b/builder/docker-compose.yml @@ -96,6 +96,19 @@ services: volumes: - /tmp/alpine-323:/tmp/output/alpine-323 - ./build.sh:/build.sh:ro + alpine-324: + command: ./build.sh + environment: + - R_VERSION=${R_VERSION} + - R_INSTALL_PATH=${R_INSTALL_PATH} + - LOCAL_STORE=/tmp/output + build: + context: . + dockerfile: Dockerfile.alpine-324 + image: r-builds:alpine-324 + volumes: + - /tmp/alpine-324:/tmp/output/alpine-324 + - ./build.sh:/build.sh:ro debian-10: command: ./build.sh environment: diff --git a/builder/package.alpine-324 b/builder/package.alpine-324 new file mode 100644 index 0000000..2e0b28e --- /dev/null +++ b/builder/package.alpine-324 @@ -0,0 +1,74 @@ +#!/bin/bash + +if [[ ! -d /tmp/output/${OS_IDENTIFIER} ]]; then + mkdir -p "/tmp/output/${OS_IDENTIFIER}" +fi + +# R 3.x requires PCRE1. On Ubuntu 24, R 3.x also requires PCRE2 for Pango support. +pcre_libs='- pcre2-dev' +if [[ "${R_VERSION}" =~ ^3 ]]; then + pcre_libs='- pcre2-dev +- libpcre3-dev' +fi + +deflate_libs='# - libdeflate-dev' +if grep -q '^LIBS *=.*[-]ldeflate' ${R_INSTALL_PATH}/lib/R/etc/Makeconf; then + deflate_libs='- libdeflate-dev' +fi + +cat < /tmp/nfpm.yml +name: r-${R_VERSION} +version: 1 +version_schema: none +section: universe/math +priority: optional +arch: $(arch) +maintainer: Posit Software, PBC +description: | + GNU R statistical computation and graphics system +vendor: Posit Software, PBC +homepage: https://www.r-project.org +license: GPL-2 +depends: +- g++ +- gcc +- gfortran +- libbz2 +# - libc6 +- cairo +- libcurl +${deflate_libs} +# - libglib2.0-0t64 +# - libgomp1 +- icu-dev +- jpeg-dev +# - liblzma-dev +- openblas-dev +- pango +# - libpangocairo-1.0-0 +- libpaper +${pcre_libs} +- libpng-dev +- readline-dev +- tcl +- tiff-dev +- libtirpc-dev +- tk +# - libx11-6 +# - libxt6t64 +- make +# - ucf +- unzip +- zip +- zlib-ng-dev +contents: +- src: ${R_INSTALL_PATH} + dst: ${R_INSTALL_PATH} +EOF + +nfpm package \ + -f /tmp/nfpm.yml \ + -p apk \ + -t "/tmp/output/${OS_IDENTIFIER}" + +export PKG_FILE=$(ls /tmp/output/${OS_IDENTIFIER}/r-${R_VERSION}*.apk | head -1) From 56c75e2df060fc325b0ea432fe844d8f624fb9ae Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 12 Jun 2026 17:16:43 +0000 Subject: [PATCH 2/3] ci: build last four minor R versions instead of only the latest (#6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The crow build pipeline previously built only the single latest stable R version (from the brew formula). It now builds the latest patch release of each of the **last four minor R versions** and uploads any that are missing from S3, so older supported minors stay available rather than only the newest stable. - **Get R versions to build**: derive the four versions from posit's `versions.json` (the same source upstream's `test/get_r_versions.py` uses) via jq — keep only `x.y.z` releases, take the latest patch per minor, then the top four minors (e.g. `4.6.0, 4.5.3, 4.4.3, 4.3.3`). - **Check which R versions already exist in s3**: iterate the four versions against the S3 listing for the platform/arch. - **Build**: build only the versions that are missing. In steady state nothing builds; when a new minor lands, only that one is built. ## awk fix The existence check used an awk regex (`$0 ~ ver`), so the unescaped dots in a version made `4.4.3` spuriously match `4.3.3` (the substring `4/4.3` satisfies the pattern). Harmless with a single version, but across four adjacent versions it would wrongly skip builds. Switched to literal substring matching via awk `index($0, ver)`. Reviewed-on: https://git.devxy.io/devxy/r-builds/pulls/6 --- .crow/build.yaml | 52 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index b573f23..1d7cb47 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -122,15 +122,26 @@ steps: kubernetes.io/arch: "${ARCH}" - - name: Get latest R version + - name: Get R versions to build image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl jq && break; sleep 5; done - - curl -sf https://formulae.brew.sh/api/formula/r.json | jq -er '.versions.stable' > r-version-to-build.txt - # - echo "4.1.3" > r-version-to-build.txt - - cat r-version-to-build.txt + # Latest patch release of each of the last 4 minor R versions (e.g. 4.6.0, 4.5.3, 4.4.3, 4.3.3). + - | + curl -sf https://cdn.posit.co/r/versions.json | jq -r ' + .r_versions + | map(select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))) + | group_by(split(".")[0:2] | join(".")) + | map(max_by(split(".") | map(tonumber))) + | sort_by(split(".") | map(tonumber)) + | reverse + | .[0:4] + | .[]' > r-versions-to-build.txt + # To pin specific versions for testing, overwrite the file, e.g.: + # - printf '4.1.3\n' > r-versions-to-build.txt + - cat r-versions-to-build.txt backend_options: kubernetes: resources: @@ -144,19 +155,23 @@ steps: kubernetes.io/arch: "${ARCH}" - - name: Check if files for R version already exist in s3 + - name: Check which R versions already exist in s3 image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done - | - VERSION=$(cat r-version-to-build.txt) - curl -s "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ - grep -oE '[^<]+' | sed 's/<[^>]*>//g' | \ - awk -v arch="${ARCH_ID}" -v ver="${VERSION}" \ - '$0 ~ arch && $0 ~ ver {print substr($NF, 3, 5)}' > r-version-s3.txt - - cat r-version-s3.txt + LISTING=$(curl -s "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ + grep -oE '[^<]+' | sed 's/<[^>]*>//g') + : > r-versions-existing.txt + for VERSION in $(cat r-versions-to-build.txt); do + if printf '%s\n' "$LISTING" | \ + awk -v arch="${ARCH_ID}" -v ver="$VERSION" 'index($0, arch) && index($0, ver) { found=1 } END { exit !found }'; then + echo "$VERSION" >> r-versions-existing.txt + fi + done + - echo "Already present in s3:"; cat r-versions-existing.txt backend_options: kubernetes: nodeSelector: @@ -172,9 +187,20 @@ steps: commands: | ip link set dev eth0 mtu 1280 2>/dev/null || true # docker info - if ! grep -qF "$(cat r-version-to-build.txt)" r-version-s3.txt; then + TO_BUILD="" + for VERSION in $(cat r-versions-to-build.txt); do + if grep -qxF "$VERSION" r-versions-existing.txt; then + echo "R $VERSION already exists for ${PLATFORM} (${ARCH_ID}), skipping" + else + TO_BUILD="$TO_BUILD $VERSION" + fi + done + if [ -n "$TO_BUILD" ]; then for i in 1 2 3 4 5; do apk add -q --no-cache make just docker-compose && break; sleep 5; done - just build-r-${PLATFORM} $(cat r-version-to-build.txt) + for VERSION in $TO_BUILD; do + echo "Building R $VERSION for ${PLATFORM}" + just build-r-${PLATFORM} "$VERSION" + done fi backend_options: kubernetes: From eef805bd76d637d463027247fde03d9bd64f5e63 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 29 Jun 2026 13:00:26 +0200 Subject: [PATCH 3/3] fix(ci): authenticate B2 list-bucket request in existing-version check Backblaze B2 requires authentication for the list-bucket API; anonymous access only works for individual public-read objects. The unauthenticated curl returned AccessDenied, so r-version-s3.txt was always empty and every R version was rebuilt despite already existing in s3. Sign the listing request with --aws-sigv4 using the same B2 credentials as the upload step. Also fix the broken version extraction: the old substr($NF, 3, 5) ran against the prefixed key and yielded garbage, so the Build step's grep could never match. Print the actual version on a match and use a literal index() instead of a regex comparison. --- .crow/build.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index 1d7cb47..b3b6c0e 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -158,11 +158,23 @@ steps: - name: Check which R versions already exist in s3 image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true + environment: + AWS_ACCESS_KEY_ID: + from_secret: B2_S3_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: + from_secret: B2_S3_SECRET_KEY commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done - | - LISTING=$(curl -s "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ + # Backblaze B2 requires authentication for the list-bucket API (anonymous + # GET works only for individual public-read objects), so the request must be + # SigV4-signed with the same credentials used for the upload step. Without + # this the listing returns AccessDenied, r-versions-existing.txt stays empty, + # and every version is rebuilt even though it already exists. + LISTING=$(curl -s --aws-sigv4 "aws:amz:eu-central-003:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ grep -oE '[^<]+' | sed 's/<[^>]*>//g') : > r-versions-existing.txt for VERSION in $(cat r-versions-to-build.txt); do