diff --git a/.crow/build.yaml b/.crow/build.yaml index 57478f7..b3b6c0e 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -56,6 +56,14 @@ matrix: # ARCH: arm64 # ARCH_ID: aarch64 # INSTANCE_TYPE: cax31 + - PLATFORM: alpine-324 + PLATFORM_ID: alpine324 + ARCH: arm64 + ARCH_ID: aarch64 + - PLATFORM: alpine-324 + PLATFORM_ID: alpine324 + ARCH: amd64 + ARCH_ID: x86_64 - PLATFORM: alpine-323 PLATFORM_ID: alpine323 ARCH: arm64 @@ -114,15 +122,26 @@ steps: kubernetes.io/arch: "${ARCH}" - - name: Get latest R version + - name: Get R versions to build image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl jq && break; sleep 5; done - - curl -sf https://formulae.brew.sh/api/formula/r.json | jq -er '.versions.stable' > r-version-to-build.txt - # - echo "4.1.3" > r-version-to-build.txt - - cat r-version-to-build.txt + # Latest patch release of each of the last 4 minor R versions (e.g. 4.6.0, 4.5.3, 4.4.3, 4.3.3). + - | + curl -sf https://cdn.posit.co/r/versions.json | jq -r ' + .r_versions + | map(select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))) + | group_by(split(".")[0:2] | join(".")) + | map(max_by(split(".") | map(tonumber))) + | sort_by(split(".") | map(tonumber)) + | reverse + | .[0:4] + | .[]' > r-versions-to-build.txt + # To pin specific versions for testing, overwrite the file, e.g.: + # - printf '4.1.3\n' > r-versions-to-build.txt + - cat r-versions-to-build.txt backend_options: kubernetes: resources: @@ -136,19 +155,35 @@ steps: kubernetes.io/arch: "${ARCH}" - - name: Check if files for R version already exist in s3 + - name: Check which R versions already exist in s3 image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true + environment: + AWS_ACCESS_KEY_ID: + from_secret: B2_S3_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: + from_secret: B2_S3_SECRET_KEY commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done - | - VERSION=$(cat r-version-to-build.txt) - curl -s "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ - grep -oE '[^<]+' | sed 's/<[^>]*>//g' | \ - awk -v arch="${ARCH_ID}" -v ver="${VERSION}" \ - '$0 ~ arch && $0 ~ ver {print substr($NF, 3, 5)}' > r-version-s3.txt - - cat r-version-s3.txt + # Backblaze B2 requires authentication for the list-bucket API (anonymous + # GET works only for individual public-read objects), so the request must be + # SigV4-signed with the same credentials used for the upload step. Without + # this the listing returns AccessDenied, r-versions-existing.txt stays empty, + # and every version is rebuilt even though it already exists. + LISTING=$(curl -s --aws-sigv4 "aws:amz:eu-central-003:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ + grep -oE '[^<]+' | sed 's/<[^>]*>//g') + : > r-versions-existing.txt + for VERSION in $(cat r-versions-to-build.txt); do + if printf '%s\n' "$LISTING" | \ + awk -v arch="${ARCH_ID}" -v ver="$VERSION" 'index($0, arch) && index($0, ver) { found=1 } END { exit !found }'; then + echo "$VERSION" >> r-versions-existing.txt + fi + done + - echo "Already present in s3:"; cat r-versions-existing.txt backend_options: kubernetes: nodeSelector: @@ -164,9 +199,20 @@ steps: commands: | ip link set dev eth0 mtu 1280 2>/dev/null || true # docker info - if ! grep -qF "$(cat r-version-to-build.txt)" r-version-s3.txt; then + TO_BUILD="" + for VERSION in $(cat r-versions-to-build.txt); do + if grep -qxF "$VERSION" r-versions-existing.txt; then + echo "R $VERSION already exists for ${PLATFORM} (${ARCH_ID}), skipping" + else + TO_BUILD="$TO_BUILD $VERSION" + fi + done + if [ -n "$TO_BUILD" ]; then for i in 1 2 3 4 5; do apk add -q --no-cache make just docker-compose && break; sleep 5; done - just build-r-${PLATFORM} $(cat r-version-to-build.txt) + for VERSION in $TO_BUILD; do + echo "Building R $VERSION for ${PLATFORM}" + just build-r-${PLATFORM} "$VERSION" + done fi backend_options: kubernetes: diff --git a/Justfile b/Justfile index 1a9d969..d402218 100644 --- a/Justfile +++ b/Justfile @@ -47,3 +47,8 @@ build-r-alpine-323 R_VERSION: export PLATFORM=alpine-323; \ export R_VERSION={{R_VERSION}}; \ make build-r-$PLATFORM + +build-r-alpine-324 R_VERSION: + export PLATFORM=alpine-324; \ + export R_VERSION={{R_VERSION}}; \ + make build-r-$PLATFORM diff --git a/Makefile b/Makefile index 4b3bde4..82bdcc0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PLATFORMS := ubuntu-2004 ubuntu-2204 ubuntu-2404 ubuntu-2604 debian-10 debian-11 debian-12 centos-7 centos-8 rhel-9 opensuse-155 fedora-38 fedora-39 fedora-40 alpine-321 alpine-322 alpine-323 rhel-10 +PLATFORMS := ubuntu-2004 ubuntu-2204 ubuntu-2404 ubuntu-2604 debian-10 debian-11 debian-12 centos-7 centos-8 rhel-9 opensuse-155 fedora-38 fedora-39 fedora-40 alpine-321 alpine-322 alpine-323 alpine-324 rhel-10 SLS_BINARY ?= ./node_modules/serverless/bin/serverless.js deps: diff --git a/builder/Dockerfile.alpine-324 b/builder/Dockerfile.alpine-324 new file mode 100644 index 0000000..b3daa01 --- /dev/null +++ b/builder/Dockerfile.alpine-324 @@ -0,0 +1,26 @@ +FROM reg.devxy.io/docker.io/library/alpine:3.24 + +ENV OS_IDENTIFIER alpine-324 + +RUN set -x \ + && apk add -q R-dev curl ca-certificates bash g++ tzdata openjdk17 texmf-dist texlive-full tar sed patch tcl tk tk-dev xvfb libdeflate libdeflate-dev + +# Install s5cmd for S3 uploads (much faster than AWS CLI) +RUN ARCH=$(uname -m); if [ "$ARCH" = "x86_64" ]; then S5CMD_ARCH="64bit"; else S5CMD_ARCH="arm64"; fi && \ + curl -sL "https://github.com/peak/s5cmd/releases/download/v2.3.0/s5cmd_2.3.0_Linux-${S5CMD_ARCH}.tar.gz" | tar xz -C /usr/local/bin s5cmd + +RUN curl -LO "https://github.com/goreleaser/nfpm/releases/download/v2.39.0/nfpm_2.39.0_$(arch).apk" && \ + apk add --allow-untrusted "./nfpm_2.39.0_$(arch).apk" && \ + rm "nfpm_2.39.0_$(arch).apk" + +RUN chmod 0777 /opt + +# Override the default pager used by R +ENV PAGER /usr/bin/pager + +ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-blas --with-lapack" + +COPY package.alpine-324 /package.sh +COPY build.sh . +COPY patches /patches +ENTRYPOINT ./build.sh diff --git a/builder/docker-compose.yml b/builder/docker-compose.yml index ee72ab7..a44093e 100644 --- a/builder/docker-compose.yml +++ b/builder/docker-compose.yml @@ -96,6 +96,19 @@ services: volumes: - /tmp/alpine-323:/tmp/output/alpine-323 - ./build.sh:/build.sh:ro + alpine-324: + command: ./build.sh + environment: + - R_VERSION=${R_VERSION} + - R_INSTALL_PATH=${R_INSTALL_PATH} + - LOCAL_STORE=/tmp/output + build: + context: . + dockerfile: Dockerfile.alpine-324 + image: r-builds:alpine-324 + volumes: + - /tmp/alpine-324:/tmp/output/alpine-324 + - ./build.sh:/build.sh:ro debian-10: command: ./build.sh environment: diff --git a/builder/package.alpine-324 b/builder/package.alpine-324 new file mode 100644 index 0000000..2e0b28e --- /dev/null +++ b/builder/package.alpine-324 @@ -0,0 +1,74 @@ +#!/bin/bash + +if [[ ! -d /tmp/output/${OS_IDENTIFIER} ]]; then + mkdir -p "/tmp/output/${OS_IDENTIFIER}" +fi + +# R 3.x requires PCRE1. On Ubuntu 24, R 3.x also requires PCRE2 for Pango support. +pcre_libs='- pcre2-dev' +if [[ "${R_VERSION}" =~ ^3 ]]; then + pcre_libs='- pcre2-dev +- libpcre3-dev' +fi + +deflate_libs='# - libdeflate-dev' +if grep -q '^LIBS *=.*[-]ldeflate' ${R_INSTALL_PATH}/lib/R/etc/Makeconf; then + deflate_libs='- libdeflate-dev' +fi + +cat < /tmp/nfpm.yml +name: r-${R_VERSION} +version: 1 +version_schema: none +section: universe/math +priority: optional +arch: $(arch) +maintainer: Posit Software, PBC +description: | + GNU R statistical computation and graphics system +vendor: Posit Software, PBC +homepage: https://www.r-project.org +license: GPL-2 +depends: +- g++ +- gcc +- gfortran +- libbz2 +# - libc6 +- cairo +- libcurl +${deflate_libs} +# - libglib2.0-0t64 +# - libgomp1 +- icu-dev +- jpeg-dev +# - liblzma-dev +- openblas-dev +- pango +# - libpangocairo-1.0-0 +- libpaper +${pcre_libs} +- libpng-dev +- readline-dev +- tcl +- tiff-dev +- libtirpc-dev +- tk +# - libx11-6 +# - libxt6t64 +- make +# - ucf +- unzip +- zip +- zlib-ng-dev +contents: +- src: ${R_INSTALL_PATH} + dst: ${R_INSTALL_PATH} +EOF + +nfpm package \ + -f /tmp/nfpm.yml \ + -p apk \ + -t "/tmp/output/${OS_IDENTIFIER}" + +export PKG_FILE=$(ls /tmp/output/${OS_IDENTIFIER}/r-${R_VERSION}*.apk | head -1)