diff --git a/.crow/build.yaml b/.crow/build.yaml index b3b6c0e..7cad51b 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -56,14 +56,6 @@ matrix: # ARCH: arm64 # ARCH_ID: aarch64 # INSTANCE_TYPE: cax31 - - PLATFORM: alpine-324 - PLATFORM_ID: alpine324 - ARCH: arm64 - ARCH_ID: aarch64 - - PLATFORM: alpine-324 - PLATFORM_ID: alpine324 - ARCH: amd64 - ARCH_ID: x86_64 - PLATFORM: alpine-323 PLATFORM_ID: alpine323 ARCH: arm64 @@ -122,26 +114,15 @@ steps: kubernetes.io/arch: "${ARCH}" - - name: Get R versions to build + - name: Get latest R version image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl jq && break; sleep 5; done - # Latest patch release of each of the last 4 minor R versions (e.g. 4.6.0, 4.5.3, 4.4.3, 4.3.3). - - | - curl -sf https://cdn.posit.co/r/versions.json | jq -r ' - .r_versions - | map(select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))) - | group_by(split(".")[0:2] | join(".")) - | map(max_by(split(".") | map(tonumber))) - | sort_by(split(".") | map(tonumber)) - | reverse - | .[0:4] - | .[]' > r-versions-to-build.txt - # To pin specific versions for testing, overwrite the file, e.g.: - # - printf '4.1.3\n' > r-versions-to-build.txt - - cat r-versions-to-build.txt + - curl -sf https://formulae.brew.sh/api/formula/r.json | jq -er '.versions.stable' > r-version-to-build.txt + # - echo "4.1.3" > r-version-to-build.txt + - cat r-version-to-build.txt backend_options: kubernetes: resources: @@ -155,7 +136,7 @@ steps: kubernetes.io/arch: "${ARCH}" - - name: Check which R versions already exist in s3 + - name: Check if files for R version already exist in s3 image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true environment: @@ -170,20 +151,16 @@ steps: # Backblaze B2 requires authentication for the list-bucket API (anonymous # GET works only for individual public-read objects), so the request must be # SigV4-signed with the same credentials used for the upload step. Without - # this the listing returns AccessDenied, r-versions-existing.txt stays empty, - # and every version is rebuilt even though it already exists. - LISTING=$(curl -s --aws-sigv4 "aws:amz:eu-central-003:s3" \ + # this the listing returns AccessDenied, r-version-s3.txt stays empty, and + # every version is rebuilt even though it already exists. + VERSION=$(cat r-version-to-build.txt) + curl -s --aws-sigv4 "aws:amz:eu-central-003:s3" \ --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ - grep -oE '[^<]+' | sed 's/<[^>]*>//g') - : > r-versions-existing.txt - for VERSION in $(cat r-versions-to-build.txt); do - if printf '%s\n' "$LISTING" | \ - awk -v arch="${ARCH_ID}" -v ver="$VERSION" 'index($0, arch) && index($0, ver) { found=1 } END { exit !found }'; then - echo "$VERSION" >> r-versions-existing.txt - fi - done - - echo "Already present in s3:"; cat r-versions-existing.txt + grep -oE '[^<]+' | sed 's/<[^>]*>//g' | \ + awk -v arch="${ARCH_ID}" -v ver="${VERSION}" \ + 'index($0, arch) && index($0, ver) { found = 1 } END { if (found) print ver }' > r-version-s3.txt + - cat r-version-s3.txt backend_options: kubernetes: nodeSelector: @@ -199,20 +176,9 @@ steps: commands: | ip link set dev eth0 mtu 1280 2>/dev/null || true # docker info - TO_BUILD="" - for VERSION in $(cat r-versions-to-build.txt); do - if grep -qxF "$VERSION" r-versions-existing.txt; then - echo "R $VERSION already exists for ${PLATFORM} (${ARCH_ID}), skipping" - else - TO_BUILD="$TO_BUILD $VERSION" - fi - done - if [ -n "$TO_BUILD" ]; then + if ! grep -qF "$(cat r-version-to-build.txt)" r-version-s3.txt; then for i in 1 2 3 4 5; do apk add -q --no-cache make just docker-compose && break; sleep 5; done - for VERSION in $TO_BUILD; do - echo "Building R $VERSION for ${PLATFORM}" - just build-r-${PLATFORM} "$VERSION" - done + just build-r-${PLATFORM} $(cat r-version-to-build.txt) fi backend_options: kubernetes: diff --git a/Justfile b/Justfile index d402218..1a9d969 100644 --- a/Justfile +++ b/Justfile @@ -47,8 +47,3 @@ build-r-alpine-323 R_VERSION: export PLATFORM=alpine-323; \ export R_VERSION={{R_VERSION}}; \ make build-r-$PLATFORM - -build-r-alpine-324 R_VERSION: - export PLATFORM=alpine-324; \ - export R_VERSION={{R_VERSION}}; \ - make build-r-$PLATFORM diff --git a/Makefile b/Makefile index 82bdcc0..4b3bde4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PLATFORMS := ubuntu-2004 ubuntu-2204 ubuntu-2404 ubuntu-2604 debian-10 debian-11 debian-12 centos-7 centos-8 rhel-9 opensuse-155 fedora-38 fedora-39 fedora-40 alpine-321 alpine-322 alpine-323 alpine-324 rhel-10 +PLATFORMS := ubuntu-2004 ubuntu-2204 ubuntu-2404 ubuntu-2604 debian-10 debian-11 debian-12 centos-7 centos-8 rhel-9 opensuse-155 fedora-38 fedora-39 fedora-40 alpine-321 alpine-322 alpine-323 rhel-10 SLS_BINARY ?= ./node_modules/serverless/bin/serverless.js deps: diff --git a/builder/Dockerfile.alpine-324 b/builder/Dockerfile.alpine-324 deleted file mode 100644 index b3daa01..0000000 --- a/builder/Dockerfile.alpine-324 +++ /dev/null @@ -1,26 +0,0 @@ -FROM reg.devxy.io/docker.io/library/alpine:3.24 - -ENV OS_IDENTIFIER alpine-324 - -RUN set -x \ - && apk add -q R-dev curl ca-certificates bash g++ tzdata openjdk17 texmf-dist texlive-full tar sed patch tcl tk tk-dev xvfb libdeflate libdeflate-dev - -# Install s5cmd for S3 uploads (much faster than AWS CLI) -RUN ARCH=$(uname -m); if [ "$ARCH" = "x86_64" ]; then S5CMD_ARCH="64bit"; else S5CMD_ARCH="arm64"; fi && \ - curl -sL "https://github.com/peak/s5cmd/releases/download/v2.3.0/s5cmd_2.3.0_Linux-${S5CMD_ARCH}.tar.gz" | tar xz -C /usr/local/bin s5cmd - -RUN curl -LO "https://github.com/goreleaser/nfpm/releases/download/v2.39.0/nfpm_2.39.0_$(arch).apk" && \ - apk add --allow-untrusted "./nfpm_2.39.0_$(arch).apk" && \ - rm "nfpm_2.39.0_$(arch).apk" - -RUN chmod 0777 /opt - -# Override the default pager used by R -ENV PAGER /usr/bin/pager - -ENV CONFIGURE_OPTIONS "--enable-R-shlib --with-tcltk --enable-memory-profiling --with-x=no --with-blas --with-lapack" - -COPY package.alpine-324 /package.sh -COPY build.sh . -COPY patches /patches -ENTRYPOINT ./build.sh diff --git a/builder/docker-compose.yml b/builder/docker-compose.yml index a44093e..ee72ab7 100644 --- a/builder/docker-compose.yml +++ b/builder/docker-compose.yml @@ -96,19 +96,6 @@ services: volumes: - /tmp/alpine-323:/tmp/output/alpine-323 - ./build.sh:/build.sh:ro - alpine-324: - command: ./build.sh - environment: - - R_VERSION=${R_VERSION} - - R_INSTALL_PATH=${R_INSTALL_PATH} - - LOCAL_STORE=/tmp/output - build: - context: . - dockerfile: Dockerfile.alpine-324 - image: r-builds:alpine-324 - volumes: - - /tmp/alpine-324:/tmp/output/alpine-324 - - ./build.sh:/build.sh:ro debian-10: command: ./build.sh environment: diff --git a/builder/package.alpine-324 b/builder/package.alpine-324 deleted file mode 100644 index 2e0b28e..0000000 --- a/builder/package.alpine-324 +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -if [[ ! -d /tmp/output/${OS_IDENTIFIER} ]]; then - mkdir -p "/tmp/output/${OS_IDENTIFIER}" -fi - -# R 3.x requires PCRE1. On Ubuntu 24, R 3.x also requires PCRE2 for Pango support. -pcre_libs='- pcre2-dev' -if [[ "${R_VERSION}" =~ ^3 ]]; then - pcre_libs='- pcre2-dev -- libpcre3-dev' -fi - -deflate_libs='# - libdeflate-dev' -if grep -q '^LIBS *=.*[-]ldeflate' ${R_INSTALL_PATH}/lib/R/etc/Makeconf; then - deflate_libs='- libdeflate-dev' -fi - -cat < /tmp/nfpm.yml -name: r-${R_VERSION} -version: 1 -version_schema: none -section: universe/math -priority: optional -arch: $(arch) -maintainer: Posit Software, PBC -description: | - GNU R statistical computation and graphics system -vendor: Posit Software, PBC -homepage: https://www.r-project.org -license: GPL-2 -depends: -- g++ -- gcc -- gfortran -- libbz2 -# - libc6 -- cairo -- libcurl -${deflate_libs} -# - libglib2.0-0t64 -# - libgomp1 -- icu-dev -- jpeg-dev -# - liblzma-dev -- openblas-dev -- pango -# - libpangocairo-1.0-0 -- libpaper -${pcre_libs} -- libpng-dev -- readline-dev -- tcl -- tiff-dev -- libtirpc-dev -- tk -# - libx11-6 -# - libxt6t64 -- make -# - ucf -- unzip -- zip -- zlib-ng-dev -contents: -- src: ${R_INSTALL_PATH} - dst: ${R_INSTALL_PATH} -EOF - -nfpm package \ - -f /tmp/nfpm.yml \ - -p apk \ - -t "/tmp/output/${OS_IDENTIFIER}" - -export PKG_FILE=$(ls /tmp/output/${OS_IDENTIFIER}/r-${R_VERSION}*.apk | head -1)