From 1e278cdb3a8f358d434adaa99422a1023b471662 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 29 Jun 2026 11:02:04 +0000 Subject: [PATCH 1/3] fix(ci): authenticate B2 list-bucket request in existing-version check (#7) Backblaze B2 requires authentication for the list-bucket API; anonymous access only works for individual public-read objects. The unauthenticated curl returned AccessDenied, so r-version-s3.txt was always empty and every R version was rebuilt despite already existing in s3. Sign the listing request with --aws-sigv4 using the same B2 credentials as the upload step. Also fix the broken version extraction: the old substr($NF, 3, 5) ran against the prefixed key and yielded garbage, so the Build step's grep could never match. Print the actual version on a match and use a literal index() instead of a regex comparison. Reviewed-on: https://git.devxy.io/devxy/r-builds/pulls/7 --- .crow/build.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index 1d7cb47..b3b6c0e 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -158,11 +158,23 @@ steps: - name: Check which R versions already exist in s3 image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true + environment: + AWS_ACCESS_KEY_ID: + from_secret: B2_S3_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: + from_secret: B2_S3_SECRET_KEY commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done - | - LISTING=$(curl -s "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ + # Backblaze B2 requires authentication for the list-bucket API (anonymous + # GET works only for individual public-read objects), so the request must be + # SigV4-signed with the same credentials used for the upload step. Without + # this the listing returns AccessDenied, r-versions-existing.txt stays empty, + # and every version is rebuilt even though it already exists. + LISTING=$(curl -s --aws-sigv4 "aws:amz:eu-central-003:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ grep -oE '[^<]+' | sed 's/<[^>]*>//g') : > r-versions-existing.txt for VERSION in $(cat r-versions-to-build.txt); do From b917759865d9e07794bce0130b8951b0f5bd64a6 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 29 Jun 2026 13:00:26 +0200 Subject: [PATCH 2/3] fix(ci): detect existing R versions via public object HEAD probe The existing-version check listed the bucket with an anonymous curl, but Backblaze B2 requires authentication for the list-bucket API, so it returned AccessDenied and r-versions-existing.txt was always empty -> every version was rebuilt despite already existing in s3. Signing the request is not workable here: the B2 credentials are scoped to the plugin-s3 image in crow and are not injected into this plain alpine step (curl saw --user ":"). Individual artifacts are public-read, so probe each expected object URL with an anonymous HEAD instead of listing the bucket. The filename mirrors what nfpm produces (version/release pinned to 1): r-_1_.{apk,deb} for alpine/ubuntu and R--1-1..rpm for el*. Verified against the live bucket for all three packager families. --- .crow/build.yaml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index 1d7cb47..c980b4b 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -162,12 +162,23 @@ steps: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done - | - LISTING=$(curl -s "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ - grep -oE '[^<]+' | sed 's/<[^>]*>//g') + # Backblaze B2's list-bucket API requires authentication, but the upload + # credentials are scoped to the plugin-s3 image and aren't available here. + # Individual objects are public-read, so probe each expected artifact URL + # with an anonymous HEAD instead of listing the bucket. The filename mirrors + # what nfpm produces in builder/package.${PLATFORM} (version/release pinned + # to 1): r-_1_.{apk,deb} for alpine/ubuntu, R--1-1..rpm + # for el*. + case "${PLATFORM}" in + alpine-*) FILE="r-VER_1_${ARCH_ID}.apk" ;; + ubuntu-*) FILE="r-VER_1_${ARCH_ID}.deb" ;; + *) FILE="R-VER-1-1.${ARCH_ID}.rpm" ;; + esac + BASE="https://s3.eu-central-003.backblazeb2.com/devxy-r-builds/${PLATFORM_ID}" : > r-versions-existing.txt for VERSION in $(cat r-versions-to-build.txt); do - if printf '%s\n' "$LISTING" | \ - awk -v arch="${ARCH_ID}" -v ver="$VERSION" 'index($0, arch) && index($0, ver) { found=1 } END { exit !found }'; then + URL="${BASE}/$(echo "$FILE" | sed "s/VER/${VERSION}/")" + if [ "$(curl -s -o /dev/null -w '%{http_code}' -I "$URL")" = "200" ]; then echo "$VERSION" >> r-versions-existing.txt fi done From 60d67022e2b48cf8687f41c124df4e0ca5933581 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 29 Jun 2026 13:00:26 +0200 Subject: [PATCH 3/3] fix(ci): detect existing R versions via public object HEAD probe The existing-version check listed the bucket with an anonymous curl, but Backblaze B2 requires authentication for the list-bucket API, so it returned AccessDenied and r-versions-existing.txt was always empty -> every version was rebuilt despite already existing in s3. Signing the request is not workable here: the B2 credentials are scoped to the plugin-s3 image in crow and are not injected into this plain alpine step (curl saw --user ":"). Individual artifacts are public-read, so probe each expected object URL with an anonymous HEAD instead of listing the bucket. The filename mirrors what nfpm produces (version/release pinned to 1): r-_1_.{apk,deb} for alpine/ubuntu and R--1-1..rpm for el*. Verified against the live bucket for all three packager families. --- .crow/build.yaml | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/.crow/build.yaml b/.crow/build.yaml index b3b6c0e..c980b4b 100644 --- a/.crow/build.yaml +++ b/.crow/build.yaml @@ -158,28 +158,27 @@ steps: - name: Check which R versions already exist in s3 image: reg.devxy.io/docker.io/library/alpine:3.23 privileged: true - environment: - AWS_ACCESS_KEY_ID: - from_secret: B2_S3_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: - from_secret: B2_S3_SECRET_KEY commands: - ip link set dev eth0 mtu 1280 2>/dev/null || true - for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done - | - # Backblaze B2 requires authentication for the list-bucket API (anonymous - # GET works only for individual public-read objects), so the request must be - # SigV4-signed with the same credentials used for the upload step. Without - # this the listing returns AccessDenied, r-versions-existing.txt stays empty, - # and every version is rebuilt even though it already exists. - LISTING=$(curl -s --aws-sigv4 "aws:amz:eu-central-003:s3" \ - --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ - "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \ - grep -oE '[^<]+' | sed 's/<[^>]*>//g') + # Backblaze B2's list-bucket API requires authentication, but the upload + # credentials are scoped to the plugin-s3 image and aren't available here. + # Individual objects are public-read, so probe each expected artifact URL + # with an anonymous HEAD instead of listing the bucket. The filename mirrors + # what nfpm produces in builder/package.${PLATFORM} (version/release pinned + # to 1): r-_1_.{apk,deb} for alpine/ubuntu, R--1-1..rpm + # for el*. + case "${PLATFORM}" in + alpine-*) FILE="r-VER_1_${ARCH_ID}.apk" ;; + ubuntu-*) FILE="r-VER_1_${ARCH_ID}.deb" ;; + *) FILE="R-VER-1-1.${ARCH_ID}.rpm" ;; + esac + BASE="https://s3.eu-central-003.backblazeb2.com/devxy-r-builds/${PLATFORM_ID}" : > r-versions-existing.txt for VERSION in $(cat r-versions-to-build.txt); do - if printf '%s\n' "$LISTING" | \ - awk -v arch="${ARCH_ID}" -v ver="$VERSION" 'index($0, arch) && index($0, ver) { found=1 } END { exit !found }'; then + URL="${BASE}/$(echo "$FILE" | sed "s/VER/${VERSION}/")" + if [ "$(curl -s -o /dev/null -w '%{http_code}' -I "$URL")" = "200" ]; then echo "$VERSION" >> r-versions-existing.txt fi done