devxy/r-builds
Watch
2
0
Fork
You've already forked r-builds
 0
Code Issues 1 Pull requests Releases Activity
r-builds/.crow/build.yaml
pat-s eef805bd76
 fix(ci): authenticate B2 list-bucket request in existing-version check 
Backblaze B2 requires authentication for the list-bucket API; anonymous
access only works for individual public-read objects. The unauthenticated
curl returned AccessDenied, so r-version-s3.txt was always empty and every
R version was rebuilt despite already existing in s3.

Sign the listing request with --aws-sigv4 using the same B2 credentials as
the upload step. Also fix the broken version extraction: the old
substr($NF, 3, 5) ran against the prefixed key and yielded garbage, so the
Build step's grep could never match. Print the actual version on a match
and use a literal index() instead of a regex comparison.
2026-06-29 13:01:19 +02:00

281 lines
8.7 KiB
YAML
Raw Blame History

when:
event: [cron, manual]
matrix:
include:
- PLATFORM: rhel-9
PLATFORM_ID: el9
ARCH: arm64
ARCH_ID: aarch64
- PLATFORM: rhel-9
PLATFORM_ID: el9
ARCH: amd64
ARCH_ID: x86_64
- PLATFORM: rhel-10
PLATFORM_ID: el10
ARCH: arm64
ARCH_ID: aarch64
- PLATFORM: rhel-10
PLATFORM_ID: el10
ARCH: amd64
ARCH_ID: x86_64
- PLATFORM: centos-8
PLATFORM_ID: el8
ARCH: arm64
ARCH_ID: aarch64
- PLATFORM: centos-8
PLATFORM_ID: el8
ARCH: amd64
ARCH_ID: x86_64
- PLATFORM: ubuntu-2204
PLATFORM_ID: 2204
ARCH: arm64
ARCH_ID: arm64
- PLATFORM: ubuntu-2204
PLATFORM_ID: 2204
ARCH: amd64
ARCH_ID: amd64
- PLATFORM: ubuntu-2404
PLATFORM_ID: 2404
ARCH: arm64
ARCH_ID: arm64
- PLATFORM: ubuntu-2404
PLATFORM_ID: 2404
ARCH: amd64
ARCH_ID: amd64
- PLATFORM: ubuntu-2604
PLATFORM_ID: 2604
ARCH: arm64
ARCH_ID: arm64
- PLATFORM: ubuntu-2604
PLATFORM_ID: 2604
ARCH: amd64
ARCH_ID: amd64
# - PLATFORM: alpine-320
# PLATFORM_ID: alpine320
# ARCH: arm64
# ARCH_ID: aarch64
# INSTANCE_TYPE: cax31
- PLATFORM: alpine-324
PLATFORM_ID: alpine324
ARCH: arm64
ARCH_ID: aarch64
- PLATFORM: alpine-324
PLATFORM_ID: alpine324
ARCH: amd64
ARCH_ID: x86_64
- PLATFORM: alpine-323
PLATFORM_ID: alpine323
ARCH: arm64
ARCH_ID: aarch64
- PLATFORM: alpine-323
PLATFORM_ID: alpine323
ARCH: amd64
ARCH_ID: x86_64
- PLATFORM: alpine-322
PLATFORM_ID: alpine322
ARCH: arm64
ARCH_ID: aarch64
- PLATFORM: alpine-322
PLATFORM_ID: alpine322
ARCH: amd64
ARCH_ID: x86_64
steps:
# Ensures that we're always using the latest available upstream build chain
# - name: Pull upstream
# image: reg.devxy.io/docker.io/library/alpine:3.20
# commands:
# - apk add -q --no-cache git
# - git remote add upstream https://github.com/cynkra/r-builds.git
# - git config --global user.email "you@example.com"
# - git config --global user.name "Your Name"
# - git config pull.ff only
# - git fetch upstream
# - git merge upstream/main --allow-unrelated-histories
#.
- name: Net debug
image: reg.devxy.io/docker.io/library/alpine:3.23
privileged: true
commands:
- ip link set dev eth0 mtu 1280 2>/dev/null || true
- echo "=== pod interface ==="
- cat /sys/class/net/eth0/mtu
- ip -o link show eth0
- echo "=== route ==="
- ip route
- echo "=== resolv.conf ==="
- cat /etc/resolv.conf
- for i in 1 2 3; do apk add -q --no-cache iputils && break; sleep 5; done || echo "apk failed; falling back to busybox ping (no -M do)"
- |
echo "=== PMTUD probes (DF=1) to dl-cdn.alpinelinux.org ==="
for sz in 1472 1392 1352 1252 1200; do
echo "--- payload=$sz (wire ~$((sz+28))) ---"
ping -c 2 -W 3 -M do -s $sz dl-cdn.alpinelinux.org || echo "(size=$sz failed)"
done
- ping -c 3 -W 5 dl-cdn.alpinelinux.org || true
- wget --tries=1 --timeout=10 -O /dev/null -nv https://dl-cdn.alpinelinux.org/ 2>&1 || true
backend_options:
kubernetes:
nodeSelector:
kubernetes.io/arch: "${ARCH}"
- name: Get R versions to build
image: reg.devxy.io/docker.io/library/alpine:3.23
privileged: true
commands:
- ip link set dev eth0 mtu 1280 2>/dev/null || true
- for i in 1 2 3 4 5; do apk add -q --no-cache curl jq && break; sleep 5; done
# Latest patch release of each of the last 4 minor R versions (e.g. 4.6.0, 4.5.3, 4.4.3, 4.3.3).
- |
curl -sf https://cdn.posit.co/r/versions.json | jq -r '
.r_versions
| map(select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$")))
| group_by(split(".")[0:2] | join("."))
| map(max_by(split(".") | map(tonumber)))
| sort_by(split(".") | map(tonumber))
| reverse
| .[0:4]
| .[]' > r-versions-to-build.txt
# To pin specific versions for testing, overwrite the file, e.g.:
# - printf '4.1.3\n' > r-versions-to-build.txt
- cat r-versions-to-build.txt
backend_options:
kubernetes:
resources:
requests:
memory: 50Mi
cpu: 200m
limits:
memory: 200Mi
cpu: 500m
nodeSelector:
kubernetes.io/arch: "${ARCH}"
- name: Check which R versions already exist in s3
image: reg.devxy.io/docker.io/library/alpine:3.23
privileged: true
environment:
AWS_ACCESS_KEY_ID:
from_secret: B2_S3_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY:
from_secret: B2_S3_SECRET_KEY
commands:
- ip link set dev eth0 mtu 1280 2>/dev/null || true
- for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done
- |
# Backblaze B2 requires authentication for the list-bucket API (anonymous
# GET works only for individual public-read objects), so the request must be
# SigV4-signed with the same credentials used for the upload step. Without
# this the listing returns AccessDenied, r-versions-existing.txt stays empty,
# and every version is rebuilt even though it already exists.
LISTING=$(curl -s --aws-sigv4 "aws:amz:eu-central-003:s3" \
--user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \
"https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \
grep -oE '<Key>[^<]+</Key>' | sed 's/<[^>]*>//g')
: > r-versions-existing.txt
for VERSION in $(cat r-versions-to-build.txt); do
if printf '%s\n' "$LISTING" | \
awk -v arch="${ARCH_ID}" -v ver="$VERSION" 'index($0, arch) && index($0, ver) { found=1 } END { exit !found }'; then
echo "$VERSION" >> r-versions-existing.txt
fi
done
- echo "Already present in s3:"; cat r-versions-existing.txt
backend_options:
kubernetes:
nodeSelector:
kubernetes.io/arch: "${ARCH}"
# ref: https://github.com/woodpecker-ci/woodpecker/discussions/2721
- name: Build
image: reg.devxy.io/docker.io/library/alpine:3.23
privileged: true
environment:
DOCKER_HOST: tcp://docker:2375
commands: |
ip link set dev eth0 mtu 1280 2>/dev/null || true
# docker info
TO_BUILD=""
for VERSION in $(cat r-versions-to-build.txt); do
if grep -qxF "$VERSION" r-versions-existing.txt; then
echo "R $VERSION already exists for ${PLATFORM} (${ARCH_ID}), skipping"
else
TO_BUILD="$TO_BUILD $VERSION"
fi
done
if [ -n "$TO_BUILD" ]; then
for i in 1 2 3 4 5; do apk add -q --no-cache make just docker-compose && break; sleep 5; done
for VERSION in $TO_BUILD; do
echo "Building R $VERSION for ${PLATFORM}"
just build-r-${PLATFORM} "$VERSION"
done
fi
backend_options:
kubernetes:
resources:
requests:
memory: 200Mi
cpu: 500m
ephemeral-storage: 2Gi
limits:
memory: 1000Mi
cpu: 4000m
ephemeral-storage: 4Gi
nodeSelector:
kubernetes.io/arch: "${ARCH}"
- name: Upload to Backblaze S3
image: reg.devxy.io/docker.io/woodpeckerci/plugin-s3:1.5.2
pull: true
settings:
bucket: devxy-r-builds
strip_prefix: /tmp/${PLATFORM}/
source: /tmp/${PLATFORM}/*
target: ${PLATFORM_ID}/
region: eu-central-003
acl: public-read
endpoint: https://s3.eu-central-003.backblazeb2.com
access_key:
from_secret: B2_S3_ACCESS_KEY_ID
secret_key:
from_secret: B2_S3_SECRET_KEY
overwrite: false
volumes:
- ci-tmp-rwx:/tmp # agent on k8s
# - /tmp:/tmp # agent on VM
backend_options:
kubernetes:
resources:
requests:
memory: 100Mi
cpu: 200m
limits:
memory: 1000Mi
cpu: 1000m
nodeSelector:
kubernetes.io/arch: "${ARCH}"
services:
docker:
environment:
DOCKER_TLS_CERTDIR: ""
image: reg.devxy.io/docker.io/library/docker:29-dind
commands:
- ip link set dev eth0 mtu 1280 2>/dev/null || true
- mkdir -p /tmp/${PLATFORM}/${ARCH}
- dockerd --tls=false --mtu 1280 --host=tcp://0.0.0.0:2375
privileged: true
ports:
- 2375
volumes:
- ci-tmp-rwx:/tmp # agent on k8s
backend_options:
kubernetes:
nodeSelector:
kubernetes.io/arch: "${ARCH}"
Reference in a new issue View git blame Copy permalink