pat-s
637a321bccFollow-up to #11, which landed the trigger-form handling but not the rest. Three things still stood between FORCE_REBUILD and an actual republish. Declare FORCE_REBUILD as a top-level variable so the UI offers it as a dropdown on a manual run, rather than requiring a hand-typed key. Test the value rather than emptiness. A declared variable substitutes its default, so the `[ -n ... ]` check from #11 would have seen the literal string "false" and rebuilt every platform on every cron run. Set overwrite: true on the upload. plugin-s3 skips an object that already exists when overwrite is false, logging "File already exists, skipping upload" and still exiting green, so a forced rebuild would have recompiled every package and published none of them. Normal runs are unaffected, since dedup means only not-yet-published versions are ever built. Also drop the Net debug step, the commented-out Pull upstream step and a stray `# docker info`. The MTU workaround the debug step was diagnosing stays, as does the commented-out alpine-320 matrix entry, which records a deliberately disabled platform rather than dead scaffolding.
276 lines
9 KiB
YAML
276 lines
9 KiB
YAML
when:
|
|
event: [cron, manual]
|
|
|
|
variables:
|
|
FORCE_REBUILD:
|
|
description: >-
|
|
Rebuild every targeted R version even when the artifact is already in the
|
|
bucket. Needed after a builder-image change that alters the resulting R
|
|
binary, because artifact filenames carry only the R version and would
|
|
otherwise dedup to the stale build forever.
|
|
options:
|
|
- "false"
|
|
- "true"
|
|
default: "false"
|
|
|
|
matrix:
|
|
include:
|
|
- PLATFORM: rhel-9
|
|
PLATFORM_ID: el9
|
|
ARCH: arm64
|
|
ARCH_ID: aarch64
|
|
- PLATFORM: rhel-9
|
|
PLATFORM_ID: el9
|
|
ARCH: amd64
|
|
ARCH_ID: x86_64
|
|
- PLATFORM: rhel-10
|
|
PLATFORM_ID: el10
|
|
ARCH: arm64
|
|
ARCH_ID: aarch64
|
|
- PLATFORM: rhel-10
|
|
PLATFORM_ID: el10
|
|
ARCH: amd64
|
|
ARCH_ID: x86_64
|
|
- PLATFORM: centos-8
|
|
PLATFORM_ID: el8
|
|
ARCH: arm64
|
|
ARCH_ID: aarch64
|
|
- PLATFORM: centos-8
|
|
PLATFORM_ID: el8
|
|
ARCH: amd64
|
|
ARCH_ID: x86_64
|
|
- PLATFORM: ubuntu-2204
|
|
PLATFORM_ID: 2204
|
|
ARCH: arm64
|
|
ARCH_ID: arm64
|
|
- PLATFORM: ubuntu-2204
|
|
PLATFORM_ID: 2204
|
|
ARCH: amd64
|
|
ARCH_ID: amd64
|
|
- PLATFORM: ubuntu-2404
|
|
PLATFORM_ID: 2404
|
|
ARCH: arm64
|
|
ARCH_ID: arm64
|
|
- PLATFORM: ubuntu-2404
|
|
PLATFORM_ID: 2404
|
|
ARCH: amd64
|
|
ARCH_ID: amd64
|
|
- PLATFORM: ubuntu-2604
|
|
PLATFORM_ID: 2604
|
|
ARCH: arm64
|
|
ARCH_ID: arm64
|
|
- PLATFORM: ubuntu-2604
|
|
PLATFORM_ID: 2604
|
|
ARCH: amd64
|
|
ARCH_ID: amd64
|
|
# - PLATFORM: alpine-320
|
|
# PLATFORM_ID: alpine320
|
|
# ARCH: arm64
|
|
# ARCH_ID: aarch64
|
|
# INSTANCE_TYPE: cax31
|
|
- PLATFORM: alpine-324
|
|
PLATFORM_ID: alpine324
|
|
ARCH: arm64
|
|
ARCH_ID: aarch64
|
|
- PLATFORM: alpine-324
|
|
PLATFORM_ID: alpine324
|
|
ARCH: amd64
|
|
ARCH_ID: x86_64
|
|
- PLATFORM: alpine-323
|
|
PLATFORM_ID: alpine323
|
|
ARCH: arm64
|
|
ARCH_ID: aarch64
|
|
- PLATFORM: alpine-323
|
|
PLATFORM_ID: alpine323
|
|
ARCH: amd64
|
|
ARCH_ID: x86_64
|
|
- PLATFORM: alpine-322
|
|
PLATFORM_ID: alpine322
|
|
ARCH: arm64
|
|
ARCH_ID: aarch64
|
|
- PLATFORM: alpine-322
|
|
PLATFORM_ID: alpine322
|
|
ARCH: amd64
|
|
ARCH_ID: x86_64
|
|
|
|
steps:
|
|
- name: Get R versions to build
|
|
image: reg.devxy.io/docker.io/library/alpine:3.23
|
|
privileged: true
|
|
commands:
|
|
- ip link set dev eth0 mtu 1280 2>/dev/null || true
|
|
- for i in 1 2 3 4 5; do apk add -q --no-cache curl jq && break; sleep 5; done
|
|
# Latest patch release of each of the last 4 minor R versions (e.g. 4.6.0, 4.5.3, 4.4.3, 4.3.3).
|
|
- |
|
|
curl -sf https://cdn.posit.co/r/versions.json | jq -r '
|
|
.r_versions
|
|
| map(select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$")))
|
|
| group_by(split(".")[0:2] | join("."))
|
|
| map(max_by(split(".") | map(tonumber)))
|
|
| sort_by(split(".") | map(tonumber))
|
|
| reverse
|
|
| .[0:4]
|
|
| .[]' > r-versions-to-build.txt
|
|
# To pin specific versions for testing, overwrite the file, e.g.:
|
|
# - printf '4.1.3\n' > r-versions-to-build.txt
|
|
- cat r-versions-to-build.txt
|
|
backend_options:
|
|
kubernetes:
|
|
resources:
|
|
requests:
|
|
memory: 50Mi
|
|
cpu: 200m
|
|
limits:
|
|
memory: 200Mi
|
|
cpu: 500m
|
|
nodeSelector:
|
|
kubernetes.io/arch: "${ARCH}"
|
|
|
|
|
|
- name: Check which R versions already exist in s3
|
|
image: reg.devxy.io/docker.io/library/alpine:3.23
|
|
privileged: true
|
|
commands:
|
|
- ip link set dev eth0 mtu 1280 2>/dev/null || true
|
|
- for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done
|
|
- |
|
|
# Backblaze B2's list-bucket API requires authentication, but the upload
|
|
# credentials are scoped to the plugin-s3 image and aren't available here.
|
|
# Individual objects are public-read, so probe each expected artifact URL
|
|
# with an anonymous HEAD instead of listing the bucket. The filename mirrors
|
|
# what nfpm produces in builder/package.${PLATFORM} (version/release pinned
|
|
# to 1): r-<ver>_1_<arch>.{apk,deb} for alpine/ubuntu, R-<ver>-1-1.<arch>.rpm
|
|
# for el*. Shell variables must use bare $name, not ${name}: crow performs
|
|
# its own ${VAR} substitution on these commands before the shell runs, so
|
|
# ${VAR} for a shell var would be blanked out (only matrix vars resolve there).
|
|
case "${PLATFORM}" in
|
|
alpine-*) FMT="r-%s_1_%s.apk" ;;
|
|
ubuntu-*) FMT="r-%s_1_%s.deb" ;;
|
|
*) FMT="R-%s-1-1.%s.rpm" ;;
|
|
esac
|
|
BASE="https://s3.eu-central-003.backblazeb2.com/devxy-r-builds/${PLATFORM_ID}"
|
|
: > r-versions-existing.txt
|
|
# FORCE_REBUILD is declared as a top-level variable, so the UI offers it
|
|
# as a dropdown on a manual run. The two trigger paths deliver it
|
|
# differently, so accept either:
|
|
# UI, or crow pipeline create --var FORCE_REBUILD=true
|
|
# -> a substitution variable, expanded into ${FORCE_REBUILD} below
|
|
# crow pipeline start -p FORCE_REBUILD=true
|
|
# -> injected into the step environment, read as bare $FORCE_REBUILD
|
|
# The declared default is the string "false", so test the value rather
|
|
# than emptiness; an empty value means neither path supplied one.
|
|
FORCE_VALUE="${FORCE_REBUILD}"
|
|
[ -n "$FORCE_VALUE" ] || FORCE_VALUE="$FORCE_REBUILD"
|
|
case "$FORCE_VALUE" in
|
|
true|TRUE|True|1|yes|YES) FORCE_ON=1 ;;
|
|
*) FORCE_ON="" ;;
|
|
esac
|
|
if [ -n "$FORCE_ON" ]; then
|
|
echo "FORCE_REBUILD=$FORCE_VALUE, ignoring artifacts already in the bucket"
|
|
else
|
|
for VERSION in $(cat r-versions-to-build.txt); do
|
|
URL="$BASE/$(printf "$FMT" "$VERSION" "${ARCH_ID}")"
|
|
if [ "$(curl -s -o /dev/null -w '%{http_code}' -I "$URL")" = "200" ]; then
|
|
echo "$VERSION" >> r-versions-existing.txt
|
|
fi
|
|
done
|
|
fi
|
|
- echo "Already present in s3:"; cat r-versions-existing.txt
|
|
backend_options:
|
|
kubernetes:
|
|
nodeSelector:
|
|
kubernetes.io/arch: "${ARCH}"
|
|
|
|
|
|
# ref: https://github.com/woodpecker-ci/woodpecker/discussions/2721
|
|
- name: Build
|
|
image: reg.devxy.io/docker.io/library/alpine:3.23
|
|
privileged: true
|
|
environment:
|
|
DOCKER_HOST: tcp://docker:2375
|
|
commands: |
|
|
ip link set dev eth0 mtu 1280 2>/dev/null || true
|
|
TO_BUILD=""
|
|
for VERSION in $(cat r-versions-to-build.txt); do
|
|
if grep -qxF "$VERSION" r-versions-existing.txt; then
|
|
echo "R $VERSION already exists for ${PLATFORM} (${ARCH_ID}), skipping"
|
|
else
|
|
TO_BUILD="$TO_BUILD $VERSION"
|
|
fi
|
|
done
|
|
if [ -n "$TO_BUILD" ]; then
|
|
for i in 1 2 3 4 5; do apk add -q --no-cache make just docker-compose && break; sleep 5; done
|
|
for VERSION in $TO_BUILD; do
|
|
echo "Building R $VERSION for ${PLATFORM}"
|
|
just build-r-${PLATFORM} "$VERSION"
|
|
done
|
|
fi
|
|
backend_options:
|
|
kubernetes:
|
|
resources:
|
|
requests:
|
|
memory: 200Mi
|
|
cpu: 500m
|
|
ephemeral-storage: 2Gi
|
|
limits:
|
|
memory: 1000Mi
|
|
cpu: 4000m
|
|
ephemeral-storage: 4Gi
|
|
nodeSelector:
|
|
kubernetes.io/arch: "${ARCH}"
|
|
|
|
|
|
- name: Upload to Backblaze S3
|
|
image: reg.devxy.io/docker.io/woodpeckerci/plugin-s3:1.5.2
|
|
pull: true
|
|
settings:
|
|
bucket: devxy-r-builds
|
|
strip_prefix: /tmp/${PLATFORM}/
|
|
source: /tmp/${PLATFORM}/*
|
|
target: ${PLATFORM_ID}/
|
|
region: eu-central-003
|
|
acl: public-read
|
|
endpoint: https://s3.eu-central-003.backblazeb2.com
|
|
access_key:
|
|
from_secret: B2_S3_ACCESS_KEY_ID
|
|
secret_key:
|
|
from_secret: B2_S3_SECRET_KEY
|
|
# Must stay true for FORCE_REBUILD to mean anything: with overwrite=false
|
|
# the plugin logs "File already exists, skipping upload" and still exits
|
|
# green, so a rebuild would be discarded. Normal runs are unaffected,
|
|
# since dedup means only not-yet-published versions are ever built.
|
|
overwrite: true
|
|
volumes:
|
|
- ci-tmp-rwx:/tmp # agent on k8s
|
|
# - /tmp:/tmp # agent on VM
|
|
backend_options:
|
|
kubernetes:
|
|
resources:
|
|
requests:
|
|
memory: 100Mi
|
|
cpu: 200m
|
|
limits:
|
|
memory: 1000Mi
|
|
cpu: 1000m
|
|
nodeSelector:
|
|
kubernetes.io/arch: "${ARCH}"
|
|
|
|
|
|
services:
|
|
docker:
|
|
environment:
|
|
DOCKER_TLS_CERTDIR: ""
|
|
image: reg.devxy.io/docker.io/library/docker:29-dind
|
|
commands:
|
|
- ip link set dev eth0 mtu 1280 2>/dev/null || true
|
|
- mkdir -p /tmp/${PLATFORM}/${ARCH}
|
|
- dockerd --tls=false --mtu 1280 --host=tcp://0.0.0.0:2375
|
|
privileged: true
|
|
ports:
|
|
- 2375
|
|
volumes:
|
|
- ci-tmp-rwx:/tmp # agent on k8s
|
|
backend_options:
|
|
kubernetes:
|
|
nodeSelector:
|
|
kubernetes.io/arch: "${ARCH}"
|