refactor(ci): consolidate per-platform crow workflows into 3 matrix files #92

Merged
pat-s merged 2 commits from refactor/consolidate-process-updates into main 2026-06-16 09:33:07 +00:00
3 changed files with 23 additions and 25 deletions
Showing only changes of commit 2a25373267 - Show all commits

refactor(ci): use docker backend + agent placement for process-updates/weekly

Match the build-all cleanup: only the docker backend is used and arch placement
is via the agent label, so the kubernetes backend_options blocks (with arch
nodeSelector and tolerations) are dead. Convert them to docker resource blocks.
weekly-audit had no agent label (it relied on the nodeSelector), so add
agent: ${AGENT} with an AGENT column (artemis=amd64, gaia=arm64) per matrix row.
Patrick Schratz 2026-06-16 11:32:16 +02:00
Signed by: pat-s
GPG key ID: 3C6318841EF78925

View file

@ -202,7 +202,7 @@ steps:
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = '${OS_ID}', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
done
backend_options:
kubernetes:
docker:
resources:
requests:
memory: 5Gi
@ -210,13 +210,6 @@ steps:
limits:
memory: 18Gi
cpu: 3000m
nodeSelector:
kubernetes.io/arch: ${ARCH}
tolerations:
- key: 'CI'
operator: 'Equal'
value: 'true'
effect: 'NoSchedule'
- name: Purge CDN cache
image: reg.devxy.io/docker.io/library/alpine:3.24

View file

@ -5,7 +5,7 @@
# its matching matrix row (via CI_PIPELINE_CRON).
# - manual: pick a target from the `weekly_audit_missing` dropdown
# ("all" = every os/arch).
# Arch placement is via the arch nodeSelector (this family uses no agent label).
# Arch placement is via the agent label (artemis=amd64, gaia=arm64).
variables:
weekly_audit_missing:
description: "Manual run target: a specific <os>-<arch>, or 'all' for every os/arch."
@ -37,70 +37,89 @@ when:
skip_clone: true
labels:
agent: ${AGENT}
matrix:
include:
- OS: alpine-321
ARCH: amd64
AGENT: artemis
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-321
ARCH: arm64
AGENT: gaia
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-322
ARCH: amd64
AGENT: artemis
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-322
ARCH: arm64
AGENT: gaia
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-323
ARCH: amd64
AGENT: artemis
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: alpine-323
ARCH: arm64
AGENT: gaia
R_VERSION: 4.5.3
IMG: alpine:3.24
- OS: redhat-8
ARCH: amd64
AGENT: artemis
R_VERSION: 4.4.3
IMG: redhat:8
- OS: redhat-8
ARCH: arm64
AGENT: gaia
R_VERSION: 4.4.3
IMG: redhat:8
- OS: redhat-9
ARCH: amd64
AGENT: artemis
R_VERSION: 4.4.3
IMG: redhat:9
- OS: redhat-9
ARCH: arm64
AGENT: gaia
R_VERSION: 4.4.3
IMG: redhat:9
- OS: redhat-10
ARCH: amd64
AGENT: artemis
R_VERSION: 4.5.3
IMG: redhat:10
- OS: redhat-10
ARCH: arm64
AGENT: gaia
R_VERSION: 4.5.3
IMG: redhat:10
- OS: ubuntu-2204
ARCH: amd64
AGENT: artemis
R_VERSION: 4.4.3
IMG: ubuntu:jammy
- OS: ubuntu-2204
ARCH: arm64
AGENT: gaia
R_VERSION: 4.4.3
IMG: ubuntu:jammy
- OS: ubuntu-2404
ARCH: amd64
AGENT: artemis
R_VERSION: 4.4.3
IMG: ubuntu:noble
- OS: ubuntu-2404
ARCH: arm64
AGENT: gaia
R_VERSION: 4.4.3
IMG: ubuntu:noble
@ -132,7 +151,7 @@ steps:
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))'
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")'
backend_options:
kubernetes:
docker:
resources:
requests:
memory: 2Gi
@ -140,10 +159,3 @@ steps:
limits:
memory: 4Gi
cpu: 2000m
nodeSelector:
kubernetes.io/arch: ${ARCH}
tolerations:
- key: 'CI'
operator: 'Equal'
value: 'true'
effect: 'NoSchedule'

View file

@ -150,7 +150,7 @@ steps:
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1
backend_options:
kubernetes:
docker:
resources:
requests:
memory: 5Gi
@ -158,10 +158,3 @@ steps:
limits:
memory: 18Gi
cpu: 3000m
nodeSelector:
kubernetes.io/arch: ${ARCH}
tolerations:
- key: 'CI'
operator: 'Equal'
value: 'true'
effect: 'NoSchedule'