From 2a253732675ba0f69c88687f35585d4be4929a04 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 16 Jun 2026 11:32:16 +0200 Subject: [PATCH] refactor(ci): use docker backend + agent placement for process-updates/weekly Match the build-all cleanup: only the docker backend is used and arch placement is via the agent label, so the kubernetes backend_options blocks (with arch nodeSelector and tolerations) are dead. Convert them to docker resource blocks. weekly-audit had no agent label (it relied on the nodeSelector), so add agent: ${AGENT} with an AGENT column (artemis=amd64, gaia=arm64) per matrix row. --- .crow/process-updates.yaml | 9 +-------- .crow/weekly-audit-missing.yaml | 30 +++++++++++++++++++++--------- .crow/weekly-rebuild-missing.yaml | 9 +-------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 76c70f4..3c8141a 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -202,7 +202,7 @@ steps: /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = '${OS_ID}', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true done backend_options: - kubernetes: + docker: resources: requests: memory: 5Gi @@ -210,13 +210,6 @@ steps: limits: memory: 18Gi cpu: 3000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - name: Purge CDN cache image: reg.devxy.io/docker.io/library/alpine:3.24 diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 869b2e3..7a44f2b 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -5,7 +5,7 @@ # its matching matrix row (via CI_PIPELINE_CRON). # - manual: pick a target from the `weekly_audit_missing` dropdown # ("all" = every os/arch). -# Arch placement is via the arch nodeSelector (this family uses no agent label). +# Arch placement is via the agent label (artemis=amd64, gaia=arm64). variables: weekly_audit_missing: description: "Manual run target: a specific -, or 'all' for every os/arch." @@ -37,70 +37,89 @@ when: skip_clone: true +labels: + agent: ${AGENT} + matrix: include: - OS: alpine-321 ARCH: amd64 + AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-321 ARCH: arm64 + AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-322 ARCH: amd64 + AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-322 ARCH: arm64 + AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-323 ARCH: amd64 + AGENT: artemis R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: alpine-323 ARCH: arm64 + AGENT: gaia R_VERSION: 4.5.3 IMG: alpine:3.24 - OS: redhat-8 ARCH: amd64 + AGENT: artemis R_VERSION: 4.4.3 IMG: redhat:8 - OS: redhat-8 ARCH: arm64 + AGENT: gaia R_VERSION: 4.4.3 IMG: redhat:8 - OS: redhat-9 ARCH: amd64 + AGENT: artemis R_VERSION: 4.4.3 IMG: redhat:9 - OS: redhat-9 ARCH: arm64 + AGENT: gaia R_VERSION: 4.4.3 IMG: redhat:9 - OS: redhat-10 ARCH: amd64 + AGENT: artemis R_VERSION: 4.5.3 IMG: redhat:10 - OS: redhat-10 ARCH: arm64 + AGENT: gaia R_VERSION: 4.5.3 IMG: redhat:10 - OS: ubuntu-2204 ARCH: amd64 + AGENT: artemis R_VERSION: 4.4.3 IMG: ubuntu:jammy - OS: ubuntu-2204 ARCH: arm64 + AGENT: gaia R_VERSION: 4.4.3 IMG: ubuntu:jammy - OS: ubuntu-2404 ARCH: amd64 + AGENT: artemis R_VERSION: 4.4.3 IMG: ubuntu:noble - OS: ubuntu-2404 ARCH: arm64 + AGENT: gaia R_VERSION: 4.4.3 IMG: ubuntu:noble @@ -132,7 +151,7 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' backend_options: - kubernetes: + docker: resources: requests: memory: 2Gi @@ -140,10 +159,3 @@ steps: limits: memory: 4Gi cpu: 2000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index c4b0767..e06edf0 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -150,7 +150,7 @@ steps: - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: - kubernetes: + docker: resources: requests: memory: 5Gi @@ -158,10 +158,3 @@ steps: limits: memory: 18Gi cpu: 3000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule'