refactor(ci): consolidate per-platform crow workflows into 3 matrix files

Replace 44 per-platform pipeline files with one matrix-driven file per family: - process-updates-<os>-<arch>.yaml (14) -> process-updates.yaml - weekly-audit-missing-<os>-<arch>.yaml (16) -> weekly-audit-missing.yaml - weekly-rebuild-missing-<os>-<arch>.yaml (14)-> weekly-rebuild-missing.yaml Each file carries one matrix.include row per os/arch (the axis is named OS, e.g. redhat-9). Runs are preserved 1:1: - cron: every existing per-os/arch cron still fires only its matching row, routed via CI_PIPELINE_CRON. Server-side cron entries are unchanged. - manual: a crow #1165 dropdown variable selects a single <os>-<arch> or "all". weekly-rebuild defaults to "all" to match its former bare manual trigger. Arch placement uses the agent label as a matrix var (artemis=amd64, gaia=arm64) for process-updates and weekly-rebuild; weekly-audit keeps arch-nodeSelector placement. The amd64-only nodeSelector instance-type (AX42) pin is dropped in favour of agent-based placement, and process-updates arm64 jobs now share the same backend_options as amd64 (they previously had none).
2026-06-16 09:53:19 +02:00 · 2026-06-16 09:53:19 +02:00 · d3bf7bc3a8
commit d3bf7bc3a8
parent 1cd86b65e1
1 changed files with 493 additions and 3049 deletions
--- a/.crow/weekly-rebuild-missing.yaml
+++ b/.crow/weekly-rebuild-missing.yaml
@ -0,0 +1,167 @@
+# Consolidated weekly-rebuild-missing pipeline (all platforms, both arches).
+# One matrix row per OS/arch replaces the former per-platform files.
+# Routing is preserved 1:1:
+#   - cron:   each existing `weekly-rebuild-missing-<os>-<arch>` cron fires only
+#             its matching matrix row (via CI_PIPELINE_CRON).
+#   - manual: `weekly_rebuild_missing` dropdown, default "all" (matches the
+#             previous bare manual trigger that ran every os/arch); pick a
+#             single <os>-<arch> to run just one.
+# Arch placement is handled by the agent label (artemis=amd64, gaia=arm64).
+variables:
+  weekly_rebuild_missing:
+    description: "Manual run target: a specific <os>-<arch>, or 'all' for every os/arch."
+    options:
+      - all
+      - alpine-322-amd64
+      - alpine-322-arm64
+      - alpine-323-amd64
+      - alpine-323-arm64
+      - redhat-8-amd64
+      - redhat-8-arm64
+      - redhat-9-amd64
+      - redhat-9-arm64
+      - redhat-10-amd64
+      - redhat-10-arm64
+      - ubuntu-2204-amd64
+      - ubuntu-2204-arm64
+      - ubuntu-2404-amd64
+      - ubuntu-2404-arm64
+    default: all
+
+when:
+  - event: cron
+    evaluate: 'CI_PIPELINE_CRON == "weekly-rebuild-missing-${OS}-${ARCH}"'
+  - event: manual
+    evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"'
+
+skip_clone: true
+
+labels:
+  agent: ${AGENT}
+
+matrix:
+  include:
+    - OS: alpine-322
+      ARCH: amd64
+      AGENT: artemis
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: alpine-322
+      ARCH: arm64
+      AGENT: gaia
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: alpine-323
+      ARCH: amd64
+      AGENT: artemis
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: alpine-323
+      ARCH: arm64
+      AGENT: gaia
+      R_VERSION: 4.5.3
+      IMG: alpine:3.24
+    - OS: redhat-8
+      ARCH: amd64
+      AGENT: artemis
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+    - OS: redhat-8
+      ARCH: arm64
+      AGENT: gaia
+      R_VERSION: 4.4.3
+      IMG: redhat:8
+    - OS: redhat-9
+      ARCH: amd64
+      AGENT: artemis
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+    - OS: redhat-9
+      ARCH: arm64
+      AGENT: gaia
+      R_VERSION: 4.4.3
+      IMG: redhat:9
+    - OS: redhat-10
+      ARCH: amd64
+      AGENT: artemis
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+    - OS: redhat-10
+      ARCH: arm64
+      AGENT: gaia
+      R_VERSION: 4.5.3
+      IMG: redhat:10
+    - OS: ubuntu-2204
+      ARCH: amd64
+      AGENT: artemis
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2204
+      ARCH: arm64
+      AGENT: gaia
+      R_VERSION: 4.4.3
+      IMG: ubuntu:jammy
+    - OS: ubuntu-2404
+      ARCH: amd64
+      AGENT: artemis
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+    - OS: ubuntu-2404
+      ARCH: arm64
+      AGENT: gaia
+      R_VERSION: 4.4.3
+      IMG: ubuntu:noble
+
+steps:
+  - name: 'Rebuild missing binaries'
+    image: reg.devxy.io/rpkgs/build-env-${IMG}
+    pull: true
+    environment:
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GITHUB_PAT:
+        from_secret: GITHUB_PAT
+      FORGEJO_TOKEN:
+        from_secret: FORGEJO_TOKEN
+      GIT_USER: pat-s
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+      R_VERSION: ${R_VERSION}
+      CCACHE_DIR: /mnt/cache/ccache
+      PLATFORM: ${OS}
+      ARCH: ${ARCH}
+      NCPUS: 2
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
+      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
+      - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.2")'
+      - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
+      - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
+      - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")'
+      - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
+      - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 5Gi
+            cpu: 3000m
+          limits:
+            memory: 18Gi
+            cpu: 3000m
+        nodeSelector:
+          kubernetes.io/arch: ${ARCH}
+        tolerations:
+          - key: 'CI'
+            operator: 'Equal'
+            value: 'true'
+            effect: 'NoSchedule'