fix(ci): split oversized weekly rebuild matrix

2026-08-14 06:56:41 +00:00 · 2026-08-14 06:56:41 +00:00 · b75fd2f1c4
commit b75fd2f1c4
parent 132d1d2d3c
4 changed files with 253 additions and 47 deletions
--- a/.crow/weekly-rebuild-missing-ubuntu-2604.yaml
+++ b/.crow/weekly-rebuild-missing-ubuntu-2604.yaml
@ -0,0 +1,140 @@
+# ubuntu-2604 rows split from weekly-rebuild-missing.yaml.
+#
+# The complete rebuild matrix has 54 permutations, while Crow accepts at most
+# 50 per workflow. Keeping these six rows in a companion workflow lets a manual
+# "all" run fan out every slot and shard without exceeding that compiler limit.
+
+variables:
+  # This is the same gate as the main rebuild workflow. The safe default must
+  # match no row because every file in .crow/ is evaluated on a manual run.
+  weekly_rebuild_missing:
+    description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
+    options:
+      - none
+      - all
+      - alpine-322-amd64
+      - alpine-322-arm64
+      - alpine-323-amd64
+      - alpine-323-arm64
+      - alpine-324-amd64
+      - alpine-324-arm64
+      - redhat-8-amd64
+      - redhat-8-arm64
+      - redhat-9-amd64
+      - redhat-9-arm64
+      - redhat-10-amd64
+      - redhat-10-arm64
+      - ubuntu-2204-amd64
+      - ubuntu-2204-arm64
+      - ubuntu-2404-amd64
+      - ubuntu-2404-arm64
+      - ubuntu-2604-amd64
+      - ubuntu-2604-arm64
+    default: none
+
+when:
+  - event: cron
+    cron: weekly-rebuild-missing-${OS}-${ARCH}
+  - event: manual
+    evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"'
+
+skip_clone: true
+
+labels:
+  group: rpkgs-${ARCH}
+
+matrix:
+  include:
+    - OS: ubuntu-2604
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: ubuntu-2604
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: ubuntu-2604
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
+    - OS: ubuntu-2604
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 1
+    - OS: ubuntu-2604
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 2
+    - OS: ubuntu-2604
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+      SPLIT_INTO: 3
+      SPLIT_INDEX: 3
+
+steps:
+  - name: 'Rebuild missing binaries'
+    image: reg.devxy.io/rpkgs/build-env-${IMG}
+    pull: true
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GITHUB_PAT:
+        from_secret: GITHUB_PAT
+      FORGEJO_TOKEN:
+        from_secret: FORGEJO_TOKEN
+      GIT_USER: pat-s
+      UVR_CACHE_DIR: /mnt/cache/uvr/cache
+      UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
+      R_LIBS_USER: /mnt/cache/R-pkgs
+      R_VERSION: ${R_VERSION}
+      CCACHE_DIR: /mnt/cache/ccache
+      PLATFORM: ${OS}
+      ARCH: ${ARCH}
+      NCPUS: 2
+      SPLIT_INTO: ${SPLIT_INTO}
+      SPLIT_INDEX: ${SPLIT_INDEX}
+      # Wall clock after which the shard stops cleanly instead of having to be
+      # killed. A kill matches neither `success` nor `failure`, so it would skip
+      # the dependent re-index and leave rebuilt binaries behind a stale edge.
+      REBUILD_BUDGET_HOURS: 20
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
+      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
+      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
+      - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
+      - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
+      - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2
+      - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
+      - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1
+    backend_options:
+      docker:
+        resources:
+          requests:
+            memory: 5Gi
+            cpu: 3000m
+          limits:
+            memory: 18Gi
+            cpu: 3000m
--- a/.crow/weekly-rebuild-missing.yaml
+++ b/.crow/weekly-rebuild-missing.yaml
@ -1,5 +1,7 @@
-# Consolidated weekly-rebuild-missing pipeline (all platforms, both arches).
+# Main weekly-rebuild-missing workflow (all slots except ubuntu-2604).
 # Three matrix rows per OS/arch, one per shard of that slot's rebuild list.
+# ubuntu-2604 lives in weekly-rebuild-missing-ubuntu-2604.yaml so a manual
+# "all" run stays below Crow's 50-permutation matrix limit.
 # Routing is preserved 1:1:
 #   - cron:   each existing `weekly-rebuild-missing-<os>-<arch>` cron fires only
 #             its matching matrix rows (via the per-row `cron:` name filter),
@ -346,43 +348,6 @@ matrix:
      IMG: ubuntu:noble
      SPLIT_INTO: 3
      SPLIT_INDEX: 3
-    - OS: ubuntu-2604
-      ARCH: amd64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:resolute
-      SPLIT_INTO: 3
-      SPLIT_INDEX: 1
-    - OS: ubuntu-2604
-      ARCH: amd64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:resolute
-      SPLIT_INTO: 3
-      SPLIT_INDEX: 2
-    - OS: ubuntu-2604
-      ARCH: amd64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:resolute
-      SPLIT_INTO: 3
-      SPLIT_INDEX: 3
-    - OS: ubuntu-2604
-      ARCH: arm64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:resolute
-      SPLIT_INTO: 3
-      SPLIT_INDEX: 1
-    - OS: ubuntu-2604
-      ARCH: arm64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:resolute
-      SPLIT_INTO: 3
-      SPLIT_INDEX: 2
-    - OS: ubuntu-2604
-      ARCH: arm64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:resolute
-      SPLIT_INTO: 3
-      SPLIT_INDEX: 3
-
 steps:
  - name: 'Rebuild missing binaries'
    image: reg.devxy.io/rpkgs/build-env-${IMG}
--- a/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml
+++ b/.crow/weekly-rebuild-reindex-ubuntu-2604.yaml
@ -0,0 +1,110 @@
+# Re-index ubuntu-2604 after its split rebuild workflow finishes.
+
+variables:
+  # Mirrors the shared rebuild gate. The default must match no matrix row so
+  # unrelated manual runs do not rewrite package indexes.
+  weekly_rebuild_missing:
+    description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
+    options:
+      - none
+      - all
+      - alpine-322-amd64
+      - alpine-322-arm64
+      - alpine-323-amd64
+      - alpine-323-arm64
+      - alpine-324-amd64
+      - alpine-324-arm64
+      - redhat-8-amd64
+      - redhat-8-arm64
+      - redhat-9-amd64
+      - redhat-9-arm64
+      - redhat-10-amd64
+      - redhat-10-arm64
+      - ubuntu-2204-amd64
+      - ubuntu-2204-arm64
+      - ubuntu-2404-amd64
+      - ubuntu-2404-arm64
+      - ubuntu-2604-amd64
+      - ubuntu-2604-arm64
+    default: none
+
+when:
+  - event: cron
+    cron: weekly-rebuild-missing-${OS}-${ARCH}
+  - event: manual
+    evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"'
+
+depends_on:
+  - weekly-rebuild-missing-ubuntu-2604
+
+runs_on: [success, failure]
+
+skip_clone: true
+
+labels:
+  group: rpkgs-${ARCH}
+
+matrix:
+  include:
+    - OS: ubuntu-2604
+      ARCH: amd64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+    - OS: ubuntu-2604
+      ARCH: arm64
+      R_VERSION: 4.4.3
+      IMG: ubuntu:resolute
+
+steps:
+  - name: 'Re-index the slot'
+    image: reg.devxy.io/rpkgs/build-env-${IMG}
+    pull: true
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GIT_USER: pat-s
+      R_LIBS_USER: /mnt/cache/R-pkgs
+      R_VERSION: ${R_VERSION}
+      PLATFORM: ${OS}
+      ARCH: ${ARCH}
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - mkdir -p /mnt/cache/R-pkgs
+      - rm -rf /mnt/cache/R-pkgs/00LOCK-*
+      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
+      # The codename is detected from the image's /etc/os-release.
+      - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
+      - |
+        for RBIN in /opt/R/[0-9]*/bin/R; do
+          RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
+          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
+        done
+
+  - name: Purge CDN cache
+    image: reg.devxy.io/docker.io/library/alpine:3.24
+    environment:
+      OTEL_R_TRACES_EXPORTER: none
+      OTEL_R_LOGS_EXPORTER: none
+      OTEL_R_METRICS_EXPORTER: none
+      BUNNYNET_API_KEY:
+        from_secret: BUNNYNET_API_KEY
+      # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
+      # Bunny pull zones, so both must be purged after the shared origin changes.
+      BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io'
+    commands:
+      - apk add --no-cache -q bash curl jq
+      # Crow carries the checkout from the re-index step into this step.
+      - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
+    # Run even when the re-index above failed: the objects were still replaced,
+    # and a stale edge is exactly what keeps them hidden.
+    when:
+      - status: [success, failure]
--- a/.crow/weekly-rebuild-reindex.yaml
+++ b/.crow/weekly-rebuild-reindex.yaml
@ -122,15 +122,6 @@ matrix:
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
-    - OS: ubuntu-2604
-      ARCH: amd64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:resolute
-    - OS: ubuntu-2604
-      ARCH: arm64
-      R_VERSION: 4.4.3
-      IMG: ubuntu:resolute
-
 steps:
  - name: 'Re-index the slot'
    image: reg.devxy.io/rpkgs/build-env-${IMG}