feat: add rhel10 package update workflows

2026-04-20 11:51:27 +02:00 · 2026-04-20 11:51:27 +02:00 · 8974bc4c92
commit 8974bc4c92
parent 8a98cf96a6
4 changed files with 297 additions and 0 deletions
--- a/.crow/process-updates-redhat-10-amd64.yaml
+++ b/.crow/process-updates-redhat-10-amd64.yaml
@ -0,0 +1,77 @@
+when:
+  - event: [cron]
+    cron: process-cran-updates-redhat-10-amd64
+  - event: manual
+    evaluate: 'process_cran_updates == "all"'
+  - event: manual
+    evaluate: 'process_cran_updates == "redhat-10-amd64"'
+
+skip_clone: true
+
+labels:
+  agent: artemis
+
+steps:
+  - name: 'Processing Updates'
+    image: reg.devxy.io/rpkgs/build-env-redhat:10-4.5.3
+    pull: true
+    environment:
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GITHUB_PAT:
+        from_secret: GITHUB_PAT
+      # normal env vars
+      GIT_USER: pat-s
+      NTFY_SERVER:
+        from_secret: NTFY_SERVER
+      NTFY_TOPIC:
+        from_secret: NTFY_TOPIC
+      NTFY_AUTH: TRUE
+      NTFY_PASSWORD:
+        from_secret: ntfy_token
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+      CCACHE_DIR: /mnt/cache/ccache
+      PLATFORM: redhat-10
+      ARCH: amd64
+      NCPUS: 2
+      MEMORY: 18Gi
+      STRATEGY: sequential
+      INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
+    # volumes:
+    #   - amd64-binaries-r-dep-cache-redhat-10:/mnt/cache
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft
+      - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")'
+      - R -q -e 'packageVersion("bincraft")'
+      - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
+      # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"
+      - R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 5Gi
+            cpu: 3000m
+          limits:
+            memory: 18Gi
+            cpu: 3000m
+        nodeSelector:
+          kubernetes.io/arch: amd64
+          node.kubernetes.io/instance-type: AX42
+        tolerations:
+          - key: 'CI'
+            operator: 'Equal'
+            value: 'true'
+            effect: 'NoSchedule'
--- a/.crow/process-updates-redhat-10-arm64.yaml
+++ b/.crow/process-updates-redhat-10-arm64.yaml
@ -0,0 +1,58 @@
+when:
+  - event: [cron]
+    cron: process-cran-updates-redhat-10-arm64
+  - event: manual
+    evaluate: 'process_cran_updates == "all"'
+  - event: manual
+    evaluate: 'process_cran_updates == "redhat-10-arm64"'
+
+skip_clone: true
+
+labels:
+  agent: gaia
+
+steps:
+  - name: 'Processing Updates'
+    image: reg.devxy.io/rpkgs/build-env-redhat:10-4.5.3
+    pull: true
+    environment:
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GITHUB_PAT:
+        from_secret: GITHUB_PAT
+      # normal env vars
+      GIT_USER: pat-s
+      NTFY_SERVER:
+        from_secret: NTFY_SERVER
+      NTFY_TOPIC:
+        from_secret: NTFY_TOPIC
+      NTFY_AUTH: TRUE
+      NTFY_PASSWORD:
+        from_secret: ntfy_token
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+      CCACHE_DIR: /mnt/cache/ccache
+      PLATFORM: redhat-10
+      ARCH: arm64
+      NCPUS: 2
+      INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
+    # volumes:
+    #   - arm64-binaries-r-dep-cache-redhat-10:/mnt/cache
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft
+      - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")'
+      - R -q -e 'packageVersion("bincraft")'
+      - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
+      # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-10', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"
+      - R -q -e 'library(bincraft); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
--- a/.crow/update-package-index-redhat-10-amd64.yaml
+++ b/.crow/update-package-index-redhat-10-amd64.yaml
@ -0,0 +1,81 @@
+when:
+  - event: [cron]
+    cron: update-package-index-redhat-10-amd64
+  - event: [cron]
+    cron: update-package-index
+  - event: manual
+    evaluate: 'update_package_index == "redhat-10-amd64"'
+
+skip_clone: true
+
+depends_on:
+  - update-package-index-redhat-8-arm64
+
+steps:
+  - name: Upload PACKAGES files
+    image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.5
+    pull: true
+    environment:
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GITHUB_PAT:
+        from_secret: GITHUB_PAT
+      # normal env vars
+      GIT_USER: pat-s
+      NTFY_SERVER:
+        from_secret: NTFY_SERVER
+      NTFY_TOPIC:
+        from_secret: NTFY_TOPIC
+      NTFY_AUTH: TRUE
+      NTFY_PASSWORD:
+        from_secret: ntfy_token
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - rm -rf /mnt/cache/packages/*
+      - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")'
+      - R -q -e 'pak::pak("pat-s/cranlike@s3")'
+      - R -q -e 'pak::pak("pat-s/desc@description-from-remote")'
+      - R -q -e 'packageVersion("bincraft")'
+      - cd /mnt/cache/packages
+      - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 2600Mi
+            cpu: 2000m
+          limits:
+            memory: 2600Mi
+            cpu: 8000m
+        nodeSelector:
+          kubernetes.io/arch: 'amd64'
+        tolerations:
+          - key: 'CI'
+            operator: 'Equal'
+            value: 'true'
+            effect: 'NoSchedule'
+
+  - name: Purge CDN cache
+    image: reg.devxy.io/docker.io/library/alpine:3.23
+    environment:
+      BUNNYNET_API_KEY:
+        from_secret: BUNNYNET_API_KEY
+      SUBDOMAIN1: 'cran.devxy.io'
+      SUBDOMAIN2: 'cran.allianceswisspass.devxy.io'
+      SUBDOMAIN3: 'cran.rpkgs.com'
+      OS_ID: rhel10
+      ARCH: amd64
+    commands:
+      - apk add --no-cache -q bash curl
+      - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3"
--- a/.crow/update-package-index-redhat-10-arm64.yaml
+++ b/.crow/update-package-index-redhat-10-arm64.yaml
@ -0,0 +1,81 @@
+when:
+  - event: [cron]
+    cron: update-package-index-redhat-10-arm64
+  - event: [cron]
+    cron: update-package-index
+  - event: manual
+    evaluate: 'update_package_index == "redhat-10-arm64"'
+
+skip_clone: true
+
+depends_on:
+  - update-package-index-redhat-10-amd64
+
+steps:
+  - name: Upload PACKAGES files
+    image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4
+    pull: true
+    environment:
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      B2_S3_ACCESS_KEY:
+        from_secret: B2_S3_ACCESS_KEY
+      B2_S3_SECRET_KEY:
+        from_secret: B2_S3_SECRET_KEY
+      PGPASS:
+        from_secret: PGPASS
+      REPO_RO_TOKEN:
+        from_secret: REPO_RO_TOKEN
+      GITHUB_PAT:
+        from_secret: GITHUB_PAT
+      # normal env vars
+      GIT_USER: pat-s
+      NTFY_SERVER:
+        from_secret: NTFY_SERVER
+      NTFY_TOPIC:
+        from_secret: NTFY_TOPIC
+      NTFY_AUTH: TRUE
+      NTFY_PASSWORD:
+        from_secret: ntfy_token
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+    commands:
+      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - rm -rf /mnt/cache/packages/*
+      - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")'
+      - R -q -e 'pak::pak("pat-s/cranlike@s3")'
+      - R -q -e 'pak::pak("pat-s/desc@description-from-remote")'
+      - R -q -e 'packageVersion("bincraft")'
+      - cd /mnt/cache/packages
+      - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel10", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 2600Mi
+            cpu: 2000m
+          limits:
+            memory: 2600Mi
+            cpu: 8000m
+        nodeSelector:
+          kubernetes.io/arch: 'arm64'
+        tolerations:
+          - key: 'CI'
+            operator: 'Equal'
+            value: 'true'
+            effect: 'NoSchedule'
+
+  - name: Purge CDN cache
+    image: reg.devxy.io/docker.io/library/alpine:3.23
+    environment:
+      BUNNYNET_API_KEY:
+        from_secret: BUNNYNET_API_KEY
+      SUBDOMAIN1: 'cran.devxy.io'
+      SUBDOMAIN2: 'cran.allianceswisspass.devxy.io'
+      SUBDOMAIN3: 'cran.rpkgs.com'
+      OS_ID: rhel10
+      ARCH: arm64
+    commands:
+      - apk add --no-cache -q bash curl
+      - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3"