From 84c59d198478b59cfa9cc84ae45a87ccd8fd272c Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 21 Apr 2026 20:30:33 +0200 Subject: [PATCH] fix: audit workflows --- .crow/process-updates-alpine-321-amd64.yaml | 76 ---------------- .crow/process-updates-alpine-321-arm64.yaml | 58 ------------- .crow/process-updates-alpine-322-amd64.yaml | 2 +- .crow/process-updates-alpine-322-arm64.yaml | 4 +- ...weekly-audit-missing-alpine-321-amd64.yaml | 4 +- ...weekly-audit-missing-alpine-321-arm64.yaml | 4 +- ...weekly-audit-missing-alpine-322-amd64.yaml | 4 +- ...weekly-audit-missing-alpine-322-arm64.yaml | 2 + ...weekly-audit-missing-alpine-323-amd64.yaml | 4 +- ...weekly-audit-missing-alpine-323-arm64.yaml | 4 +- .../weekly-audit-missing-redhat-10-amd64.yaml | 50 +++++++++++ .../weekly-audit-missing-redhat-10-arm64.yaml | 50 +++++++++++ .../weekly-audit-missing-redhat-8-amd64.yaml | 4 +- .../weekly-audit-missing-redhat-8-arm64.yaml | 4 +- .../weekly-audit-missing-redhat-9-amd64.yaml | 4 +- .../weekly-audit-missing-redhat-9-arm64.yaml | 4 +- ...eekly-audit-missing-ubuntu-2204-amd64.yaml | 4 +- ...eekly-audit-missing-ubuntu-2204-arm64.yaml | 4 +- ...eekly-audit-missing-ubuntu-2404-amd64.yaml | 4 +- ...eekly-audit-missing-ubuntu-2404-arm64.yaml | 4 +- ...ekly-rebuild-missing-alpine-322-amd64.yaml | 14 +-- ...ekly-rebuild-missing-alpine-322-arm64.yaml | 14 +-- ...ekly-rebuild-missing-alpine-323-amd64.yaml | 12 ++- ...ekly-rebuild-missing-alpine-323-arm64.yaml | 12 ++- ...ekly-rebuild-missing-redhat-10-amd64.yaml} | 18 ++-- ...ekly-rebuild-missing-redhat-10-arm64.yaml} | 18 ++-- ...weekly-rebuild-missing-redhat-8-amd64.yaml | 12 ++- ...weekly-rebuild-missing-redhat-8-arm64.yaml | 12 ++- ...weekly-rebuild-missing-redhat-9-amd64.yaml | 12 ++- ...weekly-rebuild-missing-redhat-9-arm64.yaml | 12 ++- ...kly-rebuild-missing-ubuntu-2204-amd64.yaml | 12 ++- ...kly-rebuild-missing-ubuntu-2204-arm64.yaml | 12 ++- ...kly-rebuild-missing-ubuntu-2404-amd64.yaml | 12 ++- ...kly-rebuild-missing-ubuntu-2404-arm64.yaml | 12 ++- local/fetch-rebuild-packages-from-issue.R | 87 +++++++++++++++++++ local/weekly-missing-binaries-audit.R | 86 +++++++++--------- 36 files changed, 384 insertions(+), 267 deletions(-) delete mode 100644 .crow/process-updates-alpine-321-amd64.yaml delete mode 100644 .crow/process-updates-alpine-321-arm64.yaml create mode 100644 .crow/weekly-audit-missing-redhat-10-amd64.yaml create mode 100644 .crow/weekly-audit-missing-redhat-10-arm64.yaml rename .crow/{weekly-rebuild-missing-alpine-321-amd64.yaml => weekly-rebuild-missing-redhat-10-amd64.yaml} (53%) rename .crow/{weekly-rebuild-missing-alpine-321-arm64.yaml => weekly-rebuild-missing-redhat-10-arm64.yaml} (52%) create mode 100644 local/fetch-rebuild-packages-from-issue.R diff --git a/.crow/process-updates-alpine-321-amd64.yaml b/.crow/process-updates-alpine-321-amd64.yaml deleted file mode 100644 index e09ce9f..0000000 --- a/.crow/process-updates-alpine-321-amd64.yaml +++ /dev/null @@ -1,76 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-321-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-321-amd64"' - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-alpine-321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - R -q -e 'library(bincraft); upload_package_index(codename = "alpine321", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/process-updates-alpine-321-arm64.yaml b/.crow/process-updates-alpine-321-arm64.yaml deleted file mode 100644 index 0a9ecdf..0000000 --- a/.crow/process-updates-alpine-321-arm64.yaml +++ /dev/null @@ -1,58 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-321-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-321-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-alpine-321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - R -q -e 'library(bincraft); upload_package_index(codename = "alpine321", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index f8265c5..ed39ccc 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -13,7 +13,7 @@ labels: steps: - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 + image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index 76d3c8b..57d3891 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -13,7 +13,7 @@ labels: steps: - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 + image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 pull: true environment: RED_HAT_DEV_PW: @@ -56,5 +56,3 @@ steps: # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - diff --git a/.crow/weekly-audit-missing-alpine-321-amd64.yaml b/.crow/weekly-audit-missing-alpine-321-amd64.yaml index 8e35b52..77b29a4 100644 --- a/.crow/weekly-audit-missing-alpine-321-amd64.yaml +++ b/.crow/weekly-audit-missing-alpine-321-amd64.yaml @@ -22,11 +22,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: alpine-321 ARCH: amd64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-alpine321:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-alpine-321-arm64.yaml b/.crow/weekly-audit-missing-alpine-321-arm64.yaml index ac1ff10..80ca905 100644 --- a/.crow/weekly-audit-missing-alpine-321-arm64.yaml +++ b/.crow/weekly-audit-missing-alpine-321-arm64.yaml @@ -22,11 +22,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: alpine-321 ARCH: arm64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-alpine321:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-alpine-322-amd64.yaml b/.crow/weekly-audit-missing-alpine-322-amd64.yaml index b7ae740..b3f9ba8 100644 --- a/.crow/weekly-audit-missing-alpine-322-amd64.yaml +++ b/.crow/weekly-audit-missing-alpine-322-amd64.yaml @@ -21,11 +21,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: alpine-322 ARCH: amd64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-alpine322:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-alpine-322-arm64.yaml b/.crow/weekly-audit-missing-alpine-322-arm64.yaml index 8e6e4cc..1815163 100644 --- a/.crow/weekly-audit-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-audit-missing-alpine-322-arm64.yaml @@ -20,6 +20,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: alpine-322 ARCH: arm64 R_LIBS_USER: /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-alpine-323-amd64.yaml b/.crow/weekly-audit-missing-alpine-323-amd64.yaml index 2a7f5e8..3ef9715 100644 --- a/.crow/weekly-audit-missing-alpine-323-amd64.yaml +++ b/.crow/weekly-audit-missing-alpine-323-amd64.yaml @@ -21,11 +21,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: alpine-323 ARCH: amd64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-alpine323:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-alpine-323-arm64.yaml b/.crow/weekly-audit-missing-alpine-323-arm64.yaml index aabe7c9..f3433ea 100644 --- a/.crow/weekly-audit-missing-alpine-323-arm64.yaml +++ b/.crow/weekly-audit-missing-alpine-323-arm64.yaml @@ -21,11 +21,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: alpine-323 ARCH: arm64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-alpine323:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-redhat-10-amd64.yaml b/.crow/weekly-audit-missing-redhat-10-amd64.yaml new file mode 100644 index 0000000..be0adce --- /dev/null +++ b/.crow/weekly-audit-missing-redhat-10-amd64.yaml @@ -0,0 +1,50 @@ +when: + - event: cron + cron: weekly-audit-missing-redhat-10-amd64 + - event: manual + evaluate: 'task == "weekly-audit-missing-redhat-10-amd64"' + +skip_clone: true + +steps: + - name: 'Audit missing binaries' + image: reg.devxy.io/rpkgs/build-env-redhat:10-4.5.3 + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + PLATFORM: redhat-10 + ARCH: amd64 + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' + - R -q -e 'source("local/weekly-missing-binaries-audit.R")' + backend_options: + kubernetes: + resources: + requests: + memory: 2Gi + cpu: 2000m + limits: + memory: 4Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-10-arm64.yaml b/.crow/weekly-audit-missing-redhat-10-arm64.yaml new file mode 100644 index 0000000..fcfcce0 --- /dev/null +++ b/.crow/weekly-audit-missing-redhat-10-arm64.yaml @@ -0,0 +1,50 @@ +when: + - event: cron + cron: weekly-audit-missing-redhat-10-arm64 + - event: manual + evaluate: 'task == "weekly-audit-missing-redhat-10-arm64"' + +skip_clone: true + +steps: + - name: 'Audit missing binaries' + image: reg.devxy.io/rpkgs/build-env-redhat:10-4.5.3 + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + PLATFORM: redhat-10 + ARCH: arm64 + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' + - R -q -e 'source("local/weekly-missing-binaries-audit.R")' + backend_options: + kubernetes: + resources: + requests: + memory: 2Gi + cpu: 2000m + limits: + memory: 4Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: arm64 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-8-amd64.yaml b/.crow/weekly-audit-missing-redhat-8-amd64.yaml index 11c98c0..31e5bd3 100644 --- a/.crow/weekly-audit-missing-redhat-8-amd64.yaml +++ b/.crow/weekly-audit-missing-redhat-8-amd64.yaml @@ -22,11 +22,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: redhat-8 ARCH: amd64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-redhat8:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-redhat-8-arm64.yaml b/.crow/weekly-audit-missing-redhat-8-arm64.yaml index a53b0a5..c947e6c 100644 --- a/.crow/weekly-audit-missing-redhat-8-arm64.yaml +++ b/.crow/weekly-audit-missing-redhat-8-arm64.yaml @@ -22,11 +22,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: redhat-8 ARCH: arm64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-redhat8:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-redhat-9-amd64.yaml b/.crow/weekly-audit-missing-redhat-9-amd64.yaml index fafaa41..9715204 100644 --- a/.crow/weekly-audit-missing-redhat-9-amd64.yaml +++ b/.crow/weekly-audit-missing-redhat-9-amd64.yaml @@ -22,11 +22,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: redhat-9 ARCH: amd64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-redhat9:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-redhat-9-arm64.yaml b/.crow/weekly-audit-missing-redhat-9-arm64.yaml index 9f887ca..3ebb5a2 100644 --- a/.crow/weekly-audit-missing-redhat-9-arm64.yaml +++ b/.crow/weekly-audit-missing-redhat-9-arm64.yaml @@ -22,11 +22,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: redhat-9 ARCH: arm64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-redhat9:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml b/.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml index 15a4ae4..66af6b4 100644 --- a/.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml +++ b/.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml @@ -22,11 +22,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: ubuntu-2204 ARCH: amd64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-ubuntu2204:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml b/.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml index 59ead84..e1de5f5 100644 --- a/.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml +++ b/.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml @@ -22,11 +22,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: ubuntu-2204 ARCH: arm64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-ubuntu2204:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml b/.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml index a32b63b..6ee080b 100644 --- a/.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml +++ b/.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml @@ -22,11 +22,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: ubuntu-2404 ARCH: amd64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-ubuntu2404:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml b/.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml index 4e6f3bc..9c7e093 100644 --- a/.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml +++ b/.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml @@ -22,11 +22,11 @@ steps: GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: ubuntu-2404 ARCH: arm64 R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-ubuntu2404:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs diff --git a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml index 05b4912..86f0680 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml @@ -2,13 +2,15 @@ when: - event: cron cron: weekly-rebuild-missing-alpine-322-amd64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-322-amd64"' skip_clone: true +labels: + agent: artemis + steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 + image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 pull: true environment: RED_HAT_DEV_PW: @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: alpine-322 ARCH: amd64 NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-alpine322:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-322_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml index b00d2f5..c985d1e 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml @@ -2,13 +2,15 @@ when: - event: cron cron: weekly-rebuild-missing-alpine-322-arm64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-322-arm64"' skip_clone: true +labels: + agent: gaia + steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 + image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 pull: true environment: RED_HAT_DEV_PW: @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: alpine-322 ARCH: arm64 NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-alpine322:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-322_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml index 1196f6c..1b40307 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml @@ -2,10 +2,12 @@ when: - event: cron cron: weekly-rebuild-missing-alpine-323-amd64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-323-amd64"' skip_clone: true +labels: + agent: artemis + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: alpine-323 ARCH: amd64 NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-alpine323:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-323_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml index 13eb4f9..b76b3c8 100644 --- a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml @@ -2,10 +2,12 @@ when: - event: cron cron: weekly-rebuild-missing-alpine-323-arm64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-323-arm64"' skip_clone: true +labels: + agent: gaia + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: alpine-323 ARCH: arm64 NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-alpine323:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-323_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-alpine-321-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml similarity index 53% rename from .crow/weekly-rebuild-missing-alpine-321-amd64.yaml rename to .crow/weekly-rebuild-missing-redhat-10-amd64.yaml index 62f3b6e..cfe3d01 100644 --- a/.crow/weekly-rebuild-missing-alpine-321-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-10-amd64.yaml @@ -1,14 +1,16 @@ when: - event: cron - cron: weekly-rebuild-missing-alpine-321-amd64 + cron: weekly-rebuild-missing-redhat-10-amd64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-321-amd64"' skip_clone: true +labels: + agent: artemis + steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 + image: reg.devxy.io/rpkgs/build-env-redhat:10-4.5.3 pull: true environment: RED_HAT_DEV_PW: @@ -23,15 +25,15 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 + PLATFORM: redhat-10 ARCH: amd64 NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-alpine321:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-321_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-alpine-321-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml similarity index 52% rename from .crow/weekly-rebuild-missing-alpine-321-arm64.yaml rename to .crow/weekly-rebuild-missing-redhat-10-arm64.yaml index de00c9a..89d7c45 100644 --- a/.crow/weekly-rebuild-missing-alpine-321-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-10-arm64.yaml @@ -1,14 +1,16 @@ when: - event: cron - cron: weekly-rebuild-missing-alpine-321-arm64 + cron: weekly-rebuild-missing-redhat-10-arm64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-321-arm64"' skip_clone: true +labels: + agent: gaia + steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 + image: reg.devxy.io/rpkgs/build-env-redhat:10-4.5.3 pull: true environment: RED_HAT_DEV_PW: @@ -23,15 +25,15 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 + PLATFORM: redhat-10 ARCH: arm64 NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-alpine321:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-321_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml index 879f9f9..62d11b3 100644 --- a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml @@ -2,10 +2,12 @@ when: - event: cron cron: weekly-rebuild-missing-redhat-8-amd64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-redhat-8-amd64"' skip_clone: true +labels: + agent: artemis + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-redhat:8-4.4.3 @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: redhat-8 ARCH: amd64 NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-redhat8:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_redhat-8_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml index 9e57bee..2194fee 100644 --- a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml @@ -2,10 +2,12 @@ when: - event: cron cron: weekly-rebuild-missing-redhat-8-arm64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-redhat-8-arm64"' skip_clone: true +labels: + agent: gaia + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-redhat:8-4.4.3 @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: redhat-8 ARCH: arm64 NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-redhat8:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_redhat-8_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml index ae2bf6c..4c7f404 100644 --- a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml @@ -2,10 +2,12 @@ when: - event: cron cron: weekly-rebuild-missing-redhat-9-amd64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-redhat-9-amd64"' skip_clone: true +labels: + agent: artemis + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: redhat-9 ARCH: amd64 NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-redhat9:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_redhat-9_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml index eeb8a8b..64cf358 100644 --- a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml +++ b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml @@ -2,10 +2,12 @@ when: - event: cron cron: weekly-rebuild-missing-redhat-9-arm64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-redhat-9-arm64"' skip_clone: true +labels: + agent: gaia + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: redhat-9 ARCH: arm64 NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-redhat9:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_redhat-9_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml index 696e7bf..94cdc34 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml @@ -2,10 +2,12 @@ when: - event: cron cron: weekly-rebuild-missing-ubuntu-2204-amd64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-ubuntu-2204-amd64"' skip_clone: true +labels: + agent: artemis + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: ubuntu-2204 ARCH: amd64 NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-ubuntu2204:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_ubuntu-2204_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml index 5de2a7b..c8305a6 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml @@ -2,10 +2,12 @@ when: - event: cron cron: weekly-rebuild-missing-ubuntu-2204-arm64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-ubuntu-2204-arm64"' skip_clone: true +labels: + agent: gaia + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: ubuntu-2204 ARCH: arm64 NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-ubuntu2204:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_ubuntu-2204_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml index 5c8a0d5..7da31a0 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml @@ -2,10 +2,12 @@ when: - event: cron cron: weekly-rebuild-missing-ubuntu-2404-amd64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-ubuntu-2404-amd64"' skip_clone: true +labels: + agent: artemis + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: ubuntu-2404 ARCH: amd64 NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-ubuntu2404:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_ubuntu-2404_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml index 52c8d46..5bd25c7 100644 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml +++ b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml @@ -2,10 +2,12 @@ when: - event: cron cron: weekly-rebuild-missing-ubuntu-2404-arm64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-ubuntu-2404-arm64"' skip_clone: true +labels: + agent: gaia + steps: - name: 'Rebuild missing binaries' image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: ubuntu-2404 ARCH: arm64 NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-ubuntu2404:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_ubuntu-2404_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/local/fetch-rebuild-packages-from-issue.R b/local/fetch-rebuild-packages-from-issue.R new file mode 100644 index 0000000..d944dd0 --- /dev/null +++ b/local/fetch-rebuild-packages-from-issue.R @@ -0,0 +1,87 @@ +library(httr2, quietly = TRUE) + +forgejo_base <- "https://git.devxy.io/api/v1" +repo <- "devxy/build-cran-binaries" +platform <- Sys.getenv("PLATFORM") +arch <- Sys.getenv("ARCH") +token <- Sys.getenv("FORGEJO_TOKEN") +output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") + +if (nchar(platform) == 0) stop("PLATFORM env var is not set") +if (nchar(arch) == 0) stop("ARCH env var is not set") +if (nchar(token) == 0) stop("FORGEJO_TOKEN env var is not set") + +os_family <- if (grepl("^ubuntu", platform)) { + "Ubuntu" +} else if (grepl("^alpine", platform)) { + "Alpine" +} else if (grepl("^redhat", platform)) { + "Red Hat" +} else { + platform +} + +issue_title <- sprintf("Missing package binaries for latest version (%s)", os_family) +cat(sprintf("Searching for issue: %s\n", issue_title)) + +search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, repo, + utils::URLencode(issue_title, reserved = TRUE) +) +search_resp <- request(search_url) |> + req_headers(Authorization = paste("token", token)) |> + req_perform() + +issues <- resp_body_json(search_resp, simplifyVector = FALSE) +match_idx <- which(vapply(issues, function(x) x$title, character(1)) == issue_title) + +if (length(match_idx) == 0) { + cat("No matching issue found - nothing to rebuild\n") + writeLines(character(0), output_file) + q("no") +} + +body <- issues[[match_idx[1]]]$body +if (is.null(body) || nchar(body) == 0) { + cat("Issue body is empty - nothing to rebuild\n") + writeLines(character(0), output_file) + q("no") +} + +lines <- strsplit(body, "\n", fixed = TRUE)[[1]] + +# Find the platform section (## alpine-322) +plat_header <- sprintf("## %s", platform) +plat_idx <- which(lines == plat_header) +if (length(plat_idx) == 0) { + cat(sprintf("No section found for platform %s - nothing to rebuild\n", platform)) + writeLines(character(0), output_file) + q("no") +} + +# Find the arch subsection (### arm64 (...)) +arch_pattern <- sprintf("^### %s ", arch) +arch_idx <- which(grepl(arch_pattern, lines) & seq_along(lines) > plat_idx[1]) +if (length(arch_idx) == 0) { + cat(sprintf("No section found for arch %s under %s - nothing to rebuild\n", arch, platform)) + writeLines(character(0), output_file) + q("no") +} + +ai <- arch_idx[1] + +# Find the end of the rebuildable section: stop at ####, ###, ##, ---, or EOF +end_idx <- which(grepl("^####|^###|^##|^---", lines) & seq_along(lines) > ai) +end_idx <- if (length(end_idx) > 0) end_idx[1] - 1 else length(lines) + +section_lines <- lines[seq(ai + 1, end_idx)] + +# Extract package names from "- PackageName (Version)" lines +pkg_lines <- section_lines[grepl("^- ", section_lines)] +pkgs <- sub("^- ([^ ]+) \\(.*\\)$", "\\1", pkg_lines) +pkgs <- pkgs[nchar(pkgs) > 0 & pkgs != "_None_"] + +cat(sprintf("Found %d rebuildable packages for %s/%s\n", length(pkgs), platform, arch)) +writeLines(pkgs, output_file) +cat(sprintf("Wrote package list to %s\n", output_file)) diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index 9c2b4ae..c6b67e5 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -20,8 +20,11 @@ arch <- Sys.getenv("ARCH") if (nchar(platform) == 0) stop("PLATFORM env var is not set") if (nchar(arch) == 0) stop("ARCH env var is not set") -# e.g. "ubuntu-2204" -> "ubuntu2204" +# Map platform names to S3 codenames s3_codename <- gsub("-", "", platform) +s3_codename <- sub("^redhat", "rhel", s3_codename) +s3_codename <- sub("^ubuntu2204$", "jammy", s3_codename) +s3_codename <- sub("^ubuntu2404$", "noble", s3_codename) # Derive OS family for the Forgejo issue title os_family <- if (grepl("^ubuntu", platform)) { @@ -38,7 +41,38 @@ cat(sprintf("Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n", platform, arch, s3_codename, os_family)) # --------------------------------------------------------------------------- -# 1. CRAN release packages +# 1. Query PostgreSQL for known build failures (before s3fs init to avoid +# C++ pointer conflicts between s3fs/curl and RPostgres/libpq) +# --------------------------------------------------------------------------- +cat("Connecting to PostgreSQL...\n") +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) + +errored_pkgs <- DBI::dbGetQuery( + con, + sprintf( + "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE AND platform = '%s' AND arch = '%s'", + platform, arch + ) +) +DBI::dbDisconnect(con) + +errored_dt <- as.data.table(errored_pkgs) +if (nrow(errored_dt) > 0) { + setnames(errored_dt, c("Package", "Version")) + setkey(errored_dt, Package, Version) +} +cat(sprintf("Found %d known build failures for %s/%s\n", nrow(errored_dt), platform, arch)) + +# --------------------------------------------------------------------------- +# 2. CRAN release packages (uses curl internally - must come after PG) # --------------------------------------------------------------------------- cat("Fetching CRAN package database...\n") cran_release <- tools::CRAN_package_db() @@ -48,7 +82,7 @@ cran_dt <- data.table( ) # --------------------------------------------------------------------------- -# 2. S3 tarballs +# 3. S3 tarballs # --------------------------------------------------------------------------- cat("Connecting to S3...\n") s3fs::s3_file_system( @@ -74,15 +108,19 @@ file_names <- basename(s3_pkgs) matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) parts <- regmatches(file_names, matches) parts <- parts[sapply(parts, length) == 3] -s3_dt <- data.table( - Package = sapply(parts, `[`, 2), - Version = sapply(parts, `[`, 3) -) +if (length(parts) == 0) { + s3_dt <- data.table(Package = character(0), Version = character(0)) +} else { + s3_dt <- data.table( + Package = sapply(parts, `[`, 2), + Version = sapply(parts, `[`, 3) + ) +} cat(sprintf("S3 contains %d tarballs for %s/%s\n", nrow(s3_dt), arch, s3_codename)) # --------------------------------------------------------------------------- -# 3. Find missing packages (CRAN release version not in S3) +# 4. Find missing packages (CRAN release version not in S3) # --------------------------------------------------------------------------- setkey(cran_dt, Package, Version) setkey(s3_dt, Package, Version) @@ -90,7 +128,7 @@ missing_dt <- cran_dt[!s3_dt] cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) # --------------------------------------------------------------------------- -# 4. Remove excluded packages +# 5. Remove excluded packages # --------------------------------------------------------------------------- script_dir <- tryCatch( dirname(normalizePath( @@ -114,36 +152,6 @@ if (file.exists(excluded_path)) { cat(sprintf("No excluded-packages.json found at %s -- skipping exclusion step\n", excluded_path)) } -# --------------------------------------------------------------------------- -# 5. Query PostgreSQL for known build failures -# --------------------------------------------------------------------------- -cat("Connecting to PostgreSQL...\n") -con <- DBI::dbConnect( - RPostgres::Postgres(), - dbname = "build_metadata", - host = "r-binaries.devxy.io", - port = 15432, - user = "rpkgs", - password = Sys.getenv("PGPASS"), - sslmode = "require" -) -on.exit(DBI::dbDisconnect(con), add = TRUE) - -errored_pkgs <- DBI::dbGetQuery( - con, - sprintf( - "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE AND platform = '%s' AND arch = '%s'", - platform, arch - ) -) -errored_dt <- as.data.table(errored_pkgs) -if (nrow(errored_dt) > 0) { - setnames(errored_dt, c("Package", "Version")) - setkey(errored_dt, Package, Version) -} - -cat(sprintf("Found %d known build failures for %s/%s\n", nrow(errored_dt), platform, arch)) - # --------------------------------------------------------------------------- # 6. Split into rebuildable vs known failures # ---------------------------------------------------------------------------