From d0a243a1dcf381c590d512a6a3d1a5fe8bddeafc Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 21 Apr 2026 20:30:33 +0200 Subject: [PATCH] fix: audit workflows --- .crow/process-updates-alpine-321-amd64.yaml | 76 ------------------- .crow/process-updates-alpine-321-arm64.yaml | 58 -------------- .crow/process-updates-alpine-322-amd64.yaml | 2 +- .crow/process-updates-alpine-322-arm64.yaml | 4 +- ...weekly-audit-missing-alpine-322-arm64.yaml | 2 + ...ekly-rebuild-missing-alpine-321-amd64.yaml | 59 -------------- ...ekly-rebuild-missing-alpine-321-arm64.yaml | 58 -------------- ...ekly-rebuild-missing-alpine-322-amd64.yaml | 2 +- ...ekly-rebuild-missing-alpine-322-arm64.yaml | 14 ++-- local/weekly-missing-binaries-audit.R | 69 ++++++++--------- 10 files changed, 49 insertions(+), 295 deletions(-) delete mode 100644 .crow/process-updates-alpine-321-amd64.yaml delete mode 100644 .crow/process-updates-alpine-321-arm64.yaml delete mode 100644 .crow/weekly-rebuild-missing-alpine-321-amd64.yaml delete mode 100644 .crow/weekly-rebuild-missing-alpine-321-arm64.yaml diff --git a/.crow/process-updates-alpine-321-amd64.yaml b/.crow/process-updates-alpine-321-amd64.yaml deleted file mode 100644 index e09ce9f..0000000 --- a/.crow/process-updates-alpine-321-amd64.yaml +++ /dev/null @@ -1,76 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-321-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-321-amd64"' - -skip_clone: true - -labels: - agent: artemis - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-alpine-321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - R -q -e 'library(bincraft); upload_package_index(codename = "alpine321", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/process-updates-alpine-321-arm64.yaml b/.crow/process-updates-alpine-321-arm64.yaml deleted file mode 100644 index 0a9ecdf..0000000 --- a/.crow/process-updates-alpine-321-arm64.yaml +++ /dev/null @@ -1,58 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-321-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-321-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-alpine-321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - - R -q -e 'library(bincraft); upload_package_index(codename = "alpine321", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index f8265c5..ed39ccc 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -13,7 +13,7 @@ labels: steps: - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 + image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index 76d3c8b..57d3891 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -13,7 +13,7 @@ labels: steps: - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 + image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 pull: true environment: RED_HAT_DEV_PW: @@ -56,5 +56,3 @@ steps: # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - R -q -e 'library(bincraft); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - diff --git a/.crow/weekly-audit-missing-alpine-322-arm64.yaml b/.crow/weekly-audit-missing-alpine-322-arm64.yaml index 8e6e4cc..1815163 100644 --- a/.crow/weekly-audit-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-audit-missing-alpine-322-arm64.yaml @@ -20,6 +20,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN PLATFORM: alpine-322 ARCH: arm64 R_LIBS_USER: /mnt/cache/R-pkgs diff --git a/.crow/weekly-rebuild-missing-alpine-321-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-321-amd64.yaml deleted file mode 100644 index 62f3b6e..0000000 --- a/.crow/weekly-rebuild-missing-alpine-321-amd64.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-321-amd64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-321-amd64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 - ARCH: amd64 - NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-alpine321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-321_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-321-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-321-arm64.yaml deleted file mode 100644 index de00c9a..0000000 --- a/.crow/weekly-rebuild-missing-alpine-321-arm64.yaml +++ /dev/null @@ -1,58 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-321-arm64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-321-arm64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 - ARCH: arm64 - NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-alpine321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-321_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml index 05b4912..84a7ffe 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml @@ -8,7 +8,7 @@ skip_clone: true steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 + image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 pull: true environment: RED_HAT_DEV_PW: diff --git a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml index b00d2f5..c985d1e 100644 --- a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml +++ b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml @@ -2,13 +2,15 @@ when: - event: cron cron: weekly-rebuild-missing-alpine-322-arm64 - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-322-arm64"' skip_clone: true +labels: + agent: gaia + steps: - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 + image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 pull: true environment: RED_HAT_DEV_PW: @@ -23,6 +25,8 @@ steps: from_secret: REPO_RO_TOKEN GITHUB_PAT: from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN GIT_USER: pat-s R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs @@ -30,8 +34,6 @@ steps: PLATFORM: alpine-322 ARCH: arm64 NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-alpine322:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages @@ -39,7 +41,9 @@ steps: - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-322_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 + - R -q -e 'pak::pak("httr2")' + - R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: kubernetes: resources: diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index 9c2b4ae..046204c 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -38,7 +38,38 @@ cat(sprintf("Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n", platform, arch, s3_codename, os_family)) # --------------------------------------------------------------------------- -# 1. CRAN release packages +# 1. Query PostgreSQL for known build failures (before s3fs init to avoid +# C++ pointer conflicts between s3fs/curl and RPostgres/libpq) +# --------------------------------------------------------------------------- +cat("Connecting to PostgreSQL...\n") +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) + +errored_pkgs <- DBI::dbGetQuery( + con, + sprintf( + "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE AND platform = '%s' AND arch = '%s'", + platform, arch + ) +) +DBI::dbDisconnect(con) + +errored_dt <- as.data.table(errored_pkgs) +if (nrow(errored_dt) > 0) { + setnames(errored_dt, c("Package", "Version")) + setkey(errored_dt, Package, Version) +} +cat(sprintf("Found %d known build failures for %s/%s\n", nrow(errored_dt), platform, arch)) + +# --------------------------------------------------------------------------- +# 2. CRAN release packages (uses curl internally - must come after PG) # --------------------------------------------------------------------------- cat("Fetching CRAN package database...\n") cran_release <- tools::CRAN_package_db() @@ -48,7 +79,7 @@ cran_dt <- data.table( ) # --------------------------------------------------------------------------- -# 2. S3 tarballs +# 3. S3 tarballs # --------------------------------------------------------------------------- cat("Connecting to S3...\n") s3fs::s3_file_system( @@ -82,7 +113,7 @@ s3_dt <- data.table( cat(sprintf("S3 contains %d tarballs for %s/%s\n", nrow(s3_dt), arch, s3_codename)) # --------------------------------------------------------------------------- -# 3. Find missing packages (CRAN release version not in S3) +# 4. Find missing packages (CRAN release version not in S3) # --------------------------------------------------------------------------- setkey(cran_dt, Package, Version) setkey(s3_dt, Package, Version) @@ -90,7 +121,7 @@ missing_dt <- cran_dt[!s3_dt] cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) # --------------------------------------------------------------------------- -# 4. Remove excluded packages +# 5. Remove excluded packages # --------------------------------------------------------------------------- script_dir <- tryCatch( dirname(normalizePath( @@ -114,36 +145,6 @@ if (file.exists(excluded_path)) { cat(sprintf("No excluded-packages.json found at %s -- skipping exclusion step\n", excluded_path)) } -# --------------------------------------------------------------------------- -# 5. Query PostgreSQL for known build failures -# --------------------------------------------------------------------------- -cat("Connecting to PostgreSQL...\n") -con <- DBI::dbConnect( - RPostgres::Postgres(), - dbname = "build_metadata", - host = "r-binaries.devxy.io", - port = 15432, - user = "rpkgs", - password = Sys.getenv("PGPASS"), - sslmode = "require" -) -on.exit(DBI::dbDisconnect(con), add = TRUE) - -errored_pkgs <- DBI::dbGetQuery( - con, - sprintf( - "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE AND platform = '%s' AND arch = '%s'", - platform, arch - ) -) -errored_dt <- as.data.table(errored_pkgs) -if (nrow(errored_dt) > 0) { - setnames(errored_dt, c("Package", "Version")) - setkey(errored_dt, Package, Version) -} - -cat(sprintf("Found %d known build failures for %s/%s\n", nrow(errored_dt), platform, arch)) - # --------------------------------------------------------------------------- # 6. Split into rebuildable vs known failures # ---------------------------------------------------------------------------