From 930a4acd12f2accd80335765e5bb95615f7cf35c Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 20 Oct 2026 08:46:00 +0000 Subject: [PATCH] fix(ci): install the R packages pipelines use instead of relying on the image bincraft v5.5.1 dropped lubridate, dplyr, purrr, data.table and others from its imports, so the pipelines' own calls now depend on whatever the build-env image's site library or the shared R_LIBS_USER cache happens to hold. The arm64 redhat:9 image has no progressr under R 4.4.3, which fails every process-cran-updates-redhat-9-arm64 run since v5.5.1. - Install progressr and lubridate for process-updates, which call them inline. - Install purrr for packages-to-build.R and data.table for the weekly audit. - Drop the unused dplyr and progressr library() calls from packages-to-build.R. - Install RPostgres in every pipeline that talks to the metadata DB, since bincraft silently skips the DB (and the per-minor index aborts) without it. - Reinstall bincraft in the prune re-index step, whose /tmp library does not survive from the prune step. --- .crow/backfill-abi-classification.yaml | 1 + .crow/backfill-flat-safety.yaml | 1 + .crow/build-all-versions-install-deps.yaml | 2 +- .crow/build-all-versions.yaml | 2 +- .crow/process-updates.yaml | 4 ++-- .crow/prune-stale-flat-binaries.yaml | 5 +++++ .crow/rebuild-abi-gap.yaml | 1 + .crow/reindex.yaml | 1 + .crow/repair-unsafe-flat-objects.yaml | 2 ++ .crow/weekly-audit-missing.yaml | 2 +- .crow/weekly-rebuild-missing.yaml | 2 +- .crow/weekly-rebuild-reindex.yaml | 1 + local/packages-to-build.R | 2 -- 13 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.crow/backfill-abi-classification.yaml b/.crow/backfill-abi-classification.yaml index aeab591..6eef94d 100644 --- a/.crow/backfill-abi-classification.yaml +++ b/.crow/backfill-abi-classification.yaml @@ -74,6 +74,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /tmp/R-pkgs - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/4.5.3/bin/R local/uvr-install.sh RPostgres - | set -eu LIMIT="" diff --git a/.crow/backfill-flat-safety.yaml b/.crow/backfill-flat-safety.yaml index cfb2be8..85c9462 100644 --- a/.crow/backfill-flat-safety.yaml +++ b/.crow/backfill-flat-safety.yaml @@ -155,6 +155,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /tmp/R-pkgs - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres - | set -eu LIMIT="" diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 2a6d9c2..e3f652e 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -86,7 +86,7 @@ steps: # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite purrr - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index c5dc8b9..926cdbb 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -152,7 +152,7 @@ steps: # The dependency-prep job may run on another agent whose cache volume is # not visible here. Install the packages needed to recompute the snapshot # and query build metadata in every build job. - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite purrr - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 59f2f33..4bab415 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -189,7 +189,7 @@ steps: - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres progressr lubridate - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi @@ -205,7 +205,7 @@ steps: LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true - R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true + R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres lubridate || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true # Also build current packages whose required dependency tree needs a newer R. R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION" diff --git a/.crow/prune-stale-flat-binaries.yaml b/.crow/prune-stale-flat-binaries.yaml index dc33a9a..b010914 100644 --- a/.crow/prune-stale-flat-binaries.yaml +++ b/.crow/prune-stale-flat-binaries.yaml @@ -167,6 +167,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /tmp/R-pkgs - /opt/R/4.5.3/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/4.5.3/bin/R local/uvr-install.sh RPostgres - | set -eu APPLY="" @@ -208,6 +209,10 @@ steps: echo "Dry run: nothing was deleted, so there is nothing to re-index." exit 0 fi + # /tmp is container-local, so the library installed by the prune step is gone here. + mkdir -p /tmp/R-pkgs + /opt/R/4.5.3/bin/Rscript local/install-bincraft.R + UVR_R_BIN=/opt/R/4.5.3/bin/R local/uvr-install.sh RPostgres /opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' FAILED_MINORS="" for RBIN in /opt/R/[0-9]*/bin/R; do diff --git a/.crow/rebuild-abi-gap.yaml b/.crow/rebuild-abi-gap.yaml index 71ade22..afab781 100644 --- a/.crow/rebuild-abi-gap.yaml +++ b/.crow/rebuild-abi-gap.yaml @@ -193,6 +193,7 @@ steps: export R_LIBS_USER="$$LIB" rm -rf "$$LIB"/00LOCK-* "$$RBIN/Rscript" local/install-bincraft.R + UVR_R_BIN="$$RBIN/R" local/uvr-install.sh RPostgres rc=0 for spec in $$ABI_PACKAGES; do pkg=$${spec%@*} diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml index 2815a38..bb8508f 100644 --- a/.crow/reindex.yaml +++ b/.crow/reindex.yaml @@ -158,6 +158,7 @@ steps: - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres # The codename is detected from the image's /etc/os-release. - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/.crow/repair-unsafe-flat-objects.yaml b/.crow/repair-unsafe-flat-objects.yaml index e0625c0..d11084a 100644 --- a/.crow/repair-unsafe-flat-objects.yaml +++ b/.crow/repair-unsafe-flat-objects.yaml @@ -144,6 +144,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /tmp/R-pkgs - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres - | set -eu CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))') @@ -158,6 +159,7 @@ steps: LIB="/tmp/R-pkgs-$$RMINOR" mkdir -p "$$LIB" R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" local/install-bincraft.R || true + R_LIBS_USER="$$LIB" UVR_R_BIN="$$RBIN" local/uvr-install.sh RPostgres || true while read -r pkg ver; do [ -z "$$pkg" ] && continue R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" \ diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index 2796a47..7072c65 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -137,7 +137,7 @@ steps: - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 jsonlite data.table - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' backend_options: docker: diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index a93a7bd..89badb9 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -389,7 +389,7 @@ steps: - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 backend_options: diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index c2be071..1800deb 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -161,6 +161,7 @@ steps: - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres # The codename is detected from the image's /etc/os-release. - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - | diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 940cf83..32f846e 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -5,10 +5,8 @@ options(error = function() { }) library(bincraft, quietly = TRUE) -suppressPackageStartupMessages(library(dplyr)) library(DBI, quietly = TRUE) library(purrr, quietly = TRUE) -library(progressr, quietly = TRUE) suppressPackageStartupMessages(library(data.table)) # Sys.setenv("OS" = "alpine")