devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 12 Pull requests Releases Activity
build-cran-binaries/.crow/process-updates.yaml
pat-s 930a4acd12
 fix(ci): install the R packages pipelines use instead of relying on the image 
bincraft v5.5.1 dropped lubridate, dplyr, purrr, data.table and others from
its imports, so the pipelines' own calls now depend on whatever the build-env
image's site library or the shared R_LIBS_USER cache happens to hold. The arm64
redhat:9 image has no progressr under R 4.4.3, which fails every
process-cran-updates-redhat-9-arm64 run since v5.5.1.

- Install progressr and lubridate for process-updates, which call them inline.
- Install purrr for packages-to-build.R and data.table for the weekly audit.
- Drop the unused dplyr and progressr library() calls from packages-to-build.R.
- Install RPostgres in every pipeline that talks to the metadata DB, since
  bincraft silently skips the DB (and the per-minor index aborts) without it.
- Reinstall bincraft in the prune re-index step, whose /tmp library does not
  survive from the prune step.
2026-10-20 08:46:00 +00:00

253 lines
11 KiB
YAML
Raw Blame History

# Consolidated process-updates pipeline (all platforms, both arches).
# One matrix row per OS/arch replaces the former per-platform files.
# Routing is preserved 1:1:
# - cron: each existing `process-cran-updates-<os>-<arch>` cron fires only
# its matching matrix row (via the per-row `cron:` name filter).
# - manual: pick a target from the `process_cran_updates` dropdown;
# "all" fans out every os/arch as parallel matrix workflows.
# Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64).
variables:
# Gates this pipeline. A manual pipeline creation instantiates every file in
# .crow/, and a declared default is applied even when the run never passed
# this variable, so the default must be a value that matches no matrix row.
process_cran_updates:
description: "Manual run target: a specific <os>-<arch>, 'all' to run every os/arch in parallel, or 'none' to run nothing."
options:
- none
- all
- alpine-323-amd64
- alpine-323-arm64
- alpine-324-amd64
- alpine-324-arm64
- redhat-8-amd64
- redhat-8-arm64
- redhat-9-amd64
- redhat-9-arm64
- redhat-10-amd64
- redhat-10-arm64
- ubuntu-2204-amd64
- ubuntu-2204-arm64
- ubuntu-2404-amd64
- ubuntu-2404-arm64
- ubuntu-2604-amd64
- ubuntu-2604-arm64
default: none
when:
- event: cron
cron: process-cran-updates-${OS}-${ARCH}
- event: manual
evaluate: 'process_cran_updates == "all" || process_cran_updates == "${OS}-${ARCH}"'
skip_clone: true
labels:
group: rpkgs-${ARCH}
matrix:
include:
- OS: alpine-323
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.23
OS_ID: alpine323
PROCESS_NEW: 'FALSE'
- OS: alpine-323
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.23
OS_ID: alpine323
PROCESS_NEW: 'FALSE'
- OS: alpine-324
ARCH: amd64
R_VERSION: 4.5.3
IMG: alpine:3.24
OS_ID: alpine324
PROCESS_NEW: 'FALSE'
- OS: alpine-324
ARCH: arm64
R_VERSION: 4.5.3
IMG: alpine:3.24
OS_ID: alpine324
PROCESS_NEW: 'FALSE'
- OS: redhat-8
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:8
OS_ID: rhel8
PROCESS_NEW: 'TRUE'
- OS: redhat-8
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:8
OS_ID: rhel8
PROCESS_NEW: 'TRUE'
- OS: redhat-9
ARCH: amd64
R_VERSION: 4.4.3
IMG: redhat:9
OS_ID: rhel9
PROCESS_NEW: 'TRUE'
- OS: redhat-9
ARCH: arm64
R_VERSION: 4.4.3
IMG: redhat:9
OS_ID: rhel9
PROCESS_NEW: 'TRUE'
- OS: redhat-10
ARCH: amd64
R_VERSION: 4.5.3
IMG: redhat:10
OS_ID: rhel10
PROCESS_NEW: 'TRUE'
- OS: redhat-10
ARCH: arm64
R_VERSION: 4.5.3
IMG: redhat:10
OS_ID: rhel10
PROCESS_NEW: 'TRUE'
- OS: ubuntu-2204
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
OS_ID: jammy
PROCESS_NEW: 'TRUE'
- OS: ubuntu-2204
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:jammy
OS_ID: jammy
PROCESS_NEW: 'TRUE'
- OS: ubuntu-2404
ARCH: amd64
R_VERSION: 4.4.3
IMG: ubuntu:noble
OS_ID: noble
PROCESS_NEW: 'TRUE'
- OS: ubuntu-2404
ARCH: arm64
R_VERSION: 4.4.3
IMG: ubuntu:noble
OS_ID: noble
PROCESS_NEW: 'TRUE'
- OS: ubuntu-2604
ARCH: amd64
R_VERSION: 4.5.3
IMG: ubuntu:resolute
OS_ID: resolute
PROCESS_NEW: 'TRUE'
- OS: ubuntu-2604
ARCH: arm64
R_VERSION: 4.5.3
IMG: ubuntu:resolute
OS_ID: resolute
PROCESS_NEW: 'TRUE'
steps:
- name: 'Processing Updates'
image: reg.devxy.io/rpkgs/build-env-${IMG}
pull: true
environment:
OTEL_R_TRACES_EXPORTER: none
OTEL_R_LOGS_EXPORTER: none
OTEL_R_METRICS_EXPORTER: none
RED_HAT_DEV_PW:
from_secret: RED_HAT_DEV_PW
B2_S3_ACCESS_KEY:
from_secret: B2_S3_ACCESS_KEY
B2_S3_SECRET_KEY:
from_secret: B2_S3_SECRET_KEY
PGPASS:
from_secret: PGPASS
REPO_RO_TOKEN:
from_secret: REPO_RO_TOKEN
GITHUB_PAT:
from_secret: GITHUB_PAT
# normal env vars
GIT_USER: pat-s
NTFY_SERVER:
from_secret: NTFY_SERVER
NTFY_TOPIC:
from_secret: NTFY_TOPIC
NTFY_AUTH: TRUE
NTFY_PASSWORD:
from_secret: ntfy_token
# set the location of uvr's caches, which persist the R package
# dependencies needed to install the packages themselves
UVR_CACHE_DIR: /mnt/cache/uvr/cache
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
R_LIBS_USER: /mnt/cache/R-pkgs
R_VERSION: ${R_VERSION}
CCACHE_DIR: /mnt/cache/ccache
PLATFORM: ${OS}
OS_ID: ${OS_ID}
ARCH: ${ARCH}
NCPUS: 2
INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres progressr lubridate
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
# rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"
- |
PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2)
for RBIN in /opt/R/[0-9]*/bin/R; do
RV=$(basename "$(dirname "$(dirname "$RBIN")")")
RMINOR=$(echo "$RV" | cut -d. -f1-2)
[ "$RMINOR" = "$PRIMARY_MINOR" ] && continue
echo "=== R-minor-sensitive update pass under R $RV ==="
LIB="/mnt/cache/R-pkgs-$RMINOR"
mkdir -p "$LIB"
R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres lubridate || true
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true
# Also build current packages whose required dependency tree needs a newer R.
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION"
done
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
- |
FAILED_MINORS=""
for RBIN in /opt/R/[0-9]*/bin/R; do
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = '${OS_ID}', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
done
if [ -n "$FAILED_MINORS" ]; then
echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
exit 1
fi
backend_options:
docker:
resources:
requests:
memory: 5Gi
cpu: 3000m
limits:
memory: 18Gi
cpu: 3000m
- name: Purge CDN cache
image: reg.devxy.io/docker.io/library/alpine:3.24
environment:
OTEL_R_TRACES_EXPORTER: none
OTEL_R_LOGS_EXPORTER: none
OTEL_R_METRICS_EXPORTER: none
BUNNYNET_API_KEY:
from_secret: BUNNYNET_API_KEY
SUBDOMAIN1: 'cran.devxy.io'
SUBDOMAIN2: 'cran.allianceswisspass.devxy.io'
SUBDOMAIN3: 'cran.rpkgs.com'
OS_ID: ${OS_ID}
ARCH: ${ARCH}
commands:
- apk add --no-cache -q bash curl
- bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3"
# Run it even when a per-minor re-index above failed: the generic index
# and the objects were still replaced, and a stale edge keeps them hidden.
when:
- status: [success, failure]
Reference in a new issue View git blame Copy permalink