Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
cfa552f54e |
|||
|
93d720a9e6 |
|||
|
771d3a7768 |
|||
|
4c1e9b773c |
41 changed files with 263 additions and 3795 deletions
|
|
@ -28,6 +28,10 @@ matrix:
|
|||
ARCH: amd64
|
||||
- CODENAME: redhat-10
|
||||
ARCH: arm64
|
||||
- CODENAME: alpine322
|
||||
ARCH: amd64
|
||||
- CODENAME: alpine322
|
||||
ARCH: arm64
|
||||
- CODENAME: alpine323
|
||||
ARCH: amd64
|
||||
- CODENAME: alpine323
|
||||
|
|
|
|||
|
|
@ -1,90 +0,0 @@
|
|||
# Populate the ABI classifier cache for packages that predate the per-minor scheme.
|
||||
#
|
||||
# bincraft decides whether a flat binary may be carried into a per-minor index
|
||||
# from its risky-package set. That set is now the union of the per-minor tarball
|
||||
# scan and the `abi_classification` cache, but the cache only holds verdicts for
|
||||
# packages that passed through the build pipeline after the scheme existed.
|
||||
# Everything older has no verdict, and no verdict means "carry it" - which is how
|
||||
# a base64enc built under R 4.4.3 reached R 4.6 clients and died at dyn.load()
|
||||
# on SETLENGTH.
|
||||
#
|
||||
# The verdict is computed from the package source, so it does not depend on OS,
|
||||
# arch or R minor: one run populates the cache for every slot. The arch/codename
|
||||
# below only choose which flat index to enumerate.
|
||||
#
|
||||
# Re-running is cheap. Cached verdicts are returned without touching the network,
|
||||
# so a second run costs one query per package and downloads only what is still
|
||||
# unclassified.
|
||||
#
|
||||
# crow pipeline create devxy/build-cran-binaries --var backfill_abi=run
|
||||
|
||||
variables:
|
||||
# A manual pipeline creation instantiates every file in .crow/, so the default
|
||||
# must be a value the `when` clause rejects.
|
||||
backfill_abi:
|
||||
description: "Set to 'run' to classify the flat index into the ABI cache."
|
||||
options:
|
||||
- none
|
||||
- run
|
||||
default: none
|
||||
BACKFILL_ARCH:
|
||||
description: 'Arch of the flat index to enumerate.'
|
||||
options:
|
||||
- amd64
|
||||
- arm64
|
||||
default: amd64
|
||||
BACKFILL_CODENAME:
|
||||
description: 'Codename of the flat index to enumerate.'
|
||||
default: resolute
|
||||
BACKFILL_LIMIT:
|
||||
description: 'Classify at most this many packages (0 = no limit).'
|
||||
default: '0'
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'backfill_abi == "run"'
|
||||
|
||||
skip_clone: true
|
||||
|
||||
labels:
|
||||
group: rpkgs-amd64
|
||||
|
||||
steps:
|
||||
- name: 'Classify the flat index into the ABI cache'
|
||||
image: reg.devxy.io/rpkgs/build-env-ubuntu:resolute
|
||||
pull: true
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
PGPASS:
|
||||
from_secret: PGPASS
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
GITHUB_PAT:
|
||||
from_secret: GITHUB_PAT
|
||||
GIT_USER: pat-s
|
||||
# Pipeline variables are substituted into the YAML, not exported, so they
|
||||
# have to be bound here to be readable as env vars in the script below.
|
||||
BACKFILL_ARCH: ${BACKFILL_ARCH}
|
||||
BACKFILL_CODENAME: ${BACKFILL_CODENAME}
|
||||
BACKFILL_LIMIT: ${BACKFILL_LIMIT}
|
||||
R_LIBS_USER: /tmp/R-pkgs
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /tmp/R-pkgs
|
||||
- /opt/R/4.5.3/bin/Rscript local/install-bincraft.R
|
||||
- |
|
||||
set -eu
|
||||
LIMIT=""
|
||||
if [ "$$BACKFILL_LIMIT" != "0" ]; then
|
||||
LIMIT="$$BACKFILL_LIMIT"
|
||||
fi
|
||||
/opt/R/4.5.3/bin/Rscript local/backfill-abi-classification.R \
|
||||
"$$BACKFILL_ARCH" "$$BACKFILL_CODENAME" $$LIMIT
|
||||
backend_options:
|
||||
docker:
|
||||
resources:
|
||||
requests:
|
||||
memory: 4Gi
|
||||
cpu: 1000m
|
||||
|
|
@ -1,176 +0,0 @@
|
|||
# Record, per flat-slot object, whether it loads under every supported R minor.
|
||||
#
|
||||
# `union_index_records()` keeps a carried flat record when that object is
|
||||
# recorded safe and drops it otherwise. Objects uploaded before that record
|
||||
# existed have no verdict, so they are dropped: a full re-index removed roughly
|
||||
# 2000 records per per-minor index (16398 across the 4.4 indexes, 20076 across
|
||||
# 4.5, 15532 across 4.6), each one resolving to CRAN source for clients the
|
||||
# routing serves correctly.
|
||||
#
|
||||
# This inspects what is already in the bucket and writes the verdict, which is
|
||||
# what restores that coverage. Decided by symbols rather than heuristics: a
|
||||
# binary fails at dyn.load() under minor M exactly when it references a symbol M
|
||||
# does not export, and the volatile set is derived by diffing the R builds in
|
||||
# the image.
|
||||
#
|
||||
# Re-running is cheap; recorded objects are skipped.
|
||||
#
|
||||
# crow pipeline create devxy/build-cran-binaries --var flat_safety=all
|
||||
|
||||
variables:
|
||||
# A manual pipeline creation instantiates every file in .crow/, so the default
|
||||
# must match no matrix row.
|
||||
flat_safety:
|
||||
description: "Inspect target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
|
||||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
- alpine-324-arm64
|
||||
- redhat-8-amd64
|
||||
- redhat-8-arm64
|
||||
- redhat-9-amd64
|
||||
- redhat-9-arm64
|
||||
- redhat-10-amd64
|
||||
- redhat-10-arm64
|
||||
- ubuntu-2204-amd64
|
||||
- ubuntu-2204-arm64
|
||||
- ubuntu-2404-amd64
|
||||
- ubuntu-2404-arm64
|
||||
- ubuntu-2604-amd64
|
||||
- ubuntu-2604-arm64
|
||||
default: none
|
||||
FLAT_SAFETY_LIMIT:
|
||||
description: 'Inspect at most this many objects per slot (0 = no limit).'
|
||||
default: '0'
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'flat_safety == "all" || flat_safety == "${OS}-${ARCH}"'
|
||||
|
||||
skip_clone: true
|
||||
|
||||
labels:
|
||||
group: rpkgs-${ARCH}
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
|
||||
steps:
|
||||
- name: 'Inspect flat objects and record safety'
|
||||
image: reg.devxy.io/rpkgs/build-env-${IMG}
|
||||
pull: true
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
RED_HAT_DEV_PW:
|
||||
from_secret: RED_HAT_DEV_PW
|
||||
B2_S3_ACCESS_KEY:
|
||||
from_secret: B2_S3_ACCESS_KEY
|
||||
B2_S3_SECRET_KEY:
|
||||
from_secret: B2_S3_SECRET_KEY
|
||||
PGPASS:
|
||||
from_secret: PGPASS
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
# uvr resolves bincraft's dependencies through the GitHub API, including
|
||||
# the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the
|
||||
# failure is not loud: uvr warns, falls back to the registry copy of
|
||||
# cranlike, and the index write then dies on `unused argument (built =
|
||||
# built)` because upstream has no such parameter. Intermittent by nature,
|
||||
# since it depends on the rate-limit window.
|
||||
GITHUB_PAT:
|
||||
from_secret: GITHUB_PAT
|
||||
GIT_USER: pat-s
|
||||
R_LIBS_USER: /tmp/R-pkgs
|
||||
FLAT_SAFETY_ARCH: ${ARCH}
|
||||
FLAT_SAFETY_LIMIT: ${FLAT_SAFETY_LIMIT}
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /tmp/R-pkgs
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
- |
|
||||
set -eu
|
||||
LIMIT=""
|
||||
if [ "$$FLAT_SAFETY_LIMIT" != "0" ]; then
|
||||
LIMIT="$$FLAT_SAFETY_LIMIT"
|
||||
fi
|
||||
# The codename comes from the image's /etc/os-release, exactly as the
|
||||
# re-index step derives it, so it cannot drift from the matrix.
|
||||
CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))')
|
||||
echo "slot: $$FLAT_SAFETY_ARCH/$$CODENAME"
|
||||
/opt/R/$R_VERSION/bin/Rscript local/backfill-flat-safety.R \
|
||||
"$$FLAT_SAFETY_ARCH" "$$CODENAME" $$LIMIT
|
||||
backend_options:
|
||||
docker:
|
||||
resources:
|
||||
requests:
|
||||
memory: 4Gi
|
||||
cpu: 1000m
|
||||
|
|
@ -24,6 +24,7 @@ variables:
|
|||
OS_VERSION:
|
||||
description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).'
|
||||
options:
|
||||
- '3.22'
|
||||
- '3.23'
|
||||
- '3.24'
|
||||
- '8'
|
||||
|
|
@ -34,14 +35,6 @@ variables:
|
|||
- 'resolute'
|
||||
default: '3.24'
|
||||
R_VERSION:
|
||||
# The slot's *primary* R minor: what `local/build-all.R` builds into the
|
||||
# generic slot. The loop below already runs `--sensitive-only` for every
|
||||
# other installed minor, so filling a non-primary minor's gap needs this
|
||||
# left alone, not changed.
|
||||
#
|
||||
# 4.6.0 was briefly offered here (#183) and removed: selecting it for a
|
||||
# slot whose generic binaries are 4.5-built would publish 4.6 binaries
|
||||
# into the generic slot and break every 4.5 client.
|
||||
description: 'Primary R version under /opt/R.'
|
||||
options:
|
||||
- 4.5.3
|
||||
|
|
@ -149,11 +142,7 @@ steps:
|
|||
# install). Pin bincraft here, exactly like the R-minor pass below.
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
# The dependency-prep job may run on another agent whose cache volume is
|
||||
# not visible here. Install the packages needed to recompute the snapshot
|
||||
# and query build metadata in every build job.
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite
|
||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||
- $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1
|
||||
- |
|
||||
PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2)
|
||||
|
|
@ -165,10 +154,7 @@ steps:
|
|||
LIB="/mnt/cache/R-pkgs-$RMINOR"
|
||||
mkdir -p "$LIB"
|
||||
R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
|
||||
R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
|
||||
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true
|
||||
# Also build current packages whose required dependency tree needs a newer R.
|
||||
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION" $SPLIT_INTO $SPLIT_INDEX
|
||||
done
|
||||
# archive missed packages; first arg is the codename (e.g. "alpine324"),
|
||||
# derived via bincraft like the upload step, not paste(OS, OS_VERSION).
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ variables:
|
|||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-322-amd64
|
||||
- alpine-322-arm64
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
|
|
@ -46,6 +48,18 @@ labels:
|
|||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-322
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
OS_ID: alpine322
|
||||
PROCESS_NEW: 'FALSE'
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
OS_ID: alpine322
|
||||
PROCESS_NEW: 'FALSE'
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
|
|
@ -192,7 +206,7 @@ steps:
|
|||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||
# rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
|
||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
|
||||
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"
|
||||
- |
|
||||
|
|
@ -207,8 +221,6 @@ steps:
|
|||
R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
|
||||
R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
|
||||
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true
|
||||
# Also build current packages whose required dependency tree needs a newer R.
|
||||
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION"
|
||||
done
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
||||
- |
|
||||
|
|
|
|||
|
|
@ -1,242 +0,0 @@
|
|||
# Remove flat-slot binaries recorded unable to load under some R minor, then
|
||||
# re-index.
|
||||
#
|
||||
# The list comes from `abi_flat_safety`, so it names the objects the binaries
|
||||
# themselves report as broken, not the ones a source heuristic suspects. That
|
||||
# distinction is the whole cost of the operation: the heuristic calls 93% of
|
||||
# compiled packages risky and would delete roughly 3000 objects per slot to fix
|
||||
# about 230.
|
||||
#
|
||||
# `upload_single_binary()` has written a non-portable package only to the
|
||||
# per-minor slot since v5.3.0, so this is a one-time cleanup of what was
|
||||
# uploaded before that. The flat slot is what every client that cannot be
|
||||
# routed by R minor receives, and such a binary was built under whichever R was
|
||||
# primary at the time, so it dies at load under any minor that dropped a symbol
|
||||
# it uses:
|
||||
#
|
||||
# base64enc.so: undefined symbol: SETLENGTH
|
||||
#
|
||||
# Deleting the flat object does not create a hole. The source fallback
|
||||
# republishes the CRAN source under the same key, so the package stays
|
||||
# installable everywhere and compiles against the client's own R, while routed
|
||||
# clients keep getting the per-minor binary.
|
||||
#
|
||||
# Dry run unless PRUNE_APPLY is 'yes'. Start with an explicit package list:
|
||||
#
|
||||
# crow pipeline create devxy/build-cran-binaries \
|
||||
# --var prune_flat=ubuntu-2604-amd64 \
|
||||
# --var PRUNE_PACKAGES="base64enc lobstr" \
|
||||
# --var PRUNE_APPLY=yes
|
||||
|
||||
variables:
|
||||
# A manual pipeline creation instantiates every file in .crow/, so the default
|
||||
# must match no matrix row.
|
||||
prune_flat:
|
||||
description: "Prune target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
|
||||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
- alpine-324-arm64
|
||||
- redhat-8-amd64
|
||||
- redhat-8-arm64
|
||||
- redhat-9-amd64
|
||||
- redhat-9-arm64
|
||||
- redhat-10-amd64
|
||||
- redhat-10-arm64
|
||||
- ubuntu-2204-amd64
|
||||
- ubuntu-2204-arm64
|
||||
- ubuntu-2404-amd64
|
||||
- ubuntu-2404-arm64
|
||||
- ubuntu-2604-amd64
|
||||
- ubuntu-2604-arm64
|
||||
default: none
|
||||
PRUNE_PACKAGES:
|
||||
description: 'Space-separated package names. Empty means every package the classifier cache calls risky.'
|
||||
default: ''
|
||||
PRUNE_APPLY:
|
||||
description: "Set to 'yes' to actually delete. Anything else is a dry run."
|
||||
options:
|
||||
- 'no'
|
||||
- 'yes'
|
||||
default: 'no'
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'prune_flat == "all" || prune_flat == "${OS}-${ARCH}"'
|
||||
|
||||
skip_clone: true
|
||||
|
||||
labels:
|
||||
group: rpkgs-${ARCH}
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
CODENAME: alpine323
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
CODENAME: alpine323
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
CODENAME: alpine324
|
||||
IMG: alpine:3.24
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
CODENAME: alpine324
|
||||
IMG: alpine:3.24
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
CODENAME: rhel8
|
||||
IMG: redhat:8
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
CODENAME: rhel8
|
||||
IMG: redhat:8
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
CODENAME: rhel9
|
||||
IMG: redhat:9
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
CODENAME: rhel9
|
||||
IMG: redhat:9
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
CODENAME: rhel10
|
||||
IMG: redhat:10
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
CODENAME: rhel10
|
||||
IMG: redhat:10
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
CODENAME: jammy
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
CODENAME: jammy
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
CODENAME: noble
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
CODENAME: noble
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
CODENAME: resolute
|
||||
IMG: ubuntu:resolute
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
CODENAME: resolute
|
||||
IMG: ubuntu:resolute
|
||||
|
||||
steps:
|
||||
- name: 'Prune stale flat binaries'
|
||||
image: reg.devxy.io/rpkgs/build-env-${IMG}
|
||||
pull: true
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
B2_S3_ACCESS_KEY:
|
||||
from_secret: B2_S3_ACCESS_KEY
|
||||
B2_S3_SECRET_KEY:
|
||||
from_secret: B2_S3_SECRET_KEY
|
||||
PGPASS:
|
||||
from_secret: PGPASS
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
GIT_USER: pat-s
|
||||
R_LIBS_USER: /tmp/R-pkgs
|
||||
# Pipeline variables are substituted into the YAML, not exported, so they
|
||||
# have to be bound here to be readable as env vars in the script below.
|
||||
PRUNE_ARCH: ${ARCH}
|
||||
PRUNE_CODENAME: ${CODENAME}
|
||||
PRUNE_PACKAGES: ${PRUNE_PACKAGES}
|
||||
PRUNE_APPLY: ${PRUNE_APPLY}
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /tmp/R-pkgs
|
||||
- /opt/R/4.5.3/bin/Rscript local/install-bincraft.R
|
||||
- |
|
||||
set -eu
|
||||
APPLY=""
|
||||
if [ "$$PRUNE_APPLY" = "yes" ]; then
|
||||
APPLY="--apply"
|
||||
fi
|
||||
/opt/R/4.5.3/bin/Rscript local/prune-stale-flat-binaries.R \
|
||||
"$$PRUNE_ARCH" "$$PRUNE_CODENAME" $$APPLY $$PRUNE_PACKAGES
|
||||
backend_options:
|
||||
docker:
|
||||
resources:
|
||||
requests:
|
||||
memory: 3Gi
|
||||
cpu: 1000m
|
||||
|
||||
# Deleting objects without re-indexing is worse than leaving them: the
|
||||
# published index keeps naming files that no longer exist, so every client
|
||||
# gets a 404 instead of a working source fallback. The re-index is what
|
||||
# republishes them as CRAN source, so it belongs in the same run.
|
||||
- name: 'Re-index the slot'
|
||||
image: reg.devxy.io/rpkgs/build-env-${IMG}
|
||||
pull: true
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
B2_S3_ACCESS_KEY:
|
||||
from_secret: B2_S3_ACCESS_KEY
|
||||
B2_S3_SECRET_KEY:
|
||||
from_secret: B2_S3_SECRET_KEY
|
||||
PGPASS:
|
||||
from_secret: PGPASS
|
||||
R_LIBS_USER: /tmp/R-pkgs
|
||||
PRUNE_APPLY: ${PRUNE_APPLY}
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
if [ "$$PRUNE_APPLY" != "yes" ]; then
|
||||
echo "Dry run: nothing was deleted, so there is nothing to re-index."
|
||||
exit 0
|
||||
fi
|
||||
/opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
||||
/opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
|
||||
done
|
||||
backend_options:
|
||||
docker:
|
||||
resources:
|
||||
requests:
|
||||
memory: 3Gi
|
||||
cpu: 1000m
|
||||
|
||||
# The edge caches indexes for ~370 days, so without this the deletion is
|
||||
# invisible to clients and they keep being handed the removed binary.
|
||||
- name: 'Purge CDN cache'
|
||||
image: reg.devxy.io/docker.io/library/alpine:3.24
|
||||
environment:
|
||||
BUNNYNET_API_KEY:
|
||||
from_secret: BUNNYNET_API_KEY
|
||||
# Same zones as reindex.yaml: the production pull zone id, the alliance
|
||||
# hostname and the staging zone, which shares the B2 origin.
|
||||
BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
|
||||
PRUNE_APPLY: ${PRUNE_APPLY}
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
if [ "$$PRUNE_APPLY" != "yes" ]; then
|
||||
echo "Dry run: nothing changed at the origin, so no purge is needed."
|
||||
exit 0
|
||||
fi
|
||||
apk add --no-cache -q bash curl jq
|
||||
bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
|
||||
|
|
@ -1,214 +0,0 @@
|
|||
# Rebuild specific packages into a per-R-minor slot.
|
||||
#
|
||||
# The per-minor scheme only ever covered packages that passed through the build
|
||||
# pipeline after it existed. A package built into the flat slot before that, and
|
||||
# never updated on CRAN since, is never reclassified and never gets a per-minor
|
||||
# copy - so the union index serves its flat binary to every minor. When that
|
||||
# binary references a symbol the client's R no longer exports, the install fails
|
||||
# at dyn.load():
|
||||
#
|
||||
# base64enc.so: undefined symbol: SETLENGTH
|
||||
#
|
||||
# base64enc calls SETLENGTH, its flat binary was built under R 4.4.3, and R 4.6
|
||||
# does not export SETLENGTH. There was no way to rebuild one package into one
|
||||
# minor from CI, only `just rebuild` from a workstation with the S3 and Postgres
|
||||
# credentials to hand. This adds that path.
|
||||
#
|
||||
# `--sensitive-only` is deliberately NOT passed. That flag gates on the source
|
||||
# classifier, which misses Rcpp: the repair therefore skipped Rcpp entirely,
|
||||
# leaving its 4.5-built flat binary as the only copy and breaking every R 4.4
|
||||
# client on the slot.
|
||||
#
|
||||
# Since bincraft v5.3.0 the destination is decided by the built binary's symbols
|
||||
# in `upload_single_binary()`, so building unconditionally is safe: a package
|
||||
# that needs a per-minor slot lands there, and one that does not lands in the
|
||||
# flat slot. Skipping before the build is what removed that decision.
|
||||
#
|
||||
# Trigger with the slot and an explicit package list:
|
||||
#
|
||||
# crow pipeline create devxy/build-cran-binaries \
|
||||
# --var rebuild_abi_gap=ubuntu-2604-amd64 \
|
||||
# --var ABI_PACKAGES="base64enc@0.1-6 lobstr@1.2.1" \
|
||||
# --var ABI_R_MINOR=4.6
|
||||
|
||||
variables:
|
||||
# A manual pipeline creation instantiates every file in .crow/, so the
|
||||
# default must match no matrix row.
|
||||
rebuild_abi_gap:
|
||||
description: "Rebuild target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
|
||||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
- alpine-324-arm64
|
||||
- redhat-8-amd64
|
||||
- redhat-8-arm64
|
||||
- redhat-9-amd64
|
||||
- redhat-9-arm64
|
||||
- redhat-10-amd64
|
||||
- redhat-10-arm64
|
||||
- ubuntu-2204-amd64
|
||||
- ubuntu-2204-arm64
|
||||
- ubuntu-2404-amd64
|
||||
- ubuntu-2404-arm64
|
||||
- ubuntu-2604-amd64
|
||||
- ubuntu-2604-arm64
|
||||
default: none
|
||||
ABI_PACKAGES:
|
||||
description: 'Space-separated package@version list, e.g. "base64enc@0.1-6 lobstr@1.2.1".'
|
||||
default: ''
|
||||
ABI_R_MINOR:
|
||||
description: 'The R minor to build into.'
|
||||
options:
|
||||
- '4.4'
|
||||
- '4.5'
|
||||
- '4.6'
|
||||
default: '4.6'
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'rebuild_abi_gap == "all" || rebuild_abi_gap == "${OS}-${ARCH}"'
|
||||
|
||||
skip_clone: true
|
||||
|
||||
labels:
|
||||
group: rpkgs-${ARCH}
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
|
||||
steps:
|
||||
- name: 'Rebuild packages into the per-minor slot'
|
||||
image: reg.devxy.io/rpkgs/build-env-${IMG}
|
||||
pull: true
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
RED_HAT_DEV_PW:
|
||||
from_secret: RED_HAT_DEV_PW
|
||||
B2_S3_ACCESS_KEY:
|
||||
from_secret: B2_S3_ACCESS_KEY
|
||||
B2_S3_SECRET_KEY:
|
||||
from_secret: B2_S3_SECRET_KEY
|
||||
PGPASS:
|
||||
from_secret: PGPASS
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
GITHUB_PAT:
|
||||
from_secret: GITHUB_PAT
|
||||
GIT_USER: pat-s
|
||||
# Pipeline variables are substituted into the YAML, not exported, so they
|
||||
# have to be bound here to be readable as env vars in the script below.
|
||||
ABI_PACKAGES: ${ABI_PACKAGES}
|
||||
ABI_R_MINOR: ${ABI_R_MINOR}
|
||||
UVR_CACHE_DIR: /tmp/uvr/cache
|
||||
UVR_PACKAGES_DIR: /tmp/uvr/packages
|
||||
CCACHE_DIR: /tmp/ccache
|
||||
NCPUS: 2
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p $UVR_CACHE_DIR $UVR_PACKAGES_DIR $CCACHE_DIR
|
||||
- |
|
||||
set -eu
|
||||
if [ -z "$$ABI_PACKAGES" ]; then
|
||||
echo "ABI_PACKAGES is empty; nothing to rebuild." >&2
|
||||
exit 1
|
||||
fi
|
||||
RBIN=$(ls -d /opt/R/$${ABI_R_MINOR}*/bin 2>/dev/null | head -1)
|
||||
if [ -z "$$RBIN" ]; then
|
||||
echo "R $$ABI_R_MINOR is not installed in this image." >&2
|
||||
exit 1
|
||||
fi
|
||||
# Container-local library: this workflow mounts no shared cache volume,
|
||||
# so it cannot disturb the build pipeline's per-agent caches.
|
||||
LIB="/tmp/R-pkgs-$$ABI_R_MINOR"
|
||||
mkdir -p "$$LIB"
|
||||
export R_LIBS_USER="$$LIB"
|
||||
rm -rf "$$LIB"/00LOCK-*
|
||||
"$$RBIN/Rscript" local/install-bincraft.R
|
||||
rc=0
|
||||
for spec in $$ABI_PACKAGES; do
|
||||
pkg=$${spec%@*}
|
||||
ver=$${spec#*@}
|
||||
if [ "$$pkg" = "$$ver" ]; then
|
||||
echo "malformed spec '$$spec'; expected package@version" >&2
|
||||
rc=1
|
||||
continue
|
||||
fi
|
||||
echo "=== rebuilding $$pkg $$ver into $$ABI_R_MINOR ==="
|
||||
"$$RBIN/Rscript" local/build-one.R "$$pkg" "$$ver" || rc=1
|
||||
done
|
||||
exit $$rc
|
||||
backend_options:
|
||||
docker:
|
||||
resources:
|
||||
requests:
|
||||
memory: 5Gi
|
||||
cpu: 1000m
|
||||
|
|
@ -1,198 +0,0 @@
|
|||
# Re-index every slot without rebuilding anything.
|
||||
#
|
||||
# `weekly-rebuild-reindex` exists to run after `weekly-rebuild-missing`, so it
|
||||
# depends on that workflow and shares its gate: triggering it manually also
|
||||
# starts hours of package rebuilds. That is the wrong tool when only the index
|
||||
# needs regenerating - after a bincraft release that changes how the index is
|
||||
# written, for instance, where the objects in the bucket are already correct
|
||||
# and only `PACKAGES*` is stale.
|
||||
#
|
||||
# This workflow does the index half on its own. It installs the latest bincraft
|
||||
# release, republishes the generic and per-R-minor indexes for each slot, and
|
||||
# purges the edge. No package is built.
|
||||
#
|
||||
# Trigger with the `reindex` variable set to `all` or to a single
|
||||
# `<os>-<arch>`, e.g.
|
||||
#
|
||||
# crow pipeline create devxy/build-cran-binaries --var reindex=all
|
||||
|
||||
variables:
|
||||
# A manual pipeline creation instantiates every file in .crow/, so the
|
||||
# default must match no matrix row.
|
||||
reindex:
|
||||
description: "Re-index target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
|
||||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
- alpine-324-arm64
|
||||
- redhat-8-amd64
|
||||
- redhat-8-arm64
|
||||
- redhat-9-amd64
|
||||
- redhat-9-arm64
|
||||
- redhat-10-amd64
|
||||
- redhat-10-arm64
|
||||
- ubuntu-2204-amd64
|
||||
- ubuntu-2204-arm64
|
||||
- ubuntu-2404-amd64
|
||||
- ubuntu-2404-arm64
|
||||
- ubuntu-2604-amd64
|
||||
- ubuntu-2604-arm64
|
||||
default: none
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'reindex == "all" || reindex == "${OS}-${ARCH}"'
|
||||
|
||||
skip_clone: true
|
||||
|
||||
labels:
|
||||
group: rpkgs-${ARCH}
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
|
||||
steps:
|
||||
- name: 'Re-index the slot'
|
||||
image: reg.devxy.io/rpkgs/build-env-${IMG}
|
||||
pull: true
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
RED_HAT_DEV_PW:
|
||||
from_secret: RED_HAT_DEV_PW
|
||||
B2_S3_ACCESS_KEY:
|
||||
from_secret: B2_S3_ACCESS_KEY
|
||||
B2_S3_SECRET_KEY:
|
||||
from_secret: B2_S3_SECRET_KEY
|
||||
# The index build reads the ABI classifier cache to decide which packages
|
||||
# may be carried from the flat slot into a per-minor one. Without the
|
||||
# password that query fails, and it fails soft, so the run still succeeds
|
||||
# while quietly using an empty risky set: exactly the state the cache was
|
||||
# added to fix. It logs one connection error per slot.
|
||||
PGPASS:
|
||||
from_secret: PGPASS
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
# uvr resolves bincraft's dependencies through the GitHub API, including
|
||||
# the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the
|
||||
# failure is not loud: uvr warns, falls back to the registry copy of
|
||||
# cranlike, and the index write then dies on `unused argument (built =
|
||||
# built)` because upstream has no such parameter. Intermittent by nature,
|
||||
# since it depends on the rate-limit window.
|
||||
GITHUB_PAT:
|
||||
from_secret: GITHUB_PAT
|
||||
GIT_USER: pat-s
|
||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
R_VERSION: ${R_VERSION}
|
||||
PLATFORM: ${OS}
|
||||
ARCH: ${ARCH}
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /mnt/cache/R-pkgs
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
# The codename is detected from the image's /etc/os-release.
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
||||
- |
|
||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
||||
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
|
||||
done
|
||||
|
||||
- name: Purge CDN cache
|
||||
image: reg.devxy.io/docker.io/library/alpine:3.24
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
BUNNYNET_API_KEY:
|
||||
from_secret: BUNNYNET_API_KEY
|
||||
# cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
|
||||
# Bunny pull zones, so both must be purged after the shared origin changes.
|
||||
# The staging zone is listed too. It shares the B2 origin, so an index
|
||||
# it still holds is a stale copy of the same object, and its
|
||||
# cache_expiration_time is the same ~370 days: without a purge here it
|
||||
# serves pre-reindex indexes indefinitely and any verification run
|
||||
# against it measures the past.
|
||||
BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
|
||||
commands:
|
||||
- apk add --no-cache -q bash curl jq
|
||||
# Crow carries the checkout from the re-index step into this step.
|
||||
- bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
|
||||
# Runs on every row rather than on one designated slot: a cron fires only
|
||||
# its own slot's row, so gating on a named slot would leave every other
|
||||
# slot unpurged. A manual "all" run therefore purges the zone 18 times,
|
||||
# which is a cheap API call and rare.
|
||||
#
|
||||
# Run it even when the re-index above failed: the objects were still
|
||||
# replaced, and a stale edge is exactly what keeps them hidden.
|
||||
when:
|
||||
- status: [success, failure]
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
# pipeline here gates on a variable named after itself for exactly that reason.
|
||||
#
|
||||
# crow pipeline create --branch main \
|
||||
# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.23 \
|
||||
# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \
|
||||
# --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries
|
||||
variables:
|
||||
# Gates this pipeline. A manual pipeline creation instantiates every file in
|
||||
|
|
@ -37,8 +37,9 @@ variables:
|
|||
- ubuntu
|
||||
default: alpine
|
||||
OS_VERSION:
|
||||
description: 'OS image tag. Must match OS (alpine: 3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).'
|
||||
description: 'OS image tag. Must match OS (alpine: 3.22/3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).'
|
||||
options:
|
||||
- '3.22'
|
||||
- '3.23'
|
||||
- '3.24'
|
||||
- '8'
|
||||
|
|
@ -47,7 +48,7 @@ variables:
|
|||
- 'jammy'
|
||||
- 'noble'
|
||||
- 'resolute'
|
||||
default: '3.23'
|
||||
default: '3.22'
|
||||
R_VERSION:
|
||||
description: 'R version whose stamp the slot should advertise.'
|
||||
options:
|
||||
|
|
|
|||
|
|
@ -1,173 +0,0 @@
|
|||
# Rebuild every flat object recorded as unable to load under some R minor.
|
||||
#
|
||||
# The list comes from `abi_flat_safety`, written by the flat-safety backfill
|
||||
# from the binaries themselves, so it names exactly the objects that fail at
|
||||
# dyn.load() somewhere rather than everything a source heuristic suspects. The
|
||||
# heuristic calls 93% of compiled packages risky; the binaries say 4.3%.
|
||||
#
|
||||
# Each object is rebuilt under every R minor installed in the image, with no
|
||||
# `--sensitive-only` gate. Since bincraft v5.3.0 the destination is decided by
|
||||
# the built binary's symbols in `upload_single_binary()`, so a package needing a
|
||||
# per-minor slot lands there and a portable one lands in the flat slot. Gating
|
||||
# on the source classifier is what let Rcpp slip through and break every R 4.4
|
||||
# client on the slot.
|
||||
#
|
||||
# crow pipeline create devxy/build-cran-binaries --var repair_unsafe=all
|
||||
|
||||
variables:
|
||||
# A manual pipeline creation instantiates every file in .crow/, so the default
|
||||
# must match no matrix row.
|
||||
repair_unsafe:
|
||||
description: "Repair target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
|
||||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
- alpine-324-arm64
|
||||
- redhat-8-amd64
|
||||
- redhat-8-arm64
|
||||
- redhat-9-amd64
|
||||
- redhat-9-arm64
|
||||
- redhat-10-amd64
|
||||
- redhat-10-arm64
|
||||
- ubuntu-2204-amd64
|
||||
- ubuntu-2204-arm64
|
||||
- ubuntu-2404-amd64
|
||||
- ubuntu-2404-arm64
|
||||
- ubuntu-2604-amd64
|
||||
- ubuntu-2604-arm64
|
||||
default: none
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'repair_unsafe == "all" || repair_unsafe == "${OS}-${ARCH}"'
|
||||
|
||||
skip_clone: true
|
||||
|
||||
labels:
|
||||
group: rpkgs-${ARCH}
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
|
||||
steps:
|
||||
- name: 'Rebuild objects recorded unsafe'
|
||||
image: reg.devxy.io/rpkgs/build-env-${IMG}
|
||||
pull: true
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
RED_HAT_DEV_PW:
|
||||
from_secret: RED_HAT_DEV_PW
|
||||
B2_S3_ACCESS_KEY:
|
||||
from_secret: B2_S3_ACCESS_KEY
|
||||
B2_S3_SECRET_KEY:
|
||||
from_secret: B2_S3_SECRET_KEY
|
||||
PGPASS:
|
||||
from_secret: PGPASS
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
GITHUB_PAT:
|
||||
from_secret: GITHUB_PAT
|
||||
GIT_USER: pat-s
|
||||
R_LIBS_USER: /tmp/R-pkgs
|
||||
NCPUS: 2
|
||||
REPAIR_ARCH: ${ARCH}
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /tmp/R-pkgs
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
- |
|
||||
set -eu
|
||||
CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))')
|
||||
/opt/R/$R_VERSION/bin/Rscript local/list-unsafe-flat-objects.R \
|
||||
"$$REPAIR_ARCH" "$$CODENAME" > /tmp/unsafe.txt
|
||||
echo "objects to rebuild: $(wc -l < /tmp/unsafe.txt)"
|
||||
rc=0
|
||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||
RV=$(basename "$(dirname "$(dirname "$$RBIN")")")
|
||||
RMINOR=$(echo "$$RV" | cut -d. -f1-2)
|
||||
echo "=== rebuilding under R $$RV ==="
|
||||
LIB="/tmp/R-pkgs-$$RMINOR"
|
||||
mkdir -p "$$LIB"
|
||||
R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" local/install-bincraft.R || true
|
||||
while read -r pkg ver; do
|
||||
[ -z "$$pkg" ] && continue
|
||||
R_LIBS_USER="$$LIB" "$(dirname "$$RBIN")/Rscript" \
|
||||
local/build-one.R "$$pkg" "$$ver" || rc=1
|
||||
done < /tmp/unsafe.txt
|
||||
done
|
||||
exit $$rc
|
||||
backend_options:
|
||||
docker:
|
||||
resources:
|
||||
requests:
|
||||
memory: 5Gi
|
||||
cpu: 1000m
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
# Delete a retired slot's objects from the bucket.
|
||||
#
|
||||
# The config half is a commit, so it runs on a workstation:
|
||||
# just retire-slot <codename> --apply
|
||||
#
|
||||
# This half needs the B2 credentials, which only exist in CI.
|
||||
# `local/retire-slot.R` refuses any slot still listed in `local.rpkgs_slots`,
|
||||
# so this cannot delete a live slot. Both architectures go together.
|
||||
#
|
||||
# Dry run unless RETIRE_APPLY is 'yes':
|
||||
#
|
||||
# crow pipeline create devxy/build-cran-binaries --branch main \
|
||||
# --var retire_slot=alpine322 --var RETIRE_APPLY=yes
|
||||
|
||||
variables:
|
||||
# A manual pipeline creation instantiates every file in .crow/, so the
|
||||
# default must be a value the script never accepts.
|
||||
retire_slot:
|
||||
description: "Codename of the slot to retire (e.g. alpine322), or 'none'."
|
||||
default: none
|
||||
RETIRE_APPLY:
|
||||
description: "Set to 'yes' to delete. Anything else reports and changes nothing."
|
||||
options:
|
||||
- 'no'
|
||||
- 'yes'
|
||||
default: 'no'
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'retire_slot != "none"'
|
||||
|
||||
skip_clone: true
|
||||
|
||||
labels:
|
||||
group: rpkgs-amd64
|
||||
|
||||
steps:
|
||||
- name: 'Delete the retired slot from the bucket'
|
||||
image: reg.devxy.io/rpkgs/build-env-alpine:3.24
|
||||
pull: true
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
B2_S3_ACCESS_KEY:
|
||||
from_secret: B2_S3_ACCESS_KEY
|
||||
B2_S3_SECRET_KEY:
|
||||
from_secret: B2_S3_SECRET_KEY
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
R_LIBS_USER: /tmp/R-pkgs
|
||||
# Pipeline variables are substituted into the YAML, not exported, so they
|
||||
# have to be bound here to be readable as env vars in the script below.
|
||||
RETIRE_SLOT: ${retire_slot}
|
||||
RETIRE_APPLY: ${RETIRE_APPLY}
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /tmp/R-pkgs
|
||||
- /opt/R/4.5.3/bin/Rscript local/install-bincraft.R
|
||||
- |
|
||||
set -eu
|
||||
APPLY=""
|
||||
if [ "$$RETIRE_APPLY" = "yes" ]; then
|
||||
APPLY="--apply"
|
||||
fi
|
||||
/opt/R/4.5.3/bin/Rscript local/retire-slot.R "$$RETIRE_SLOT" --assets-only $$APPLY
|
||||
backend_options:
|
||||
docker:
|
||||
resources:
|
||||
requests:
|
||||
memory: 2Gi
|
||||
cpu: 1000m
|
||||
|
|
@ -38,6 +38,14 @@ labels:
|
|||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-322
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
|
|
|
|||
|
|
@ -1,146 +0,0 @@
|
|||
# Install a fixed set of packages from a published slot under every R minor the
|
||||
# image carries, through each URL form the website tells people to configure,
|
||||
# and check that each one loads.
|
||||
#
|
||||
# This is the end-to-end counterpart to the ABI bookkeeping: `abi_flat_safety`
|
||||
# records what the binaries say about themselves, this proves what a client
|
||||
# actually receives once the CDN, the per-minor indexes and the `Path:` field
|
||||
# have had their say. It runs on the build agents rather than a workstation so
|
||||
# arm64 is covered on real hardware.
|
||||
#
|
||||
# crow pipeline create devxy/build-cran-binaries --var verify_routing=all
|
||||
|
||||
variables:
|
||||
# A manual pipeline creation instantiates every file in .crow/, so the default
|
||||
# must match no matrix row.
|
||||
verify_routing:
|
||||
description: "Verify target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
|
||||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
- alpine-324-arm64
|
||||
- redhat-8-amd64
|
||||
- redhat-8-arm64
|
||||
- redhat-9-amd64
|
||||
- redhat-9-arm64
|
||||
- redhat-10-amd64
|
||||
- redhat-10-arm64
|
||||
- ubuntu-2204-amd64
|
||||
- ubuntu-2204-arm64
|
||||
- ubuntu-2404-amd64
|
||||
- ubuntu-2404-arm64
|
||||
- ubuntu-2604-amd64
|
||||
- ubuntu-2604-arm64
|
||||
default: none
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
evaluate: 'verify_routing == "all" || verify_routing == "${OS}-${ARCH}"'
|
||||
|
||||
skip_clone: true
|
||||
|
||||
labels:
|
||||
group: rpkgs-${ARCH}
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:resolute
|
||||
|
||||
steps:
|
||||
- name: 'Verify routing serves loadable binaries'
|
||||
image: reg.devxy.io/rpkgs/build-env-${IMG}
|
||||
pull: true
|
||||
environment:
|
||||
OTEL_R_TRACES_EXPORTER: none
|
||||
OTEL_R_LOGS_EXPORTER: none
|
||||
OTEL_R_METRICS_EXPORTER: none
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
VERIFY_ARCH: ${ARCH}
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- |
|
||||
set -eu
|
||||
CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))')
|
||||
rc=0
|
||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||
RV=$(basename "$(dirname "$(dirname "$$RBIN")")")
|
||||
for FORM in arch generic flat; do
|
||||
echo "=== R $$RV via $$FORM ==="
|
||||
"$(dirname "$$RBIN")/Rscript" local/verify-repo-routing.R \
|
||||
"$$VERIFY_ARCH" "$$CODENAME" "$$FORM" || rc=1
|
||||
done
|
||||
done
|
||||
exit $$rc
|
||||
backend_options:
|
||||
docker:
|
||||
resources:
|
||||
requests:
|
||||
memory: 4Gi
|
||||
cpu: 1000m
|
||||
|
|
@ -15,6 +15,8 @@ variables:
|
|||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-322-amd64
|
||||
- alpine-322-arm64
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
|
|
@ -46,6 +48,14 @@ labels:
|
|||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-322
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ variables:
|
|||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-322-amd64
|
||||
- alpine-322-arm64
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
|
|
@ -56,6 +58,42 @@ labels:
|
|||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-322
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: alpine-322
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: alpine-322
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 1
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 2
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
SPLIT_INTO: 3
|
||||
SPLIT_INDEX: 3
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
|
|
@ -388,7 +426,7 @@ steps:
|
|||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
|
||||
- $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ variables:
|
|||
options:
|
||||
- none
|
||||
- all
|
||||
- alpine-322-amd64
|
||||
- alpine-322-arm64
|
||||
- alpine-323-amd64
|
||||
- alpine-323-arm64
|
||||
- alpine-324-amd64
|
||||
|
|
@ -56,6 +58,14 @@ labels:
|
|||
|
||||
matrix:
|
||||
include:
|
||||
- OS: alpine-322
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
|
|
@ -135,23 +145,8 @@ steps:
|
|||
from_secret: B2_S3_ACCESS_KEY
|
||||
B2_S3_SECRET_KEY:
|
||||
from_secret: B2_S3_SECRET_KEY
|
||||
# The index build reads the ABI classifier cache to decide which packages
|
||||
# may be carried from the flat slot into a per-minor one. Without the
|
||||
# password that query fails, and it fails soft, so the run still succeeds
|
||||
# while quietly using an empty risky set: exactly the state the cache was
|
||||
# added to fix. It logs one connection error per slot.
|
||||
PGPASS:
|
||||
from_secret: PGPASS
|
||||
REPO_RO_TOKEN:
|
||||
from_secret: REPO_RO_TOKEN
|
||||
# uvr resolves bincraft's dependencies through the GitHub API, including
|
||||
# the pat-s/cranlike@s3 fork. Unauthenticated it is rate limited, and the
|
||||
# failure is not loud: uvr warns, falls back to the registry copy of
|
||||
# cranlike, and the index write then dies on `unused argument (built =
|
||||
# built)` because upstream has no such parameter. Intermittent by nature,
|
||||
# since it depends on the rate-limit window.
|
||||
GITHUB_PAT:
|
||||
from_secret: GITHUB_PAT
|
||||
GIT_USER: pat-s
|
||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
R_VERSION: ${R_VERSION}
|
||||
|
|
@ -180,12 +175,7 @@ steps:
|
|||
from_secret: BUNNYNET_API_KEY
|
||||
# cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate
|
||||
# Bunny pull zones, so both must be purged after the shared origin changes.
|
||||
# The staging zone is listed too. It shares the B2 origin, so an index
|
||||
# it still holds is a stale copy of the same object, and its
|
||||
# cache_expiration_time is the same ~370 days: without a purge here it
|
||||
# serves pre-reindex indexes indefinitely and any verification run
|
||||
# against it measures the past.
|
||||
BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net'
|
||||
BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io'
|
||||
commands:
|
||||
- apk add --no-cache -q bash curl jq
|
||||
# Crow carries the checkout from the re-index step into this step.
|
||||
|
|
|
|||
|
|
@ -24,19 +24,19 @@ repos:
|
|||
args:
|
||||
- --markdown-linebreak-ext=md
|
||||
- repo: https://github.com/DavidAnson/markdownlint-cli2
|
||||
rev: v0.23.3
|
||||
rev: v0.23.2
|
||||
hooks:
|
||||
- id: markdownlint-cli2
|
||||
- repo: https://github.com/rbubley/mirrors-prettier
|
||||
rev: v3.9.10
|
||||
rev: v3.9.6
|
||||
hooks:
|
||||
- id: prettier
|
||||
- repo: https://github.com/posit-dev/air-pre-commit
|
||||
rev: 0.12.0
|
||||
rev: 0.11.0
|
||||
hooks:
|
||||
- id: air-format
|
||||
- repo: https://github.com/editorconfig-checker/editorconfig-checker
|
||||
rev: v4.0.2
|
||||
rev: v3.11.2
|
||||
hooks:
|
||||
- id: editorconfig-checker
|
||||
exclude: ^local/patches/.*\.patch$
|
||||
|
|
|
|||
134
.terraform.lock.hcl
generated
134
.terraform.lock.hcl
generated
|
|
@ -2,79 +2,79 @@
|
|||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.opentofu.org/hashicorp/http" {
|
||||
version = "3.6.2"
|
||||
version = "3.6.1"
|
||||
hashes = [
|
||||
"h1:7+Ewr07oLewLfoe5K7Fhy4T5JUU4y7I7iRGTG2hrzy4=",
|
||||
"h1:8AvJ6H30J1tgxwEoVKxe0B0Ac9XFn1gwOzidrIc+F5o=",
|
||||
"h1:AllxtaK2m88aY+QitHui6kik/qJmBakrH5o5W7vOKI0=",
|
||||
"h1:OimQGm1/9sIkdDCuUb8D5sxDorhektHeCQ337DHDfGk=",
|
||||
"h1:T5zxkFXQ9To+ZysjCZu6gFqE8G1aTGh58Ol20bCk1Fc=",
|
||||
"h1:TZlCCHsmKXSI3FyzmSgCU+TarKOHgiOft0wiYh85Y+E=",
|
||||
"h1:VgyzPdGw1U/KU8gKhefKlIhy5kQYdpoZkpNcGiTgvVs=",
|
||||
"h1:aF+Ztfy9m0Ll6nFOtC90NxN6WIEKAmfmVFhXAtFF8CI=",
|
||||
"h1:bgViIoXF7kMXA7xgNikFYfv1D15rregO8I8Sq95sLGQ=",
|
||||
"h1:d7tCAc1wr1seWpGMBx7tpRyGKOCicvozaHvn5X8ZFfM=",
|
||||
"h1:dRCrpHH2UrkKs/lCqewyA00L0VfOCETVM14NBhyxHaM=",
|
||||
"h1:eIqaNIixQTlhmr2GU1fAFH+eVYLQIfeGw7Ec+JT/bEo=",
|
||||
"h1:hrK+Jzpgivi4Jo356K3LYDoeWBV1bMqKPBEqrEHwgpA=",
|
||||
"h1:mrpIAC+TK3rQQz2jllAF8o2WLMKM14/WoclLgYKc4WA=",
|
||||
"h1:zwE/uNBapBDpkKa5I24moawVPTanbG+eKumHRm/qINw=",
|
||||
"zh:0cf6173a426f65a85e8baf4f9a7808dc94d0e02a82b34bc903555fda1eb45d7a",
|
||||
"zh:2b5fa2b0487e692014fa86745a2a98d43f34f7aaea044acd3e2cee139627c9f1",
|
||||
"zh:2c9c18893ce0f34e077559632c88237d172ac6646853bf00c84a5a4ec3b6c241",
|
||||
"zh:316c848e892764aa10a890cf2455f52b592c6051a92ac8451955ca1d47ca1d2f",
|
||||
"zh:4300e32ecaa967ca0ed56ca48f8dcd10796e70203bf6b8311e71dc43cd0d2aa1",
|
||||
"zh:4a0a555d85a31bd257399253b502ad03816ccdba7b8fbbbd1e07a313d04d83fb",
|
||||
"zh:6278773c31c1605c6d780fc982eacb83944e2807c8d8afe11534253b225c5aa3",
|
||||
"zh:81b73da8dbcedf54604f131e0acd7ebd36cf673276174b4b4f10af0b0f10de11",
|
||||
"zh:887beaa6ef6521d3f5e2b1193d458575fb91855f36bc66dd26a10988127a39fb",
|
||||
"zh:91b2260424c5f8e5968d9e6d5bdbbdb60d61ccc4ff001aa0d637427629349c54",
|
||||
"zh:a5d9d7032d00a6dd844e8d0d374b5ae89e25f9a6635a90e01c5f65e9dd14a93e",
|
||||
"zh:bcf8b2aca448774e4701f221b74d762847be55dc312022b4aad66da3b389bf5d",
|
||||
"zh:d62a3ad37ee1fb943ad0590978f4e7ee0ac064d19554c1c05ec98e9d2bb658f7",
|
||||
"zh:e517b8b81fe88bf49868923f076515b9157771d1d3a80849cc8a73b9a19039a1",
|
||||
"zh:fa9e0e39ef3e5ed2e6cf5886a0b569415ca3b89b9289fd6d01824ff11e470618",
|
||||
"h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=",
|
||||
"h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=",
|
||||
"h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=",
|
||||
"h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=",
|
||||
"h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=",
|
||||
"h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=",
|
||||
"h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=",
|
||||
"h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=",
|
||||
"h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=",
|
||||
"h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=",
|
||||
"h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=",
|
||||
"h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=",
|
||||
"h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=",
|
||||
"h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=",
|
||||
"h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=",
|
||||
"zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9",
|
||||
"zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6",
|
||||
"zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab",
|
||||
"zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b",
|
||||
"zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c",
|
||||
"zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4",
|
||||
"zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502",
|
||||
"zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69",
|
||||
"zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6",
|
||||
"zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279",
|
||||
"zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6",
|
||||
"zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7",
|
||||
"zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df",
|
||||
"zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621",
|
||||
"zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424",
|
||||
]
|
||||
}
|
||||
|
||||
provider "registry.terraform.io/bunnyway/bunnynet" {
|
||||
version = "0.19.2"
|
||||
constraints = "~> 0.19"
|
||||
version = "0.18.2"
|
||||
constraints = "~> 0.18"
|
||||
hashes = [
|
||||
"h1:1RGbDsnU2r+Fp7AVTZkPE29CIY6z/qfSzb47YnCBCTs=",
|
||||
"h1:6tyZaU4EVdCFkSaCF2Rn14s29CjPfiIZzn8rEleluH4=",
|
||||
"h1:9Hb0HF83T+cIl/jFyFBxVjTfSgUZHWlxFQpCcct767M=",
|
||||
"h1:BPwOWd/UY7MNt71Gat+TApEdQtME6LojWBLR3BAGHhw=",
|
||||
"h1:Bvg7QNBJXsniFMQfxYQvRSuZJQYl9vtrIY5k1wyuIww=",
|
||||
"h1:C+Q6JJU8K0yoaXv9n1fKKTsRz92uYaPWQ/N+jxZHXNU=",
|
||||
"h1:GExlP3x5XDYbQvPSXBq8GrWNMbWTxNP005hUnFXzIKY=",
|
||||
"h1:IyU50/IJkOjGJm1zuEH7Ui0w3//xv/V2H+lWcUWLMbM=",
|
||||
"h1:QooUpHtR0U8G0E2p5VThFHiHiy1YHGPnefNrNqFC1Oo=",
|
||||
"h1:RElSrKI43XR3coq4XGfF3AInuAS4cPphqq6NsxIeDU0=",
|
||||
"h1:cL/YPyl/h1ZOuJAJ49xZUO6mKes62uxWJ8esHi7vUmk=",
|
||||
"h1:jQYXBctHWU34H0VoIT7OULdNAgngUEsfo3HJMZIYNeE=",
|
||||
"h1:kZ5NHHgmovqTexzDrksOiAq5tdhQbQBs4SQn+nA23uk=",
|
||||
"h1:lh4A6QWikA6duGgOT8FmdvAKigOo1CEuCFnbdcqLCQs=",
|
||||
"h1:mFkPTvdXHXiDCtRmgAlPcOPYKms4IR0/VrgI0S/tUWQ=",
|
||||
"h1:rjKiZjk7HBNKY5dQPmLI0v7JO4o0zrfXeGB/mMRlnOo=",
|
||||
"h1:uV5e4h8EfgFvppbg1jUfGhEK42xkluVgddbvSzjOu5o=",
|
||||
"zh:0f46034291455bd8382716f7c7b8a942fa3a3e652c7241c3b224573d9855c93a",
|
||||
"zh:10c9f68f4df54353d2c2f3ed2dbe1de970da45418fa5847d933d916ba7c27190",
|
||||
"zh:31e62093ea11e9c234740d02ab5b83dce15cea6e1887e8eb2090584ec58e4b7b",
|
||||
"zh:3656a1c51f7bf8a60584c8798040cdd9159e6d6736160e68c4e3149a45d8b80e",
|
||||
"zh:3706d49aaae6c7be206b500505e52b1b8aa0bb6345806f421d2360d5c38368f6",
|
||||
"zh:4971a5cfa0550eac793bd4ab3fb2e50cf0afe8769a003ef7454e75ddf3c48a91",
|
||||
"zh:5d172a3f57677a37e5bd924ac1ca642125b89616fb1424146b733c73566f6138",
|
||||
"zh:61ab3d637a93032556e72aa602b663acfb4e957ea394877e2ea6c16da70e3a30",
|
||||
"zh:6a2e646c90795a054d389c6be313da284d1347607f0a542799befde5716b01b3",
|
||||
"zh:74c87f30d7b19210de708d669192228da2a29966be5c42bb8d9f4393c427902b",
|
||||
"zh:8846d868864c384004caef674a9338115080ab9ef5c0b94242c6674f83b0b2cc",
|
||||
"h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=",
|
||||
"h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=",
|
||||
"h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=",
|
||||
"h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=",
|
||||
"h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=",
|
||||
"h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=",
|
||||
"h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=",
|
||||
"h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=",
|
||||
"h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=",
|
||||
"h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=",
|
||||
"h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=",
|
||||
"h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=",
|
||||
"h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=",
|
||||
"h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=",
|
||||
"h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=",
|
||||
"h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=",
|
||||
"h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=",
|
||||
"zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c",
|
||||
"zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178",
|
||||
"zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd",
|
||||
"zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef",
|
||||
"zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738",
|
||||
"zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc",
|
||||
"zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c",
|
||||
"zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f",
|
||||
"zh:8bbfa009c19853d1208423b720d9a0e72e0df1ae2ca261a8dfb3c597d447919c",
|
||||
"zh:9c78b14820ef626e896dfb815bc7d2366e3476ef1ec7c148d433c424989c961d",
|
||||
"zh:cb5d42561f86cf20506d0f164d08f2dd93147d58b6359f973033117d74f3d559",
|
||||
"zh:d4b68f306ff8e23a5748f70871e0a7ceaff7d2ae0f4df279db0fe519de868b8b",
|
||||
"zh:e13d71751b65aa64b7d5460ab0626a61f33008b728178547534499c66ff29ae8",
|
||||
"zh:f885b9fb40d4f8da8401b2a62525951e3b8e95b96839b3f32d42b4645df7718d",
|
||||
"zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef",
|
||||
"zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3",
|
||||
"zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb",
|
||||
"zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e",
|
||||
"zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11",
|
||||
"zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7",
|
||||
"zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9",
|
||||
"zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1",
|
||||
"zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff",
|
||||
"zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339",
|
||||
]
|
||||
}
|
||||
|
|
|
|||
39
README.md
39
README.md
|
|
@ -84,23 +84,6 @@ This elastic server architecture offers a robust and performant backend while mi
|
|||
|
||||
## Technical Details
|
||||
|
||||
### Packages requiring a newer R
|
||||
|
||||
The daily update and weekly full-build pipelines run `local/build-newer-r.R` under each non-primary installed R, alongside the ABI-sensitive pass.
|
||||
It selects current CRAN versions whose `Depends: R` constraints cannot be met by the slot's primary R but can be met by the running R, including incompatibility inherited through `Depends`, `Imports`, and `LinkingTo`.
|
||||
These builds retain bincraft's ABI classification and storage routing: sensitive binaries use the running minor, while safe binaries can be shared.
|
||||
No shared S3 existence cache is passed, so bincraft checks the actual objects and distinguishes a source fallback from a binary.
|
||||
The pass reads unfiltered CRAN metadata independently of the weekly snapshot and its primary-R failure exclusions; existing binaries and failures for the running minor retain bincraft's usual skip behavior.
|
||||
Weekly workers split this extra work across their existing shards.
|
||||
This does not select older archive versions for unsupported R minors or change index filtering.
|
||||
|
||||
Preview the selection without building or accessing storage or build metadata:
|
||||
|
||||
```sh
|
||||
# Run with the target R and supply the slot's full primary R version.
|
||||
/opt/R/4.6.0/bin/Rscript local/build-newer-r.R 4.4.3 --dry-run
|
||||
```
|
||||
|
||||
### Creating/Updating the PACKAGES Index Files
|
||||
|
||||
Currently, the {cranlike} and {desc} packages only work with files on a local file system.
|
||||
|
|
@ -215,28 +198,6 @@ Example: <https://cran.devxy.io/arm64/rhel9/latest>
|
|||
|
||||
Technically, no date-based snapshots are planned, so this component from the Posit PM structure is not included.
|
||||
|
||||
## Retiring a slot
|
||||
|
||||
A slot is only reachable if it is listed in `local.rpkgs_slots` in `cdn.tf`, which feeds `UNION_SLOTS` in `edge/rpkgs-router.ts`.
|
||||
An unrouted slot serves the flat index to every client, so its binaries fail at `dyn.load()` under every R minor but one.
|
||||
|
||||
In order:
|
||||
|
||||
1. Remove it from `local.rpkgs_slots` in `cdn.tf` and from `DistroId` in the website's `src/src/data/repos.ts`, then `tofu apply`.
|
||||
2. `just retire-slot <codename> --apply`, then commit.
|
||||
3. Delete its cron jobs on the Crow server; `local/retire-slot.R` prints them with the API call.
|
||||
4. Delete its objects:
|
||||
|
||||
```sh
|
||||
crow pipeline create devxy/build-cran-binaries --branch main \
|
||||
--var retire_slot=<codename> --var RETIRE_APPLY=yes
|
||||
```
|
||||
|
||||
Steps 2 and 4 refuse any slot still listed in `cdn.tf`.
|
||||
Running step 4 before step 2 means the next nightly rebuilds everything just deleted.
|
||||
|
||||
`just check-slots` fails when a pipeline still targets an unrouted slot.
|
||||
|
||||
## Helpers
|
||||
|
||||
Helper scripts are located in `local/`.
|
||||
|
|
|
|||
16
cdn.tf
16
cdn.tf
|
|
@ -89,19 +89,9 @@ resource "bunnynet_compute_script" "rpkgs_router" {
|
|||
# slot that is not listed here would hide every package the per-minor index does
|
||||
# not carry, so this stays empty until a slot has been backfilled.
|
||||
resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" {
|
||||
script = bunnynet_compute_script.rpkgs_router.id
|
||||
name = "UNION_SLOTS"
|
||||
# Enabled. Until this was set, every client resolved against the generic
|
||||
# index and never reached a per-minor binary: an R 4.6.1 client on resolute
|
||||
# downloaded the 4.5-built rlang (2079570 bytes) while the correct 4.6 build
|
||||
# (2075106 bytes) sat unused one directory away, and died at load with
|
||||
# `undefined symbol: SETLENGTH`.
|
||||
#
|
||||
# Verified before enabling, against the staging zone with the same script and
|
||||
# the same origin: all 16 slots report zero regressions against the generic
|
||||
# slot, an excluded R minor is sent to CRAN, a client without an R minor
|
||||
# still gets the generic index, and tarball requests are never rewritten.
|
||||
default_value = join(",", local.rpkgs_slots)
|
||||
script = bunnynet_compute_script.rpkgs_router.id
|
||||
name = "UNION_SLOTS"
|
||||
default_value = ""
|
||||
required = false
|
||||
}
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue