refactor: migrate package installation from pak to uvr

bincraft dropped pak in favour of uvr, so the pipelines, helper scripts
and images in this repo move with it.

- add local/uvr-install.sh as the single replacement for pak::pak(); it
  bootstraps a pinned uvr, mints a throwaway project under TMPDIR and
  runs `uvr add --no-install` + `uvr sync --library`, because `uvr add`
  refuses to run outside a project and only `sync` honours --library
- install bincraft via its Forgejo spec (forgejo::codefloe.com/rpkgs/
  bincraft@<tag>) instead of a git:: URL, keeping the git ls-remote tag
  resolution
- pass UVR_R_BIN/UVR_TARGET_LIB from install-bincraft.R so the
  per-R-minor passes target their own R and library
- replace R_PKG_CACHE_DIR with UVR_CACHE_DIR/UVR_PACKAGES_DIR on the
  persistent volume, preserving the amd64-off/arm64-on split
- drop trim_pkgcache_metadata() and its test; uvr's cache does not grow
  the way pkgcache's _metadata dir did
- let uvr install system requirements from its vendored
  r-system-requirements rules, replacing pak::sysreqs_db_update()
- migrate the shiny app image, the alpine reprex and the CRAN loading
  test; ship uvr-install.sh in the build-one image
- track the uvr pin with renovate
This commit is contained in:
Patrick Schratz 2026-07-31 12:16:32 +00:00
commit f3c18b6805
No known key found for this signature in database
GPG key ID: 62050D5BC68AB6DC
7 changed files with 327 additions and 245 deletions

View file

@ -27,7 +27,9 @@ package_cache_files <- c(
"/mnt/cache/packages/s3_cache.rds"
)
if (!all(file.exists(package_cache_files))) {
message("Package snapshot missing from cache; recomputing via packages-to-build.R")
message(
"Package snapshot missing from cache; recomputing via packages-to-build.R"
)
dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
save_rds_atomic <- function(obj, path) {
tmp <- paste0(path, ".tmp.", Sys.getpid())
@ -36,7 +38,10 @@ if (!all(file.exists(package_cache_files))) {
}
source(file.path("local", "packages-to-build.R"))
save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds")
save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
save_rds_atomic(
pkgs[r_minor_sensitive == TRUE],
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds"
)
message("Package snapshot recomputed.")
}
@ -112,20 +117,22 @@ built <- DBI::dbGetQuery(
)
DBI::dbDisconnect(con)
before <- nrow(chunk)
chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ]
sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk))
chunk <- chunk[
!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag),
]
sprintf(
"Skipped %d already-attempted package versions; %d remaining for this job",
before - nrow(chunk),
nrow(chunk)
)
# Read pre-computed S3 listing from install-deps step
# This avoids loading s3fs/reticulate/Python in the build container,
# saving significant memory for pak subprocess forks
# saving significant memory for the dependency-installer subprocesses
s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
sprintf("S3 cache: %s files", length(s3_cache))
n <- nrow(chunk)
# Every `trim_every` packages, bound the pkgcache _metadata dir so a full-platform
# run does not accumulate thousands of ~70 MB snapshots and fill the host disk.
# No-op on amd64 (R_PKG_CACHE_DIR is empty / cache not persisted).
trim_every <- 25L
mapply(
function(pkg, ver, sens, i) {
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
@ -151,12 +158,6 @@ mapply(
upload = TRUE,
store_build_metadata = TRUE
)
if (i %% trim_every == 0L) {
removed <- trim_pkgcache_metadata()
if (removed > 0L) {
cat(sprintf(" [pkgcache trim] removed %d stale _metadata entries\n", removed))
}
}
},
chunk$Package,
chunk$Version,

View file

@ -11,10 +11,11 @@
#
# How it works: list the remote tags with `git ls-remote` (no token needed for
# the public repo), keep the `vX.Y.Z` release tags, pick the highest version,
# and install it with pak. pak is idempotent on the git ref, so re-running keeps
# the package when it is already current and only updates when a newer tag ships.
# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is
# identical across git versions and `system2()` argument handling.
# and install it with uvr via `local/uvr-install.sh`. uvr is idempotent on the
# git ref, so re-running keeps the package when it is already current and only
# updates when a newer tag ships. Filtering/sorting is done in R (not via git's
# `--sort`/refspec) so behaviour is identical across git versions and
# `system2()` argument handling.
repo_url <- Sys.getenv(
"BINCRAFT_GIT_URL",
@ -44,7 +45,43 @@ latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][
]
message(sprintf("Installing latest bincraft release: %s", latest))
pak::pak(sprintf("git::%s@%s", repo_url, latest))
# uvr addresses Forgejo repos as `forgejo::host/owner/repo@ref` rather than as a
# git URL, so drop the scheme and the trailing `.git` from `repo_url`.
spec <- sprintf(
"forgejo::%s@%s",
sub("\\.git$", "", sub("^[a-z]+://", "", repo_url)),
latest
)
# `local/uvr-install.sh` when run from the repo root, `/work/local/` in the
# build-one image, which copies the two scripts into a flatter layout.
helper <- Sys.getenv("UVR_INSTALL_SH", unset = "")
if (!nzchar(helper)) {
candidates <- c("local/uvr-install.sh", "/work/local/uvr-install.sh")
found <- candidates[file.exists(candidates)]
if (length(found) == 0L) {
stop("Could not locate uvr-install.sh; set UVR_INSTALL_SH", call. = FALSE)
}
helper <- found[1L]
}
# Point uvr at the R running this script and at the library it would install
# into, so the per-R-minor passes in the build pipelines (which call a different
# Rscript with R_LIBS_USER pointed elsewhere) target their own R and library.
Sys.setenv(
UVR_R_BIN = file.path(R.home("bin"), "R"),
UVR_TARGET_LIB = .libPaths()[1L]
)
status <- system2(helper, shQuote(spec))
if (!identical(status, 0L)) {
stop(
sprintf("uvr failed to install %s (exit %s)", spec, status),
call. = FALSE
)
}
message(sprintf(
"bincraft %s installed (%s)",
as.character(utils::packageVersion("bincraft")),

View file

@ -15,15 +15,15 @@ suppressPackageStartupMessages(library(data.table))
# Sys.setenv("OS_VERSION" = "3.22")
# Sys.setenv("ARCH" = "arm64")
arch = Sys.getenv("ARCH")
arch <- Sys.getenv("ARCH")
# target: alpine-322, ubuntu-2404, redhat-9, etc.
platform = paste(
platform <- paste(
Sys.getenv("OS"),
gsub("[.]", "", Sys.getenv("OS_VERSION")),
sep = "-"
)
# Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10")
codename = bincraft::set_codename(NULL)
codename <- bincraft::set_codename(NULL)
con <- DBI::dbConnect(
RPostgres::Postgres(),
@ -35,8 +35,8 @@ con <- DBI::dbConnect(
sslmode = "require"
)
cran_archive = tools::CRAN_archive_db()
cran_release = tools::CRAN_package_db()
cran_archive <- tools::CRAN_archive_db()
cran_release <- tools::CRAN_package_db()
# Subset cran_archive to only those packages
cran_archive_in_release <- cran_archive[
names(cran_archive) %in% cran_release$Package
@ -84,13 +84,14 @@ s3fs::s3_file_system(
region_name = "eu-central-003",
refresh = TRUE
)
s3_pkgs = s3fs::s3_dir_ls(
s3_pkgs <- s3fs::s3_dir_ls(
sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename),
recurse = TRUE
)
# Save the raw S3 file listing for the build step to use as s3_package_cache
# This avoids loading s3fs/reticulate in the build container, saving memory for pak forks
# This avoids loading s3fs/reticulate in the build container, saving memory for
# the dependency-installer subprocesses
saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds")
file_names <- basename(s3_pkgs)

View file

@ -31,42 +31,3 @@ parse_build_args <- function(args) {
ncpus = as.integer(pos[3L])
)
}
# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the
# "patched" repo mints a new content hash on every PACKAGES change, so each build
# writes a fresh ~70 MB _metadata/pkgs-<hash>.rds (+ patched-<hash>/) that is
# never reused. Keep the `keep` newest entries by mtime; only remove entries
# older than `min_age_secs`, so a concurrent split-job's in-flight files are
# never deleted (each build uses a unique hash, so aged entries are
# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are
# not matched and thus preserved. Returns the number of entries removed.
trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"),
keep = 20L,
min_age_secs = 600) {
meta <- file.path(cache_dir, "R", "pkgcache", "_metadata")
if (!nzchar(cache_dir) || !dir.exists(meta)) {
return(0L)
}
entries <- c(
Sys.glob(file.path(meta, "patched-*")),
Sys.glob(file.path(meta, "pkgs-*.rds"))
)
if (length(entries) == 0L) {
return(0L)
}
info <- file.info(entries)
order_new_first <- order(info$mtime, decreasing = TRUE)
ranked <- entries[order_new_first]
ranked_mtime <- info$mtime[order_new_first]
if (length(ranked) <= keep) {
return(0L)
}
candidates <- ranked[(keep + 1L):length(ranked)]
candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)])
removable <- candidates[candidate_age >= min_age_secs]
if (length(removable) == 0L) {
return(0L)
}
unlink(removable, recursive = TRUE, force = TRUE)
length(removable)
}

View file

@ -1,22 +1,28 @@
install.packages(
"pak",
repos = sprintf(
"https://r-lib.github.io/p/pak/stable/%s/%s/%s",
.Platform$pkgType,
R.Version()$os,
R.Version()$arch
# Installs every CRAN package one by one and checks that it loads. Dependencies
# go through uvr via local/uvr-install.sh, which bootstraps the uvr binary on
# first use and installs into .libPaths()[1].
uvr_install <- function(pkg) {
Sys.setenv(
UVR_R_BIN = file.path(R.home("bin"), "R"),
UVR_TARGET_LIB = .libPaths()[1L]
)
)
status <- system2("local/uvr-install.sh", shQuote(pkg))
if (!identical(status, 0L)) {
stop(
sprintf("uvr failed to install %s (exit %s)", pkg, status),
call. = FALSE
)
}
}
Sys.setenv(PKG_SYSREQS = TRUE)
all_pkgs <- rownames(available.packages())
to_skip = c("ABRSQOL", "ACA", "ACE.CoCo")
all_pkgs = setdiff(all_pkgs, to_skip)
to_skip <- c("ABRSQOL", "ACA", "ACE.CoCo")
all_pkgs <- setdiff(all_pkgs, to_skip)
for (i in all_pkgs) {
message(sprintf("\nInstalling %s", i))
pak::pkg_install(i)
uvr_install(i)
library(i, character.only = TRUE)
}
@ -323,7 +329,7 @@ if (length(to_process) == 0) {
} else {
for (i in to_process) {
message(sprintf("\nInstalling %s", i))
pak::pkg_install(i)
uvr_install(i)
library(i, character.only = TRUE)
}
# Update to_skip to include all up to the last processed

View file

@ -1,68 +0,0 @@
source(file.path("..", "r-minor-helpers.R"))
# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is
# set to `age_secs` in the past so we can exercise the age gate deterministically.
make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) {
root <- tempfile("pkgcache-")
meta <- file.path(root, "R", "pkgcache", "_metadata")
dir.create(meta, recursive = TRUE)
old <- Sys.time() - age_secs
mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p }
mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p }
for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i)))
for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i)))
if (keep_repos) {
mk_dir(file.path(meta, "CRAN-075c426938"))
mk_dir(file.path(meta, "BioCsoft-1ac964ed6c"))
mk_file(file.path(meta, "bioc-sysreqs.dcf.gz"))
mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive
}
root
}
n_churn <- function(root) {
meta <- file.path(root, "R", "pkgcache", "_metadata")
length(Sys.glob(file.path(meta, "patched-*"))) +
length(Sys.glob(file.path(meta, "pkgs-*.rds")))
}
test_that("empty cache_dir is a no-op", {
expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L)
})
test_that("missing _metadata dir is a no-op", {
expect_identical(
trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0),
0L
)
})
test_that("fewer than keep entries removes nothing", {
root <- make_meta(patched = 2L, pkgs = 2L)
expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L)
expect_identical(n_churn(root), 4L)
})
test_that("trims down to keep newest, leaving churn == keep", {
root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old
removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0)
expect_identical(removed, 40L)
expect_identical(n_churn(root), 20L)
})
test_that("entries younger than min_age_secs are protected", {
root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60)
# keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed
expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L)
expect_identical(n_churn(root), 30L)
})
test_that("stable repo dirs and pkg downloads are never touched", {
root <- make_meta(patched = 30L, pkgs = 30L)
trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0)
meta <- file.path(root, "R", "pkgcache", "_metadata")
expect_true(dir.exists(file.path(meta, "CRAN-075c426938")))
expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c")))
expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz")))
expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg")))
})

90
local/uvr-install.sh Executable file
View file

@ -0,0 +1,90 @@
#!/bin/sh
# Install R packages into the CI library with uvr (https://github.com/nbafrank/uvr).
#
# Usage:
# local/uvr-install.sh httr2 jsonlite
# local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft@v4.4.3
#
# Replaces `pak::pak(...)`. uvr is project-scoped: `uvr add` refuses to run
# outside a project and always writes to `.uvr/library/`, and only
# `uvr sync --library` can target an existing library. The project is therefore
# minted in a scratch directory under TMPDIR and thrown away afterwards; that
# also keeps `uvr init`'s `.Rprofile` out of the repo checkout, where it would
# hijack `.libPaths()` for every other R call in the pipeline.
#
# Pruning is a no-op here: uvr disables it whenever `--library` is passed,
# precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds
# bincraft and its dependencies alongside whatever this script installs).
#
# System dependencies come from uvr's vendored r-system-requirements rules, so
# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed.
#
# Environment:
# UVR_R_BIN R interpreter to install for; set by install-bincraft.R so
# the per-R-minor passes target their own R, not the primary
# R_VERSION fallback interpreter selector (/opt/R/<version>/bin/R)
# UVR_TARGET_LIB target library; defaults to R_LIBS_USER, then to the
# active R's .libPaths()[1] (which is where pak wrote)
# UVR_INSTALL_DIR where the uvr binary lands (default /usr/local/bin)
set -eu
# renovate: datasource=github-releases depName=nbafrank/uvr
UVR_PIN="v0.4.4"
if [ "$#" -eq 0 ]; then
echo "usage: $0 <pkg-spec>..." >&2
exit 2
fi
# The build images keep R under /opt/R/<version> and off PATH. uvr resolves the
# interpreter via PATH and never downloads one unless `uvr r install` is run, so
# put the requested R first.
r_bin="${UVR_R_BIN:-}"
if [ -z "$r_bin" ] && [ -n "${R_VERSION:-}" ] && [ -x "/opt/R/${R_VERSION}/bin/R" ]; then
r_bin="/opt/R/${R_VERSION}/bin/R"
fi
if [ -n "$r_bin" ]; then
PATH="$(dirname "$r_bin"):$PATH"
export PATH
else
r_bin="$(command -v R)"
fi
target_lib="${UVR_TARGET_LIB:-${R_LIBS_USER:-}}"
if [ -z "$target_lib" ]; then
target_lib="$("$r_bin" --no-echo --no-save -e 'cat(.libPaths()[1])')"
fi
if [ -z "$target_lib" ]; then
echo "error: could not determine a target library; set UVR_TARGET_LIB" >&2
exit 2
fi
mkdir -p "$target_lib"
# Pin the manifest to the active R so the lockfile's R stays in step with the
# library's R sentinel. Without that, uvr can decide the library is ABI-stale
# and wipe it -- and this target is shared with bincraft. uvr only discovers R
# via PATH/R_HOME (it does not scan /opt/R), so the R put on PATH above is the
# only candidate this constraint can resolve to.
# shellcheck disable=SC2016 # $major/$minor are R expressions, not shell vars
r_full="$("$r_bin" --no-echo --no-save -e 'cat(paste(R.version$major, R.version$minor, sep = "."))')"
install_dir="${UVR_INSTALL_DIR:-/usr/local/bin}"
uvr_bin="${install_dir}/uvr"
if [ ! -x "$uvr_bin" ]; then
echo "Bootstrapping uvr ${UVR_PIN} into ${install_dir}"
UVR_INSTALL_DIR="$install_dir" UVR_VERSION="$UVR_PIN" \
sh -c 'curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | sh'
fi
project_dir="${TMPDIR:-/tmp}/uvr-ci-$$"
rm -rf "$project_dir"
mkdir -p "$project_dir"
trap 'rm -rf "$project_dir"' EXIT
cd "$project_dir"
"$uvr_bin" init --here --r-version "$r_full"
# --no-install: resolve and lock only. The install happens in the sync below,
# which is the only command that honours --library.
"$uvr_bin" add --no-install "$@"
"$uvr_bin" sync --library "$target_lib" --install-system-deps