feat: R-minor-sensitive binary builds (full + iterative) #84

Merged
pat-s merged 15 commits from t3code/69b1319c into main 2026-06-14 08:04:41 +00:00
5 changed files with 106 additions and 54 deletions
Showing only changes of commit 67960b7b3b - Show all commits

style: air-format and prettier the new r-minor build files

Patrick Schratz 2026-06-13 21:14:36 +02:00
Signed by: pat-s
GPG key ID: 3C6318841EF78925

View file

@ -10,7 +10,7 @@ labels:
steps: steps:
- name: 'Install deps and bincraft' - name: 'Install deps and bincraft'
image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}'
pull: true pull: true
environment: environment:
REPO_RO_TOKEN: REPO_RO_TOKEN:

View file

@ -9,7 +9,7 @@ labels:
steps: steps:
- name: 'Install deps and bincraft' - name: 'Install deps and bincraft'
image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}'
pull: true pull: true
environment: environment:
REPO_RO_TOKEN: REPO_RO_TOKEN:

View file

@ -21,8 +21,14 @@ pkgs <- if (sensitive_only) {
readRDS("/mnt/cache/packages/pkgs_to_build.rds") readRDS("/mnt/cache/packages/pkgs_to_build.rds")
} }
# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive) # Back-compat: tolerate an older RDS without the column (treat all as non-sensitive)
if (is.null(pkgs$r_minor_sensitive)) pkgs$r_minor_sensitive <- FALSE if (is.null(pkgs$r_minor_sensitive)) {
sprintf("Total# of remaining package versions: %s (sensitive_only=%s)", nrow(pkgs), sensitive_only) pkgs$r_minor_sensitive <- FALSE
}
sprintf(
"Total# of remaining package versions: %s (sensitive_only=%s)",
nrow(pkgs),
sensitive_only
)
# Split into chunks for this worker # Split into chunks for this worker
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE)) chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
@ -40,7 +46,8 @@ s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
sprintf("S3 cache: %s files", length(s3_cache)) sprintf("S3 cache: %s files", length(s3_cache))
n <- nrow(chunk) n <- nrow(chunk)
mapply(function(pkg, ver, sens, i) { mapply(
function(pkg, ver, sens, i) {
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
bincraft::build_binary_package( bincraft::build_binary_package(
pkg, pkg,
@ -63,4 +70,9 @@ mapply(function(pkg, ver, sens, i) {
upload = TRUE, upload = TRUE,
store_build_metadata = TRUE store_build_metadata = TRUE
) )
}, chunk$Package, chunk$Version, chunk$r_minor_sensitive, seq_len(n)) },
chunk$Package,
chunk$Version,
chunk$r_minor_sensitive,
seq_len(n)
)

View file

@ -17,7 +17,11 @@ suppressPackageStartupMessages(library(data.table))
arch = Sys.getenv("ARCH") arch = Sys.getenv("ARCH")
# target: alpine-322, ubuntu-2404, redhat-9, etc. # target: alpine-322, ubuntu-2404, redhat-9, etc.
platform = paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-") platform = paste(
Sys.getenv("OS"),
gsub("[.]", "", Sys.getenv("OS_VERSION")),
sep = "-"
)
# Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10") # Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10")
codename = bincraft::set_codename(NULL) codename = bincraft::set_codename(NULL)
@ -34,7 +38,9 @@ con <- DBI::dbConnect(
cran_archive = tools::CRAN_archive_db() cran_archive = tools::CRAN_archive_db()
cran_release = tools::CRAN_package_db() cran_release = tools::CRAN_package_db()
# Subset cran_archive to only those packages # Subset cran_archive to only those packages
cran_archive_in_release <- cran_archive[names(cran_archive) %in% cran_release$Package] cran_archive_in_release <- cran_archive[
names(cran_archive) %in% cran_release$Package
]
archive_versions <- rbindlist( archive_versions <- rbindlist(
lapply(names(cran_archive), function(pkg) { lapply(names(cran_archive), function(pkg) {
@ -126,11 +132,15 @@ query_error <- function(pkg, ver) {
# Fetch all relevant columns from the database # Fetch all relevant columns from the database
errored_pkgs <- DBI::dbGetQuery( errored_pkgs <- DBI::dbGetQuery(
con, con,
sprintf("SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", platform, arch) sprintf(
"SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'",
platform,
arch
)
) )
errored_pkgs <- as.data.table(errored_pkgs) errored_pkgs <- as.data.table(errored_pkgs)
setkey(pkgs_to_build, Package, Version) setkey(pkgs_to_build, Package, Version)
setnames(errored_pkgs, c("Package","Version")) setnames(errored_pkgs, c("Package", "Version"))
setkey(errored_pkgs, Package, Version) setkey(errored_pkgs, Package, Version)
### Final subsetting ### Final subsetting
@ -152,12 +162,14 @@ release_meta <- data.table(
) )
meta <- release_meta[Package %in% unique(pkgs$Package)] meta <- release_meta[Package %in% unique(pkgs$Package)]
meta[, triage := mapply( meta[,
triage := mapply(
classify_from_metadata, classify_from_metadata,
NeedsCompilation, NeedsCompilation,
LinkingTo, LinkingTo,
MoreArgs = list(risky_deps = risky_deps) MoreArgs = list(risky_deps = risky_deps)
)] )
]
# Only the "ambiguous" compiled packages need a source grep. # Only the "ambiguous" compiled packages need a source grep.
ambiguous <- meta[triage == "ambiguous", Package] ambiguous <- meta[triage == "ambiguous", Package]
@ -165,19 +177,35 @@ sensitive_ambiguous <- character()
if (length(ambiguous) > 0L) { if (length(ambiguous) > 0L) {
tmp_src <- file.path(tempdir(), "abi_src") tmp_src <- file.path(tempdir(), "abi_src")
dir.create(tmp_src, showWarnings = FALSE, recursive = TRUE) dir.create(tmp_src, showWarnings = FALSE, recursive = TRUE)
sens <- vapply(ambiguous, function(pkg) { sens <- vapply(
out <- tryCatch({ ambiguous,
function(pkg) {
out <- tryCatch(
{
dl <- utils::download.packages( dl <- utils::download.packages(
pkg, destdir = tmp_src, pkg,
repos = "https://cloud.r-project.org", quiet = TRUE destdir = tmp_src,
repos = "https://cloud.r-project.org",
quiet = TRUE
) )
isTRUE(as.logical(bincraft::needs_per_minor_recompile(dl[1L, 2L]))) isTRUE(as.logical(bincraft::needs_per_minor_recompile(dl[1L, 2L])))
}, error = function(e) TRUE) # fail safe: treat as sensitive },
error = function(e) TRUE
) # fail safe: treat as sensitive
out out
}, logical(1L)) },
logical(1L)
)
sensitive_ambiguous <- ambiguous[sens] sensitive_ambiguous <- ambiguous[sens]
} }
sensitive_pkgs <- unique(c(meta[triage == "sensitive", Package], sensitive_ambiguous)) sensitive_pkgs <- unique(c(
meta[triage == "sensitive", Package],
sensitive_ambiguous
))
pkgs[, r_minor_sensitive := Package %in% sensitive_pkgs] pkgs[, r_minor_sensitive := Package %in% sensitive_pkgs]
sprintf("R-minor-sensitive packages: %s of %s", length(sensitive_pkgs), uniqueN(pkgs$Package)) sprintf(
"R-minor-sensitive packages: %s of %s",
length(sensitive_pkgs),
uniqueN(pkgs$Package)
)

View file

@ -2,15 +2,27 @@ source(file.path("..", "r-minor-helpers.R"))
test_that("pure-r (NeedsCompilation != yes) is not sensitive", { test_that("pure-r (NeedsCompilation != yes) is not sensitive", {
expect_identical(classify_from_metadata("no", NA, c("Rcpp")), "not-sensitive") expect_identical(classify_from_metadata("no", NA, c("Rcpp")), "not-sensitive")
expect_identical(classify_from_metadata("", "Rcpp", c("Rcpp")), "not-sensitive") expect_identical(
classify_from_metadata("", "Rcpp", c("Rcpp")),
"not-sensitive"
)
}) })
test_that("LinkingTo a risky dep is sensitive (version constraints stripped)", { test_that("LinkingTo a risky dep is sensitive (version constraints stripped)", {
expect_identical(classify_from_metadata("yes", "Rcpp (>= 1.0)", c("Rcpp")), "sensitive") expect_identical(
expect_identical(classify_from_metadata("yes", "R6,\n cpp11", c("Rcpp", "cpp11")), "sensitive") classify_from_metadata("yes", "Rcpp (>= 1.0)", c("Rcpp")),
"sensitive"
)
expect_identical(
classify_from_metadata("yes", "R6,\n cpp11", c("Rcpp", "cpp11")),
"sensitive"
)
}) })
test_that("compiled but no risky LinkingTo is ambiguous (needs source)", { test_that("compiled but no risky LinkingTo is ambiguous (needs source)", {
expect_identical(classify_from_metadata("yes", "R6", c("Rcpp", "cpp11")), "ambiguous") expect_identical(
classify_from_metadata("yes", "R6", c("Rcpp", "cpp11")),
"ambiguous"
)
expect_identical(classify_from_metadata("yes", NA, c("Rcpp")), "ambiguous") expect_identical(classify_from_metadata("yes", NA, c("Rcpp")), "ambiguous")
}) })