feat: R-minor-sensitive binary builds (full + iterative) #84
5 changed files with 106 additions and 54 deletions
style: air-format and prettier the new r-minor build files
commit
67960b7b3b
|
|
@ -10,7 +10,7 @@ labels:
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: 'Install deps and bincraft'
|
- name: 'Install deps and bincraft'
|
||||||
image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}"
|
image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}'
|
||||||
pull: true
|
pull: true
|
||||||
environment:
|
environment:
|
||||||
REPO_RO_TOKEN:
|
REPO_RO_TOKEN:
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ labels:
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: 'Install deps and bincraft'
|
- name: 'Install deps and bincraft'
|
||||||
image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}"
|
image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}'
|
||||||
pull: true
|
pull: true
|
||||||
environment:
|
environment:
|
||||||
REPO_RO_TOKEN:
|
REPO_RO_TOKEN:
|
||||||
|
|
|
||||||
|
|
@ -21,8 +21,14 @@ pkgs <- if (sensitive_only) {
|
||||||
readRDS("/mnt/cache/packages/pkgs_to_build.rds")
|
readRDS("/mnt/cache/packages/pkgs_to_build.rds")
|
||||||
}
|
}
|
||||||
# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive)
|
# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive)
|
||||||
if (is.null(pkgs$r_minor_sensitive)) pkgs$r_minor_sensitive <- FALSE
|
if (is.null(pkgs$r_minor_sensitive)) {
|
||||||
sprintf("Total# of remaining package versions: %s (sensitive_only=%s)", nrow(pkgs), sensitive_only)
|
pkgs$r_minor_sensitive <- FALSE
|
||||||
|
}
|
||||||
|
sprintf(
|
||||||
|
"Total# of remaining package versions: %s (sensitive_only=%s)",
|
||||||
|
nrow(pkgs),
|
||||||
|
sensitive_only
|
||||||
|
)
|
||||||
|
|
||||||
# Split into chunks for this worker
|
# Split into chunks for this worker
|
||||||
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
|
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
|
||||||
|
|
@ -40,27 +46,33 @@ s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
||||||
sprintf("S3 cache: %s files", length(s3_cache))
|
sprintf("S3 cache: %s files", length(s3_cache))
|
||||||
|
|
||||||
n <- nrow(chunk)
|
n <- nrow(chunk)
|
||||||
mapply(function(pkg, ver, sens, i) {
|
mapply(
|
||||||
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
function(pkg, ver, sens, i) {
|
||||||
bincraft::build_binary_package(
|
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
||||||
pkg,
|
bincraft::build_binary_package(
|
||||||
tag = ver,
|
pkg,
|
||||||
is_r_minor_sensitive = isTRUE(sens),
|
tag = ver,
|
||||||
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
is_r_minor_sensitive = isTRUE(sens),
|
||||||
s3_region = "eu-central-003",
|
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||||
s3_bucket = "devxy-rpkgs-binaries",
|
s3_region = "eu-central-003",
|
||||||
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
s3_bucket = "devxy-rpkgs-binaries",
|
||||||
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||||
s3_package_cache = s3_cache,
|
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||||
metadata_db_host = "r-binaries.devxy.io",
|
s3_package_cache = s3_cache,
|
||||||
metadata_db_name = "build_metadata",
|
metadata_db_host = "r-binaries.devxy.io",
|
||||||
metadata_db_table = "single_builds",
|
metadata_db_name = "build_metadata",
|
||||||
metadata_db_user = "rpkgs",
|
metadata_db_table = "single_builds",
|
||||||
metadata_db_password = Sys.getenv("PGPASS"),
|
metadata_db_user = "rpkgs",
|
||||||
metadata_db_sslmode = "require",
|
metadata_db_password = Sys.getenv("PGPASS"),
|
||||||
metadata_db_port = 15432,
|
metadata_db_sslmode = "require",
|
||||||
archive = TRUE,
|
metadata_db_port = 15432,
|
||||||
upload = TRUE,
|
archive = TRUE,
|
||||||
store_build_metadata = TRUE
|
upload = TRUE,
|
||||||
)
|
store_build_metadata = TRUE
|
||||||
}, chunk$Package, chunk$Version, chunk$r_minor_sensitive, seq_len(n))
|
)
|
||||||
|
},
|
||||||
|
chunk$Package,
|
||||||
|
chunk$Version,
|
||||||
|
chunk$r_minor_sensitive,
|
||||||
|
seq_len(n)
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,11 @@ suppressPackageStartupMessages(library(data.table))
|
||||||
|
|
||||||
arch = Sys.getenv("ARCH")
|
arch = Sys.getenv("ARCH")
|
||||||
# target: alpine-322, ubuntu-2404, redhat-9, etc.
|
# target: alpine-322, ubuntu-2404, redhat-9, etc.
|
||||||
platform = paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-")
|
platform = paste(
|
||||||
|
Sys.getenv("OS"),
|
||||||
|
gsub("[.]", "", Sys.getenv("OS_VERSION")),
|
||||||
|
sep = "-"
|
||||||
|
)
|
||||||
# Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10")
|
# Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10")
|
||||||
codename = bincraft::set_codename(NULL)
|
codename = bincraft::set_codename(NULL)
|
||||||
|
|
||||||
|
|
@ -34,7 +38,9 @@ con <- DBI::dbConnect(
|
||||||
cran_archive = tools::CRAN_archive_db()
|
cran_archive = tools::CRAN_archive_db()
|
||||||
cran_release = tools::CRAN_package_db()
|
cran_release = tools::CRAN_package_db()
|
||||||
# Subset cran_archive to only those packages
|
# Subset cran_archive to only those packages
|
||||||
cran_archive_in_release <- cran_archive[names(cran_archive) %in% cran_release$Package]
|
cran_archive_in_release <- cran_archive[
|
||||||
|
names(cran_archive) %in% cran_release$Package
|
||||||
|
]
|
||||||
|
|
||||||
archive_versions <- rbindlist(
|
archive_versions <- rbindlist(
|
||||||
lapply(names(cran_archive), function(pkg) {
|
lapply(names(cran_archive), function(pkg) {
|
||||||
|
|
@ -126,11 +132,15 @@ query_error <- function(pkg, ver) {
|
||||||
# Fetch all relevant columns from the database
|
# Fetch all relevant columns from the database
|
||||||
errored_pkgs <- DBI::dbGetQuery(
|
errored_pkgs <- DBI::dbGetQuery(
|
||||||
con,
|
con,
|
||||||
sprintf("SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", platform, arch)
|
sprintf(
|
||||||
|
"SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'",
|
||||||
|
platform,
|
||||||
|
arch
|
||||||
|
)
|
||||||
)
|
)
|
||||||
errored_pkgs <- as.data.table(errored_pkgs)
|
errored_pkgs <- as.data.table(errored_pkgs)
|
||||||
setkey(pkgs_to_build, Package, Version)
|
setkey(pkgs_to_build, Package, Version)
|
||||||
setnames(errored_pkgs, c("Package","Version"))
|
setnames(errored_pkgs, c("Package", "Version"))
|
||||||
setkey(errored_pkgs, Package, Version)
|
setkey(errored_pkgs, Package, Version)
|
||||||
|
|
||||||
### Final subsetting
|
### Final subsetting
|
||||||
|
|
@ -152,12 +162,14 @@ release_meta <- data.table(
|
||||||
)
|
)
|
||||||
|
|
||||||
meta <- release_meta[Package %in% unique(pkgs$Package)]
|
meta <- release_meta[Package %in% unique(pkgs$Package)]
|
||||||
meta[, triage := mapply(
|
meta[,
|
||||||
classify_from_metadata,
|
triage := mapply(
|
||||||
NeedsCompilation,
|
classify_from_metadata,
|
||||||
LinkingTo,
|
NeedsCompilation,
|
||||||
MoreArgs = list(risky_deps = risky_deps)
|
LinkingTo,
|
||||||
)]
|
MoreArgs = list(risky_deps = risky_deps)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
# Only the "ambiguous" compiled packages need a source grep.
|
# Only the "ambiguous" compiled packages need a source grep.
|
||||||
ambiguous <- meta[triage == "ambiguous", Package]
|
ambiguous <- meta[triage == "ambiguous", Package]
|
||||||
|
|
@ -165,19 +177,35 @@ sensitive_ambiguous <- character()
|
||||||
if (length(ambiguous) > 0L) {
|
if (length(ambiguous) > 0L) {
|
||||||
tmp_src <- file.path(tempdir(), "abi_src")
|
tmp_src <- file.path(tempdir(), "abi_src")
|
||||||
dir.create(tmp_src, showWarnings = FALSE, recursive = TRUE)
|
dir.create(tmp_src, showWarnings = FALSE, recursive = TRUE)
|
||||||
sens <- vapply(ambiguous, function(pkg) {
|
sens <- vapply(
|
||||||
out <- tryCatch({
|
ambiguous,
|
||||||
dl <- utils::download.packages(
|
function(pkg) {
|
||||||
pkg, destdir = tmp_src,
|
out <- tryCatch(
|
||||||
repos = "https://cloud.r-project.org", quiet = TRUE
|
{
|
||||||
)
|
dl <- utils::download.packages(
|
||||||
isTRUE(as.logical(bincraft::needs_per_minor_recompile(dl[1L, 2L])))
|
pkg,
|
||||||
}, error = function(e) TRUE) # fail safe: treat as sensitive
|
destdir = tmp_src,
|
||||||
out
|
repos = "https://cloud.r-project.org",
|
||||||
}, logical(1L))
|
quiet = TRUE
|
||||||
|
)
|
||||||
|
isTRUE(as.logical(bincraft::needs_per_minor_recompile(dl[1L, 2L])))
|
||||||
|
},
|
||||||
|
error = function(e) TRUE
|
||||||
|
) # fail safe: treat as sensitive
|
||||||
|
out
|
||||||
|
},
|
||||||
|
logical(1L)
|
||||||
|
)
|
||||||
sensitive_ambiguous <- ambiguous[sens]
|
sensitive_ambiguous <- ambiguous[sens]
|
||||||
}
|
}
|
||||||
|
|
||||||
sensitive_pkgs <- unique(c(meta[triage == "sensitive", Package], sensitive_ambiguous))
|
sensitive_pkgs <- unique(c(
|
||||||
|
meta[triage == "sensitive", Package],
|
||||||
|
sensitive_ambiguous
|
||||||
|
))
|
||||||
pkgs[, r_minor_sensitive := Package %in% sensitive_pkgs]
|
pkgs[, r_minor_sensitive := Package %in% sensitive_pkgs]
|
||||||
sprintf("R-minor-sensitive packages: %s of %s", length(sensitive_pkgs), uniqueN(pkgs$Package))
|
sprintf(
|
||||||
|
"R-minor-sensitive packages: %s of %s",
|
||||||
|
length(sensitive_pkgs),
|
||||||
|
uniqueN(pkgs$Package)
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -2,15 +2,27 @@ source(file.path("..", "r-minor-helpers.R"))
|
||||||
|
|
||||||
test_that("pure-r (NeedsCompilation != yes) is not sensitive", {
|
test_that("pure-r (NeedsCompilation != yes) is not sensitive", {
|
||||||
expect_identical(classify_from_metadata("no", NA, c("Rcpp")), "not-sensitive")
|
expect_identical(classify_from_metadata("no", NA, c("Rcpp")), "not-sensitive")
|
||||||
expect_identical(classify_from_metadata("", "Rcpp", c("Rcpp")), "not-sensitive")
|
expect_identical(
|
||||||
|
classify_from_metadata("", "Rcpp", c("Rcpp")),
|
||||||
|
"not-sensitive"
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
test_that("LinkingTo a risky dep is sensitive (version constraints stripped)", {
|
test_that("LinkingTo a risky dep is sensitive (version constraints stripped)", {
|
||||||
expect_identical(classify_from_metadata("yes", "Rcpp (>= 1.0)", c("Rcpp")), "sensitive")
|
expect_identical(
|
||||||
expect_identical(classify_from_metadata("yes", "R6,\n cpp11", c("Rcpp", "cpp11")), "sensitive")
|
classify_from_metadata("yes", "Rcpp (>= 1.0)", c("Rcpp")),
|
||||||
|
"sensitive"
|
||||||
|
)
|
||||||
|
expect_identical(
|
||||||
|
classify_from_metadata("yes", "R6,\n cpp11", c("Rcpp", "cpp11")),
|
||||||
|
"sensitive"
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
test_that("compiled but no risky LinkingTo is ambiguous (needs source)", {
|
test_that("compiled but no risky LinkingTo is ambiguous (needs source)", {
|
||||||
expect_identical(classify_from_metadata("yes", "R6", c("Rcpp", "cpp11")), "ambiguous")
|
expect_identical(
|
||||||
|
classify_from_metadata("yes", "R6", c("Rcpp", "cpp11")),
|
||||||
|
"ambiguous"
|
||||||
|
)
|
||||||
expect_identical(classify_from_metadata("yes", NA, c("Rcpp")), "ambiguous")
|
expect_identical(classify_from_metadata("yes", NA, c("Rcpp")), "ambiguous")
|
||||||
})
|
})
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue