Merge origin/main into t3code/270bb972
Resolve .crow conflicts by taking main's content (which moved bincraft pins to 4.2.1 and added per-minor build/install loops) and re-bumping every bincraft pin to 4.2.2, so all pinned pipelines use the metadata-after-upload release. build-all.R auto-merged: the restart skip-filter coexists with main's r-minor-sensitive build path.
This commit is contained in:
commit
9eeb6f5b9a
GPG key ID:
3C6318841EF78925
23 changed files with 2081 additions and 483 deletions
|
|
@ -11,16 +11,34 @@ s3fs::s3_file_system(
|
|||
arch = "arm64"
|
||||
os = "alpine321"
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", arch, os))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib",
|
||||
arch,
|
||||
os
|
||||
))
|
||||
length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1])))
|
||||
non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) x[1])[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))])
|
||||
non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) {
|
||||
x[1]
|
||||
})[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))])
|
||||
# RInno is not built because it only exists for Windows
|
||||
non_archived = setdiff(non_archived, "RInno")
|
||||
# future::plan("sequential", workers = 8)
|
||||
future::plan("sequential", workers = 1)
|
||||
# future.apply::future_lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch))
|
||||
lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch, s3_region = "hel1", s3_endpoint = "https://hel1.your-objectstorage.com", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S", s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))))
|
||||
lapply(non_archived, function(x) {
|
||||
archive_package(
|
||||
x,
|
||||
codename = os,
|
||||
arch = arch,
|
||||
s3_region = "hel1",
|
||||
s3_endpoint = "https://hel1.your-objectstorage.com",
|
||||
s3_bucket = "devxy-r-package-binaries-hel1",
|
||||
s3_access_key_id = Sys.getenv(
|
||||
"HETZNER_S3_ACCESS_KEY_K3S",
|
||||
s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S")
|
||||
)
|
||||
)
|
||||
})
|
||||
# archive_package()
|
||||
|
||||
|
||||
# grep("duckdb_", files, value = T)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
sink(stdout(), type = "message")
|
||||
options(crayon.enabled = TRUE, future.globals.onReference = NULL)
|
||||
source(file.path("local", "r-minor-helpers.R"))
|
||||
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
split_into <- as.integer(args[1])
|
||||
split_index <- as.integer(args[2])
|
||||
ncpus <- as.integer(args[3])
|
||||
parsed <- parse_build_args(args)
|
||||
split_into <- parsed$split_into
|
||||
split_index <- parsed$split_index
|
||||
ncpus <- parsed$ncpus
|
||||
sensitive_only <- parsed$sensitive_only
|
||||
options(Ncpus = ncpus)
|
||||
|
||||
# Load bincraft eagerly to avoid lazy-load memory spike during first build call
|
||||
|
|
@ -12,9 +15,20 @@ library(bincraft, quietly = TRUE)
|
|||
library(future)
|
||||
plan("sequential")
|
||||
|
||||
# Read precomputed package+version pairs
|
||||
pkgs <- readRDS("/mnt/cache/packages/pkgs_to_build.rds")
|
||||
sprintf("Total# of remaining package versions: %s", nrow(pkgs))
|
||||
pkgs <- if (sensitive_only) {
|
||||
readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
||||
} else {
|
||||
readRDS("/mnt/cache/packages/pkgs_to_build.rds")
|
||||
}
|
||||
# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive)
|
||||
if (is.null(pkgs$r_minor_sensitive)) {
|
||||
pkgs$r_minor_sensitive <- FALSE
|
||||
}
|
||||
sprintf(
|
||||
"Total# of remaining package versions: %s (sensitive_only=%s)",
|
||||
nrow(pkgs),
|
||||
sensitive_only
|
||||
)
|
||||
|
||||
# Split into chunks for this worker
|
||||
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
|
||||
|
|
@ -57,26 +71,33 @@ s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
|||
sprintf("S3 cache: %s files", length(s3_cache))
|
||||
|
||||
n <- nrow(chunk)
|
||||
mapply(function(pkg, ver, i) {
|
||||
cat(sprintf("[%d/%d] %s_%s\n", i, n, pkg, ver))
|
||||
bincraft::build_binary_package(
|
||||
pkg,
|
||||
tag = ver,
|
||||
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
s3_region = "eu-central-003",
|
||||
s3_bucket = "devxy-rpkgs-binaries",
|
||||
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
s3_package_cache = s3_cache,
|
||||
metadata_db_host = "r-binaries.devxy.io",
|
||||
metadata_db_name = "build_metadata",
|
||||
metadata_db_table = "single_builds",
|
||||
metadata_db_user = "rpkgs",
|
||||
metadata_db_password = Sys.getenv("PGPASS"),
|
||||
metadata_db_sslmode = "require",
|
||||
metadata_db_port = 15432,
|
||||
archive = TRUE,
|
||||
upload = TRUE,
|
||||
store_build_metadata = TRUE
|
||||
)
|
||||
}, chunk$Package, chunk$Version, seq_len(n))
|
||||
mapply(
|
||||
function(pkg, ver, sens, i) {
|
||||
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
||||
bincraft::build_binary_package(
|
||||
pkg,
|
||||
tag = ver,
|
||||
is_r_minor_sensitive = isTRUE(sens),
|
||||
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
s3_region = "eu-central-003",
|
||||
s3_bucket = "devxy-rpkgs-binaries",
|
||||
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
s3_package_cache = s3_cache,
|
||||
metadata_db_host = "r-binaries.devxy.io",
|
||||
metadata_db_name = "build_metadata",
|
||||
metadata_db_table = "single_builds",
|
||||
metadata_db_user = "rpkgs",
|
||||
metadata_db_password = Sys.getenv("PGPASS"),
|
||||
metadata_db_sslmode = "require",
|
||||
metadata_db_port = 15432,
|
||||
archive = TRUE,
|
||||
upload = TRUE,
|
||||
store_build_metadata = TRUE
|
||||
)
|
||||
},
|
||||
chunk$Package,
|
||||
chunk$Version,
|
||||
chunk$r_minor_sensitive,
|
||||
seq_len(n)
|
||||
)
|
||||
|
|
|
|||
163
local/build-one.R
Normal file
163
local/build-one.R
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
# Targeted (re)build of specific versions of a single package.
|
||||
# Invoked inside a build-env container (see docker/build-one.Dockerfile).
|
||||
# Usage: build-one.R [--sensitive-only] <package> <version> [<version> ...]
|
||||
# Sensitivity is auto-detected per version via bincraft's ABI classifier:
|
||||
# risky packages go to the per-minor slot, everything else to the generic slot.
|
||||
# With --sensitive-only, non-risky versions are skipped (used for the extra
|
||||
# per-minor passes under non-primary R versions).
|
||||
|
||||
options(crayon.enabled = TRUE, future.globals.onReference = NULL)
|
||||
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
sensitive_only <- "--sensitive-only" %in% args
|
||||
args <- args[args != "--sensitive-only"]
|
||||
if (length(args) < 2L) {
|
||||
stop(
|
||||
"usage: build-one.R [--sensitive-only] <package> <version> [<version> ...]",
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
package <- args[1L]
|
||||
versions <- args[-1L]
|
||||
|
||||
library(bincraft, quietly = TRUE)
|
||||
|
||||
s3 <- list(
|
||||
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
s3_region = "eu-central-003",
|
||||
s3_bucket = "devxy-rpkgs-binaries",
|
||||
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY")
|
||||
)
|
||||
|
||||
# Clone the CRAN source for a version and ask the ABI classifier whether it
|
||||
# must be rebuilt per R minor. Fails safe to TRUE so a possibly-fragile binary
|
||||
# is never served from the cross-minor generic slot by mistake.
|
||||
classify <- function(pkg, ver) {
|
||||
dest <- file.path(tempdir(), sprintf("classify_%s_%s", pkg, ver))
|
||||
on.exit(unlink(dest, recursive = TRUE, force = TRUE), add = TRUE)
|
||||
tryCatch(
|
||||
{
|
||||
message(sprintf(
|
||||
"[classify] cloning %s@%s from github.com/cran ...",
|
||||
pkg,
|
||||
ver
|
||||
))
|
||||
system2(
|
||||
"git",
|
||||
c(
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
"--branch",
|
||||
ver,
|
||||
sprintf("https://github.com/cran/%s", pkg),
|
||||
dest
|
||||
)
|
||||
)
|
||||
message(sprintf("[classify] running ABI classifier on %s ...", pkg))
|
||||
isTRUE(as.logical(bincraft::needs_per_minor_recompile(dest)))
|
||||
},
|
||||
error = function(e) {
|
||||
message(sprintf(
|
||||
"classify failed for %s %s: %s; treating as r-minor-sensitive",
|
||||
pkg,
|
||||
ver,
|
||||
conditionMessage(e)
|
||||
))
|
||||
TRUE
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
minor <- paste(
|
||||
R.version$major,
|
||||
strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L],
|
||||
sep = "."
|
||||
)
|
||||
touched_generic <- FALSE
|
||||
touched_minor <- FALSE
|
||||
|
||||
for (ver in versions) {
|
||||
sensitive <- classify(package, ver)
|
||||
if (sensitive_only && !sensitive) {
|
||||
message(sprintf(
|
||||
"Skipping %s %s under R %s (not r-minor-sensitive)",
|
||||
package,
|
||||
ver,
|
||||
minor
|
||||
))
|
||||
next
|
||||
}
|
||||
cat(sprintf(
|
||||
"Building %s %s (r_minor_sensitive=%s, R %s)\n",
|
||||
package,
|
||||
ver,
|
||||
sensitive,
|
||||
minor
|
||||
))
|
||||
bincraft::build_binary_package(
|
||||
package,
|
||||
tag = ver,
|
||||
is_r_minor_sensitive = sensitive,
|
||||
force = TRUE,
|
||||
upload = TRUE,
|
||||
archive = TRUE,
|
||||
store_build_metadata = TRUE,
|
||||
s3_endpoint = s3$s3_endpoint,
|
||||
s3_region = s3$s3_region,
|
||||
s3_bucket = s3$s3_bucket,
|
||||
s3_access_key_id = s3$s3_access_key_id,
|
||||
s3_secret_access_key = s3$s3_secret_access_key,
|
||||
metadata_db_host = "r-binaries.devxy.io",
|
||||
metadata_db_name = "build_metadata",
|
||||
metadata_db_table = "single_builds",
|
||||
metadata_db_user = "rpkgs",
|
||||
metadata_db_password = Sys.getenv("PGPASS"),
|
||||
metadata_db_sslmode = "require",
|
||||
metadata_db_port = 15432
|
||||
)
|
||||
if (sensitive) touched_minor <- TRUE else touched_generic <- TRUE
|
||||
}
|
||||
|
||||
# Signal the container wrapper whether per-minor passes are warranted (a single
|
||||
# package's sensitivity is the same across R minors, so a non-sensitive package
|
||||
# need not touch any other minor's library).
|
||||
if (touched_minor) {
|
||||
file.create(".r_minor_sensitive")
|
||||
}
|
||||
|
||||
# Refresh the PACKAGES index for each slot we wrote to, so the (re)built binary
|
||||
# is immediately resolvable by clients.
|
||||
codename <- bincraft::set_codename(NULL)
|
||||
# cranlike keeps a working PACKAGES.db in the CWD; clear any copy left by a
|
||||
# previous pass so each per-slot index is built fresh. Otherwise the 2nd index
|
||||
# update in the same container fails with "table packages already exists".
|
||||
clean_index_workdir <- function() {
|
||||
unlink(c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db"))
|
||||
}
|
||||
if (touched_generic) {
|
||||
cat("Refreshing generic index\n")
|
||||
clean_index_workdir()
|
||||
bincraft::upload_package_index(
|
||||
codename = codename,
|
||||
s3_endpoint = s3$s3_endpoint,
|
||||
s3_region = s3$s3_region,
|
||||
s3_bucket = s3$s3_bucket,
|
||||
s3_access_key_id = s3$s3_access_key_id,
|
||||
s3_secret_access_key = s3$s3_secret_access_key
|
||||
)
|
||||
}
|
||||
if (touched_minor) {
|
||||
cat(sprintf("Refreshing per-minor index %s\n", minor))
|
||||
clean_index_workdir()
|
||||
bincraft::upload_package_index(
|
||||
codename = codename,
|
||||
r_minor = minor,
|
||||
s3_endpoint = s3$s3_endpoint,
|
||||
s3_region = s3$s3_region,
|
||||
s3_bucket = s3$s3_bucket,
|
||||
s3_access_key_id = s3$s3_access_key_id,
|
||||
s3_secret_access_key = s3$s3_secret_access_key
|
||||
)
|
||||
}
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
### Scope: Check whether there are any incomplete entries in PACKAGES.rds. These will result in NA when calling available.packages()
|
||||
foo = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
|
|
@ -17,7 +16,9 @@ foo = available.packages("https://cran.devxy.io/amd64/rhel9/latest/src/contrib")
|
|||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/amd64/alpine320/latest/src/contrib")
|
||||
foo = available.packages(
|
||||
"https://cran.devxy.io/amd64/alpine320/latest/src/contrib"
|
||||
)
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
|
|
@ -40,6 +41,8 @@ foo = available.packages("https://cran.devxy.io/arm64/rhel9/latest/src/contrib")
|
|||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/arm64/alpine320/latest/src/contrib")
|
||||
foo = available.packages(
|
||||
"https://cran.devxy.io/arm64/alpine320/latest/src/contrib"
|
||||
)
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
|
|
|||
|
|
@ -2,42 +2,58 @@ library(future)
|
|||
future::plan(multisession)
|
||||
future::plan(sequential)
|
||||
time = Sys.time()
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive"), recurse = T)
|
||||
files <- s3fs::s3_dir_ls(
|
||||
sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive"
|
||||
),
|
||||
recurse = T
|
||||
)
|
||||
Sys.time() - time
|
||||
|
||||
|
||||
|
||||
archive <- llply(dirs, function(dir) {
|
||||
files <- list.files(dir, recursive = FALSE, full.names = TRUE, pattern = "*.tar.gz")
|
||||
files <- list.files(
|
||||
dir,
|
||||
recursive = FALSE,
|
||||
full.names = TRUE,
|
||||
pattern = "*.tar.gz"
|
||||
)
|
||||
if (length(files) == 0) {
|
||||
print(paste0("Error: Empty directory: ", dir))
|
||||
return(NULL)
|
||||
}
|
||||
info <- file.info(files)
|
||||
|
||||
tryCatch({
|
||||
rownames(info) <- paste0(basename(dirname(files)), "/", basename(files))
|
||||
}, error = function(e) {
|
||||
print(paste0("Error: Exception catched for Archived directory: ", dir))
|
||||
print(e)
|
||||
return(NULL)
|
||||
})
|
||||
tryCatch(
|
||||
{
|
||||
rownames(info) <- paste0(basename(dirname(files)), "/", basename(files))
|
||||
},
|
||||
error = function(e) {
|
||||
print(paste0("Error: Exception catched for Archived directory: ", dir))
|
||||
print(e)
|
||||
return(NULL)
|
||||
}
|
||||
)
|
||||
|
||||
info
|
||||
})
|
||||
|
||||
tryCatch({
|
||||
rownames(info) <- paste0(basename(dirname(files)), "/", basename(files))
|
||||
}, error = function(e) {
|
||||
print(paste0("Error: Exception catched for Archived directory: ", dir))
|
||||
print(e)
|
||||
return(NULL)
|
||||
})
|
||||
tryCatch(
|
||||
{
|
||||
rownames(info) <- paste0(basename(dirname(files)), "/", basename(files))
|
||||
},
|
||||
error = function(e) {
|
||||
print(paste0("Error: Exception catched for Archived directory: ", dir))
|
||||
print(e)
|
||||
return(NULL)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
curl::curl_download("https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db")
|
||||
curl::curl_download(
|
||||
"https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db",
|
||||
"PACKAGES.db"
|
||||
)
|
||||
con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db")
|
||||
df = DBI::dbReadTable(con, "packages")
|
||||
|
||||
|
|
@ -50,6 +66,14 @@ which(grepl("digest", df$Package))
|
|||
system("cat /tmp/PACKAGES | grep '^Package: digest' | wc -l")
|
||||
|
||||
|
||||
curl::curl_fetch_memory("https://cloud.r-project.org/src/contrib/Meta/archive.rds")
|
||||
remotes = readRDS(url("https://cloud.r-project.org/src/contrib/Meta/archive.rds", "rb"))
|
||||
packages = readRDS(url("https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds", "rb"))
|
||||
curl::curl_fetch_memory(
|
||||
"https://cloud.r-project.org/src/contrib/Meta/archive.rds"
|
||||
)
|
||||
remotes = readRDS(url(
|
||||
"https://cloud.r-project.org/src/contrib/Meta/archive.rds",
|
||||
"rb"
|
||||
))
|
||||
packages = readRDS(url(
|
||||
"https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds",
|
||||
"rb"
|
||||
))
|
||||
|
|
|
|||
|
|
@ -37,10 +37,13 @@ future.apply::future_lapply(
|
|||
files <- s3fs::s3_dir_ls(
|
||||
sprintf(
|
||||
"devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib",
|
||||
arch, codename
|
||||
arch,
|
||||
codename
|
||||
)
|
||||
)
|
||||
if (length(files) == 0) return(NULL)
|
||||
if (length(files) == 0) {
|
||||
return(NULL)
|
||||
}
|
||||
files_b <- basename(files)
|
||||
pkg_names <- sapply(strsplit(files_b, "_"), `[`, 1)
|
||||
dupes <- unique(pkg_names[duplicated(pkg_names)])
|
||||
|
|
|
|||
|
|
@ -1,77 +1,77 @@
|
|||
[
|
||||
{"package": "RInno", "reason": "windows-only"},
|
||||
{"package": "KeyboardSimulator", "reason": "windows-only"},
|
||||
{"package": "R2PPT", "reason": "windows-only"},
|
||||
{"package": "RWinEdt", "reason": "windows-only"},
|
||||
{"package": "blatr", "reason": "windows-only"},
|
||||
{"package": "excel.link", "reason": "windows-only"},
|
||||
{"package": "spectrino", "reason": "windows-only"},
|
||||
{"package": "taskscheduleR", "reason": "windows-only"},
|
||||
{"package": "MDSGUI", "reason": "windows-only"},
|
||||
{"package": "BiplotGUI", "reason": "windows-only"},
|
||||
{"package": "R2wd", "reason": "windows-only"},
|
||||
{"package": "rFUSION", "reason": "windows-only"},
|
||||
{"package": "MediaNews", "reason": "windows-only"},
|
||||
{"package": "doBy", "reason": "hang"},
|
||||
{"package": "IDPmisc", "reason": "hang"},
|
||||
{"package": "frailtypack", "reason": "hang"},
|
||||
{"package": "afex", "reason": "hang"},
|
||||
{"package": "FrF2", "reason": "hang"},
|
||||
{"package": "DoE.base", "reason": "hang"},
|
||||
{"package": "agricolae", "reason": "hang"},
|
||||
{"package": "doFuture", "reason": "hang"},
|
||||
{"package": "fscaret", "reason": "hang"},
|
||||
{"package": "PHYLOGR", "reason": "hang"},
|
||||
{"package": "seewave", "reason": "hang"},
|
||||
{"package": "pls", "reason": "hang"},
|
||||
{"package": "relaimpo", "reason": "hang"},
|
||||
{"package": "geepack", "reason": "hang"},
|
||||
{"package": "gggenes", "reason": "hang"},
|
||||
{"package": "NPCirc", "reason": "hang"},
|
||||
{"package": "repmis", "reason": "hang"},
|
||||
{"package": "PNDSIBGE", "reason": "hang"},
|
||||
{"package": "lidR", "reason": "hang"},
|
||||
{"package": "poismf", "reason": "hang"},
|
||||
{"package": "neonstore", "reason": "hang"},
|
||||
{"package": "MachineShop", "reason": "hang"},
|
||||
{"package": "mvst", "reason": "hang"},
|
||||
{"package": "MacBehaviour", "reason": "hang"},
|
||||
{"package": "mcmcderive", "reason": "hang"},
|
||||
{"package": "RGIFT", "reason": "hang"},
|
||||
{"package": "KnowBR", "reason": "hang"},
|
||||
{"package": "netmeta", "reason": "hang"},
|
||||
{"package": "spdep", "reason": "hang"},
|
||||
{"package": "Rfast", "reason": "hang"},
|
||||
{"package": "compareGroups", "reason": "hang"},
|
||||
{"package": "ff", "reason": "hang"},
|
||||
{"package": "GsymPoint", "reason": "hang"},
|
||||
{"package": "RcppDynProg", "reason": "hang"},
|
||||
{"package": "comtradr", "reason": "hang"},
|
||||
{"package": "FD", "reason": "hang"},
|
||||
{"package": "PearsonDS", "reason": "hang"},
|
||||
{"package": "DCluster", "reason": "hang"},
|
||||
{"package": "gRc", "reason": "hang"},
|
||||
{"package": "mixlm", "reason": "hang"},
|
||||
{"package": "geospt", "reason": "hang"},
|
||||
{"package": "fdth", "reason": "hang"},
|
||||
{"package": "ffmanova", "reason": "hang"},
|
||||
{"package": "fiery", "reason": "hang"},
|
||||
{"package": "ffscrapr", "reason": "hang"},
|
||||
{"package": "cold", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.DoE", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.NMBU", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.RiskDemo", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.ROC", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.TeachStat", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.TeachingDemos", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.UCA", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.WorldFlora", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.aRnova", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.depthTools", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.orloca", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.sos", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.survival", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.temis", "reason": "hang"},
|
||||
{"package": "GWlasso", "reason": "hang"},
|
||||
{"package": "GWmodelVis", "reason": "hang"}
|
||||
{ "package": "RInno", "reason": "windows-only" },
|
||||
{ "package": "KeyboardSimulator", "reason": "windows-only" },
|
||||
{ "package": "R2PPT", "reason": "windows-only" },
|
||||
{ "package": "RWinEdt", "reason": "windows-only" },
|
||||
{ "package": "blatr", "reason": "windows-only" },
|
||||
{ "package": "excel.link", "reason": "windows-only" },
|
||||
{ "package": "spectrino", "reason": "windows-only" },
|
||||
{ "package": "taskscheduleR", "reason": "windows-only" },
|
||||
{ "package": "MDSGUI", "reason": "windows-only" },
|
||||
{ "package": "BiplotGUI", "reason": "windows-only" },
|
||||
{ "package": "R2wd", "reason": "windows-only" },
|
||||
{ "package": "rFUSION", "reason": "windows-only" },
|
||||
{ "package": "MediaNews", "reason": "windows-only" },
|
||||
{ "package": "doBy", "reason": "hang" },
|
||||
{ "package": "IDPmisc", "reason": "hang" },
|
||||
{ "package": "frailtypack", "reason": "hang" },
|
||||
{ "package": "afex", "reason": "hang" },
|
||||
{ "package": "FrF2", "reason": "hang" },
|
||||
{ "package": "DoE.base", "reason": "hang" },
|
||||
{ "package": "agricolae", "reason": "hang" },
|
||||
{ "package": "doFuture", "reason": "hang" },
|
||||
{ "package": "fscaret", "reason": "hang" },
|
||||
{ "package": "PHYLOGR", "reason": "hang" },
|
||||
{ "package": "seewave", "reason": "hang" },
|
||||
{ "package": "pls", "reason": "hang" },
|
||||
{ "package": "relaimpo", "reason": "hang" },
|
||||
{ "package": "geepack", "reason": "hang" },
|
||||
{ "package": "gggenes", "reason": "hang" },
|
||||
{ "package": "NPCirc", "reason": "hang" },
|
||||
{ "package": "repmis", "reason": "hang" },
|
||||
{ "package": "PNDSIBGE", "reason": "hang" },
|
||||
{ "package": "lidR", "reason": "hang" },
|
||||
{ "package": "poismf", "reason": "hang" },
|
||||
{ "package": "neonstore", "reason": "hang" },
|
||||
{ "package": "MachineShop", "reason": "hang" },
|
||||
{ "package": "mvst", "reason": "hang" },
|
||||
{ "package": "MacBehaviour", "reason": "hang" },
|
||||
{ "package": "mcmcderive", "reason": "hang" },
|
||||
{ "package": "RGIFT", "reason": "hang" },
|
||||
{ "package": "KnowBR", "reason": "hang" },
|
||||
{ "package": "netmeta", "reason": "hang" },
|
||||
{ "package": "spdep", "reason": "hang" },
|
||||
{ "package": "Rfast", "reason": "hang" },
|
||||
{ "package": "compareGroups", "reason": "hang" },
|
||||
{ "package": "ff", "reason": "hang" },
|
||||
{ "package": "GsymPoint", "reason": "hang" },
|
||||
{ "package": "RcppDynProg", "reason": "hang" },
|
||||
{ "package": "comtradr", "reason": "hang" },
|
||||
{ "package": "FD", "reason": "hang" },
|
||||
{ "package": "PearsonDS", "reason": "hang" },
|
||||
{ "package": "DCluster", "reason": "hang" },
|
||||
{ "package": "gRc", "reason": "hang" },
|
||||
{ "package": "mixlm", "reason": "hang" },
|
||||
{ "package": "geospt", "reason": "hang" },
|
||||
{ "package": "fdth", "reason": "hang" },
|
||||
{ "package": "ffmanova", "reason": "hang" },
|
||||
{ "package": "fiery", "reason": "hang" },
|
||||
{ "package": "ffscrapr", "reason": "hang" },
|
||||
{ "package": "cold", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.DoE", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.NMBU", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.RiskDemo", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.ROC", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.TeachStat", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.TeachingDemos", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.UCA", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.WorldFlora", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.aRnova", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.depthTools", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.orloca", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.sos", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.survival", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.temis", "reason": "hang" },
|
||||
{ "package": "GWlasso", "reason": "hang" },
|
||||
{ "package": "GWmodelVis", "reason": "hang" }
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,15 +1,21 @@
|
|||
library(httr2, quietly = TRUE)
|
||||
|
||||
forgejo_base <- "https://git.devxy.io/api/v1"
|
||||
repo <- "devxy/build-cran-binaries"
|
||||
platform <- Sys.getenv("PLATFORM")
|
||||
arch <- Sys.getenv("ARCH")
|
||||
token <- Sys.getenv("FORGEJO_TOKEN")
|
||||
output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt")
|
||||
repo <- "devxy/build-cran-binaries"
|
||||
platform <- Sys.getenv("PLATFORM")
|
||||
arch <- Sys.getenv("ARCH")
|
||||
token <- Sys.getenv("FORGEJO_TOKEN")
|
||||
output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt")
|
||||
|
||||
if (nchar(platform) == 0) stop("PLATFORM env var is not set")
|
||||
if (nchar(arch) == 0) stop("ARCH env var is not set")
|
||||
if (nchar(token) == 0) stop("FORGEJO_TOKEN env var is not set")
|
||||
if (nchar(platform) == 0) {
|
||||
stop("PLATFORM env var is not set")
|
||||
}
|
||||
if (nchar(arch) == 0) {
|
||||
stop("ARCH env var is not set")
|
||||
}
|
||||
if (nchar(token) == 0) {
|
||||
stop("FORGEJO_TOKEN env var is not set")
|
||||
}
|
||||
|
||||
os_family <- if (grepl("^ubuntu", platform)) {
|
||||
"Ubuntu"
|
||||
|
|
@ -21,12 +27,16 @@ os_family <- if (grepl("^ubuntu", platform)) {
|
|||
platform
|
||||
}
|
||||
|
||||
issue_title <- sprintf("Missing package binaries for latest version (%s)", os_family)
|
||||
issue_title <- sprintf(
|
||||
"Missing package binaries for latest version (%s)",
|
||||
os_family
|
||||
)
|
||||
cat(sprintf("Searching for issue: %s\n", issue_title))
|
||||
|
||||
search_url <- sprintf(
|
||||
"%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50",
|
||||
forgejo_base, repo,
|
||||
forgejo_base,
|
||||
repo,
|
||||
utils::URLencode(issue_title, reserved = TRUE)
|
||||
)
|
||||
search_resp <- request(search_url) |>
|
||||
|
|
@ -34,7 +44,9 @@ search_resp <- request(search_url) |>
|
|||
req_perform()
|
||||
|
||||
issues <- resp_body_json(search_resp, simplifyVector = FALSE)
|
||||
match_idx <- which(vapply(issues, function(x) x$title, character(1)) == issue_title)
|
||||
match_idx <- which(
|
||||
vapply(issues, function(x) x$title, character(1)) == issue_title
|
||||
)
|
||||
|
||||
if (length(match_idx) == 0) {
|
||||
cat("No matching issue found - nothing to rebuild\n")
|
||||
|
|
@ -55,7 +67,10 @@ lines <- strsplit(body, "\n", fixed = TRUE)[[1]]
|
|||
plat_header <- sprintf("## %s", platform)
|
||||
plat_idx <- which(lines == plat_header)
|
||||
if (length(plat_idx) == 0) {
|
||||
cat(sprintf("No section found for platform %s - nothing to rebuild\n", platform))
|
||||
cat(sprintf(
|
||||
"No section found for platform %s - nothing to rebuild\n",
|
||||
platform
|
||||
))
|
||||
writeLines(character(0), output_file)
|
||||
q("no")
|
||||
}
|
||||
|
|
@ -64,7 +79,11 @@ if (length(plat_idx) == 0) {
|
|||
arch_pattern <- sprintf("^### %s ", arch)
|
||||
arch_idx <- which(grepl(arch_pattern, lines) & seq_along(lines) > plat_idx[1])
|
||||
if (length(arch_idx) == 0) {
|
||||
cat(sprintf("No section found for arch %s under %s - nothing to rebuild\n", arch, platform))
|
||||
cat(sprintf(
|
||||
"No section found for arch %s under %s - nothing to rebuild\n",
|
||||
arch,
|
||||
platform
|
||||
))
|
||||
writeLines(character(0), output_file)
|
||||
q("no")
|
||||
}
|
||||
|
|
@ -82,6 +101,11 @@ pkg_lines <- section_lines[grepl("^- ", section_lines)]
|
|||
pkgs <- sub("^- ([^ ]+) \\(.*\\)$", "\\1", pkg_lines)
|
||||
pkgs <- pkgs[nchar(pkgs) > 0 & pkgs != "_None_"]
|
||||
|
||||
cat(sprintf("Found %d rebuildable packages for %s/%s\n", length(pkgs), platform, arch))
|
||||
cat(sprintf(
|
||||
"Found %d rebuildable packages for %s/%s\n",
|
||||
length(pkgs),
|
||||
platform,
|
||||
arch
|
||||
))
|
||||
writeLines(pkgs, output_file)
|
||||
cat(sprintf("Wrote package list to %s\n", output_file))
|
||||
|
|
|
|||
34
local/find-R-api-packages.sh
Normal file
34
local/find-R-api-packages.sh
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Directory to clone repos into
|
||||
WORKDIR="cran_repos"
|
||||
mkdir -p "$WORKDIR"
|
||||
cd "$WORKDIR"
|
||||
|
||||
# GitHub API paginates results, so we loop through pages
|
||||
PAGE=1
|
||||
PER_PAGE=100
|
||||
MATCHES=()
|
||||
|
||||
while :; do
|
||||
# Fetch a page of repos
|
||||
REPOS=$(curl -s "https://api.github.com/orgs/cran/repos?per_page=$PER_PAGE&page=$PAGE" | jq -r '.[].clone_url')
|
||||
[ -z "$REPOS" ] && break
|
||||
|
||||
for REPO_URL in $REPOS; do
|
||||
REPO_NAME=$(basename "$REPO_URL" .git)
|
||||
# Skip if already cloned
|
||||
[ -d "$REPO_NAME" ] && continue
|
||||
git clone --depth 1 "$REPO_URL" "$REPO_NAME" >/dev/null 2>&1
|
||||
if [ -d "$REPO_NAME/src" ]; then
|
||||
# Search for Rinternals.h in src/
|
||||
if grep -r -q 'R_VERSION < R_Version(' "$REPO_NAME/src"; then
|
||||
echo "$REPO_NAME"
|
||||
fi
|
||||
fi
|
||||
# Clean up to save space
|
||||
rm -rf "$REPO_NAME"
|
||||
done
|
||||
|
||||
PAGE=$((PAGE + 1))
|
||||
done
|
||||
|
|
@ -6,9 +6,13 @@ archived <- quickcode::archivedPkg() |>
|
|||
pull(name)
|
||||
|
||||
# Query all distinct pkgs in the DB
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
con <- DBI::dbConnect(
|
||||
RPostgres::Postgres(),
|
||||
dbname = "build_metadata",
|
||||
host = "r-binaries.devxy.io",
|
||||
port = 15432,
|
||||
user = "r_binaries",
|
||||
password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
|
|
@ -22,7 +26,9 @@ to_process <- pkgs_db[pkgs_db %in% archived]
|
|||
|
||||
# for all matches, set 'removed = TRUE'
|
||||
sapply(to_process, function(.x) {
|
||||
DBI::dbExecute(con, "UPDATE single_builds SET removed = 'TRUE' where name = $1",
|
||||
DBI::dbExecute(
|
||||
con,
|
||||
"UPDATE single_builds SET removed = 'TRUE' where name = $1",
|
||||
params = list(.x)
|
||||
)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -7,4 +7,4 @@ query_metadata_table() |>
|
|||
group_by(platform, arch) |>
|
||||
arrange(desc(timestamp)) |>
|
||||
select(name, timestamp) |>
|
||||
filter(row_number()==1)
|
||||
filter(row_number() == 1)
|
||||
|
|
|
|||
90
local/migrate-s3-hetzner-to-backblaze.sh
Normal file
90
local/migrate-s3-hetzner-to-backblaze.sh
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Migrate S3 buckets from Hetzner Object Storage to Backblaze B2 via rclone.
|
||||
#
|
||||
# Prerequisites:
|
||||
# 1. Install rclone: https://rclone.org/install/
|
||||
# 2. Configure two rclone remotes:
|
||||
# rclone config create hetzner s3 \
|
||||
# provider=Other \
|
||||
# env_auth=false \
|
||||
# access_key_id=YOUR_HETZNER_KEY \
|
||||
# secret_access_key=YOUR_HETZNER_SECRET \
|
||||
# endpoint=fsn1.your-objectstorage.com # adjust region
|
||||
#
|
||||
# rclone config create backblaze s3 \
|
||||
# provider=Other \
|
||||
# env_auth=false \
|
||||
# access_key_id=YOUR_B2_KEY \
|
||||
# secret_access_key=YOUR_B2_APP_KEY \
|
||||
# endpoint=s3.us-west-004.backblazeb2.com # adjust region
|
||||
#
|
||||
# Usage:
|
||||
# ./migrate-s3-hetzner-to-backblaze.sh <src:dst> [src:dst] ...
|
||||
# ./migrate-s3-hetzner-to-backblaze.sh hetzner-bucket:backblaze-bucket
|
||||
|
||||
HETZNER_REMOTE="${HETZNER_REMOTE:-hetzner}"
|
||||
BACKBLAZE_REMOTE="${BACKBLAZE_REMOTE:-backblaze}"
|
||||
RCLONE_FLAGS="${RCLONE_FLAGS:---transfers=64 --checkers=64 --fast-list}"
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
echo "Usage: $0 <src-bucket:dst-bucket> [src-bucket:dst-bucket...]"
|
||||
echo ""
|
||||
echo " Each argument is a source:destination bucket pair separated by a colon."
|
||||
echo ""
|
||||
echo "Environment variables:"
|
||||
echo " HETZNER_REMOTE rclone remote name for Hetzner (default: hetzner)"
|
||||
echo " BACKBLAZE_REMOTE rclone remote name for Backblaze (default: backblaze)"
|
||||
echo " RCLONE_FLAGS extra rclone flags (default: --transfers=16 --checkers=16 --fast-list)"
|
||||
echo " DRY_RUN=1 show what would be copied without copying"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for cmd in rclone; do
|
||||
if ! command -v "$cmd" &>/dev/null; then
|
||||
echo "Error: $cmd is not installed." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Verify remotes exist
|
||||
for remote in "$HETZNER_REMOTE" "$BACKBLAZE_REMOTE"; do
|
||||
if ! rclone listremotes | grep -q "^${remote}:$"; then
|
||||
echo "Error: rclone remote '${remote}' not found. Run 'rclone config' to set it up." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
DRY_RUN_FLAG=""
|
||||
if [[ "${DRY_RUN:-0}" == "1" ]]; then
|
||||
DRY_RUN_FLAG="--dry-run"
|
||||
echo "=== DRY RUN MODE ==="
|
||||
fi
|
||||
|
||||
for pair in "$@"; do
|
||||
if [[ "$pair" != *:* ]]; then
|
||||
echo "Error: '$pair' is not a valid src:dst pair. Use format 'hetzner-bucket:backblaze-bucket'." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
src_bucket="${pair%%:*}"
|
||||
dst_bucket="${pair#*:}"
|
||||
src="${HETZNER_REMOTE}:${src_bucket}"
|
||||
dst="${BACKBLAZE_REMOTE}:${dst_bucket}"
|
||||
|
||||
echo ""
|
||||
echo "--- Migrating: ${src} -> ${dst} ---"
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
rclone sync \
|
||||
${RCLONE_FLAGS} \
|
||||
${DRY_RUN_FLAG} \
|
||||
--progress \
|
||||
"$src" "$dst"
|
||||
|
||||
echo "--- Done: ${src_bucket} -> ${dst_bucket} ---"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Migration complete."
|
||||
|
|
@ -3,22 +3,34 @@ library(bincraft)
|
|||
library(dplyr)
|
||||
library(DBI)
|
||||
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
con <- DBI::dbConnect(
|
||||
RPostgres::Postgres(),
|
||||
dbname = "build_metadata",
|
||||
host = "r-binaries.devxy.io",
|
||||
port = 15432,
|
||||
user = "r_binaries",
|
||||
password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
`%nin%` <- Negate(`%in%`)
|
||||
|
||||
new_packages <- get_new_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
|
||||
removed_pkgs <- get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
|
||||
new_packages <- get_new_cran_packages(lubridate::interval(
|
||||
lubridate::today(),
|
||||
lubridate::today() - 2
|
||||
))$name
|
||||
removed_pkgs <- get_removed_cran_packages(lubridate::interval(
|
||||
lubridate::today(),
|
||||
lubridate::today() - 2
|
||||
))$name
|
||||
# filter windows packages
|
||||
# filter new packages from the last X days
|
||||
cran_pkgs <- unique(tools::CRAN_package_db() |>
|
||||
filter(`OS_type` != "windows" | is.na(`OS_type`)) |>
|
||||
filter(`Package` %nin% new_packages) |>
|
||||
pull(Package))
|
||||
cran_pkgs <- unique(
|
||||
tools::CRAN_package_db() |>
|
||||
filter(`OS_type` != "windows" | is.na(`OS_type`)) |>
|
||||
filter(`Package` %nin% new_packages) |>
|
||||
pull(Package)
|
||||
)
|
||||
|
||||
arch <- "arm64"
|
||||
platform <- "redhat-8"
|
||||
|
|
@ -28,13 +40,24 @@ platform <- "alpine-321"
|
|||
platform <- "ubuntu-2204"
|
||||
platform <- "ubuntu-2404"
|
||||
|
||||
data <- dbGetQuery(con, "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", params = list(platform, arch))
|
||||
removed_pkgs = get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
|
||||
data <- dbGetQuery(
|
||||
con,
|
||||
"SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;",
|
||||
params = list(platform, arch)
|
||||
)
|
||||
removed_pkgs = get_removed_cran_packages(lubridate::interval(
|
||||
lubridate::today(),
|
||||
lubridate::today() - 2
|
||||
))$name
|
||||
pkgs_db <- unique(data$name)
|
||||
pkgs_db = setdiff(pkgs_db, removed_pkgs)
|
||||
|
||||
pkgs <- setdiff(cran_pkgs, pkgs_db)
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)])))))
|
||||
formatted_pkgs <- gsub(
|
||||
'"',
|
||||
"'",
|
||||
capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)]))))
|
||||
)
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
|
|
@ -44,7 +67,6 @@ clipr::write_clip(formatted_pkgs_one_line)
|
|||
|
||||
# sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs)))
|
||||
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
|
||||
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
|
||||
|
|
|
|||
|
|
@ -10,19 +10,39 @@ s3fs::s3_file_system(
|
|||
)
|
||||
s3fs::s3_dir_ls("s3://devxy-r-package-binaries")
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"
|
||||
)) # done
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"
|
||||
))
|
||||
|
||||
files_b=basename(files)
|
||||
files_b = basename(files)
|
||||
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
||||
|
||||
cran_pkgs <- tools::CRAN_package_db() |>
|
||||
|
|
@ -30,7 +50,11 @@ cran_pkgs <- tools::CRAN_package_db() |>
|
|||
pull(Package)
|
||||
|
||||
pkgs = setdiff(cran_pkgs, files_b_split)
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900])))))
|
||||
formatted_pkgs <- gsub(
|
||||
'"',
|
||||
"'",
|
||||
capture.output(dput(as.character(na.omit(pkgs[1:900]))))
|
||||
)
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
|
|
|
|||
|
|
@ -7,42 +7,109 @@ s3fs::s3_file_system(
|
|||
region_name = "hel1",
|
||||
)
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"
|
||||
))
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"
|
||||
))
|
||||
|
||||
files_b=basename(files)
|
||||
files_b = basename(files)
|
||||
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
||||
files_b_split = setdiff(files_b_split, c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive"))
|
||||
files_b_split = setdiff(
|
||||
files_b_split,
|
||||
c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive")
|
||||
)
|
||||
# windows-only
|
||||
files_b_split = setdiff(files_b_split, c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive", "RInno", "KeyboardSimulator", "R2PPT", "RWinEdt", "blatr", "excel.link", "spectrino", "taskscheduleR", "MDSGUI", "BiplotGUI", "R2wd"))
|
||||
pkgs_index = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib")[, "Package"]
|
||||
pkgs_index_b=basename(pkgs_index)
|
||||
files_b_split = setdiff(
|
||||
files_b_split,
|
||||
c(
|
||||
"PACKAGES",
|
||||
"PACKAGES.gz",
|
||||
"PACKAGES.rds",
|
||||
"PACKAGES.db",
|
||||
"Archive",
|
||||
"RInno",
|
||||
"KeyboardSimulator",
|
||||
"R2PPT",
|
||||
"RWinEdt",
|
||||
"blatr",
|
||||
"excel.link",
|
||||
"spectrino",
|
||||
"taskscheduleR",
|
||||
"MDSGUI",
|
||||
"BiplotGUI",
|
||||
"R2wd"
|
||||
)
|
||||
)
|
||||
pkgs_index = available.packages(
|
||||
"https://cran.devxy.io/amd64/jammy/latest/src/contrib"
|
||||
)[, "Package"]
|
||||
pkgs_index_b = basename(pkgs_index)
|
||||
if (length(files_b_split) != length(pkgs_index_b)) {
|
||||
pkgs_missing = setdiff(files_b_split, pkgs_index_b)
|
||||
message("Packages missing in index but present in S3:")
|
||||
pkgs_missing
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs_missing[1:900])))))
|
||||
formatted_pkgs <- gsub(
|
||||
'"',
|
||||
"'",
|
||||
capture.output(dput(as.character(na.omit(pkgs_missing[1:900]))))
|
||||
)
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
}
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"
|
||||
))
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"
|
||||
))
|
||||
|
|
|
|||
|
|
@ -17,7 +17,11 @@ suppressPackageStartupMessages(library(data.table))
|
|||
|
||||
arch = Sys.getenv("ARCH")
|
||||
# target: alpine-322, ubuntu-2404, redhat-9, etc.
|
||||
platform = paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-")
|
||||
platform = paste(
|
||||
Sys.getenv("OS"),
|
||||
gsub("[.]", "", Sys.getenv("OS_VERSION")),
|
||||
sep = "-"
|
||||
)
|
||||
# Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10")
|
||||
codename = bincraft::set_codename(NULL)
|
||||
|
||||
|
|
@ -34,7 +38,9 @@ con <- DBI::dbConnect(
|
|||
cran_archive = tools::CRAN_archive_db()
|
||||
cran_release = tools::CRAN_package_db()
|
||||
# Subset cran_archive to only those packages
|
||||
cran_archive_in_release <- cran_archive[names(cran_archive) %in% cran_release$Package]
|
||||
cran_archive_in_release <- cran_archive[
|
||||
names(cran_archive) %in% cran_release$Package
|
||||
]
|
||||
|
||||
archive_versions <- rbindlist(
|
||||
lapply(names(cran_archive), function(pkg) {
|
||||
|
|
@ -126,11 +132,15 @@ query_error <- function(pkg, ver) {
|
|||
# Fetch all relevant columns from the database
|
||||
errored_pkgs <- DBI::dbGetQuery(
|
||||
con,
|
||||
sprintf("SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", platform, arch)
|
||||
sprintf(
|
||||
"SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'",
|
||||
platform,
|
||||
arch
|
||||
)
|
||||
)
|
||||
errored_pkgs <- as.data.table(errored_pkgs)
|
||||
setkey(pkgs_to_build, Package, Version)
|
||||
setnames(errored_pkgs, c("Package","Version"))
|
||||
setnames(errored_pkgs, c("Package", "Version"))
|
||||
setkey(errored_pkgs, Package, Version)
|
||||
|
||||
### Final subsetting
|
||||
|
|
@ -140,3 +150,62 @@ pkgs <- pkgs_no_error[!s3_dt]
|
|||
# Deduplicate
|
||||
pkgs <- unique(pkgs)
|
||||
setorder(pkgs, Package, Version)
|
||||
|
||||
### R-minor sensitivity (classify once per package, applied to all versions)
|
||||
source(file.path("local", "r-minor-helpers.R"))
|
||||
risky_deps <- bincraft::abi_risky_linking_deps()
|
||||
|
||||
release_meta <- data.table(
|
||||
Package = cran_release$Package,
|
||||
NeedsCompilation = cran_release$NeedsCompilation,
|
||||
LinkingTo = cran_release$LinkingTo
|
||||
)
|
||||
|
||||
meta <- release_meta[Package %in% unique(pkgs$Package)]
|
||||
meta[,
|
||||
triage := mapply(
|
||||
classify_from_metadata,
|
||||
NeedsCompilation,
|
||||
LinkingTo,
|
||||
MoreArgs = list(risky_deps = risky_deps)
|
||||
)
|
||||
]
|
||||
|
||||
# Only the "ambiguous" compiled packages need a source grep.
|
||||
ambiguous <- meta[triage == "ambiguous", Package]
|
||||
sensitive_ambiguous <- character()
|
||||
if (length(ambiguous) > 0L) {
|
||||
tmp_src <- file.path(tempdir(), "abi_src")
|
||||
dir.create(tmp_src, showWarnings = FALSE, recursive = TRUE)
|
||||
sens <- vapply(
|
||||
ambiguous,
|
||||
function(pkg) {
|
||||
out <- tryCatch(
|
||||
{
|
||||
dl <- utils::download.packages(
|
||||
pkg,
|
||||
destdir = tmp_src,
|
||||
repos = "https://cloud.r-project.org",
|
||||
quiet = TRUE
|
||||
)
|
||||
isTRUE(as.logical(bincraft::needs_per_minor_recompile(dl[1L, 2L])))
|
||||
},
|
||||
error = function(e) TRUE
|
||||
) # fail safe: treat as sensitive
|
||||
out
|
||||
},
|
||||
logical(1L)
|
||||
)
|
||||
sensitive_ambiguous <- ambiguous[sens]
|
||||
}
|
||||
|
||||
sensitive_pkgs <- unique(c(
|
||||
meta[triage == "sensitive", Package],
|
||||
sensitive_ambiguous
|
||||
))
|
||||
pkgs[, r_minor_sensitive := Package %in% sensitive_pkgs]
|
||||
sprintf(
|
||||
"R-minor-sensitive packages: %s of %s",
|
||||
length(sensitive_pkgs),
|
||||
uniqueN(pkgs$Package)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,24 @@
|
|||
### Lists packages without any successful binaries, i.e. pkgs for which all builds errored (per platform & arch)
|
||||
library(DBI)
|
||||
library(dplyr)
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
con <- DBI::dbConnect(
|
||||
RPostgres::Postgres(),
|
||||
dbname = "build_metadata",
|
||||
host = "r-binaries.devxy.io",
|
||||
port = 15432,
|
||||
user = "r_binaries",
|
||||
password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
cran_pkgs <- tools::CRAN_package_db() |>
|
||||
filter(`Date/Publication` <= "2024-12-02") |>
|
||||
cran_pkgs <- tools::CRAN_package_db() |>
|
||||
filter(`Date/Publication` <= "2024-12-02") |>
|
||||
pull(Package)
|
||||
|
||||
data <- dbGetQuery(con, "SELECT name,platform,arch,removed,error_occurred FROM single_builds;")
|
||||
data <- dbGetQuery(
|
||||
con,
|
||||
"SELECT name,platform,arch,removed,error_occurred FROM single_builds;"
|
||||
)
|
||||
|
||||
pkgs <- data |>
|
||||
filter(platform == "alpine-320", arch == "arm64") |>
|
||||
|
|
@ -22,7 +29,11 @@ pkgs <- data |>
|
|||
|
||||
pkgs = setdiff(cran_pkgs, pkgs)
|
||||
# Format, remove line breaks, and print as a single line
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900])))))
|
||||
formatted_pkgs <- gsub(
|
||||
'"',
|
||||
"'",
|
||||
capture.output(dput(as.character(na.omit(pkgs[1:900]))))
|
||||
)
|
||||
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
|
|
|
|||
39
local/query-pkgs-without-old-versions.R
Normal file
39
local/query-pkgs-without-old-versions.R
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
library(s3fs)
|
||||
|
||||
# List all files under contrib/<pkg>/
|
||||
all_files <- s3fs::s3_dir_ls(
|
||||
"s3://devxy-r-package-binaries-hel1/arm64/alpine322/latest/src/contrib/",
|
||||
recurse = TRUE,
|
||||
type = "file"
|
||||
)
|
||||
|
||||
pattern <- ".*/src/contrib/([^/_]+)_.*"
|
||||
matches <- regmatches(all_files, regexec(pattern, all_files))
|
||||
pkg_names <- unique(
|
||||
vapply(
|
||||
matches,
|
||||
function(x) if (length(x) > 1) x[2] else NA_character_,
|
||||
character(1)
|
||||
)
|
||||
)
|
||||
pkg_names <- pkg_names[!is.na(pkg_names)]
|
||||
|
||||
# For each package, check if Archive/<pkg>/ contains any files
|
||||
no_archive_files <- character(0)
|
||||
for (pkg in pkg_names) {
|
||||
archive_dir1 <- sprintf(
|
||||
"s3://devxy-r-package-binaries-hel1/arm64/alpine322/latest/src/contrib/Archive/%s",
|
||||
pkg
|
||||
)
|
||||
archive_files <- unique(c(
|
||||
tryCatch(
|
||||
s3fs::s3_dir_ls(archive_dir1, recurse = TRUE),
|
||||
error = function(e) character(0)
|
||||
)
|
||||
))
|
||||
if (length(archive_files) == 0) {
|
||||
no_archive_files <- c(no_archive_files, pkg)
|
||||
}
|
||||
}
|
||||
|
||||
print(no_archive_files)
|
||||
33
local/r-minor-helpers.R
Normal file
33
local/r-minor-helpers.R
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# Metadata-only ABI triage so the full build avoids downloading every source.
|
||||
# Mirrors bincraft::abi_classify rules 1-2; "ambiguous" packages still need a
|
||||
# source grep via bincraft::needs_per_minor_recompile().
|
||||
|
||||
classify_from_metadata <- function(needs_compilation, linking_to, risky_deps) {
|
||||
nc <- if (length(needs_compilation) == 0L || is.na(needs_compilation)) {
|
||||
""
|
||||
} else {
|
||||
tolower(trimws(needs_compilation))
|
||||
}
|
||||
if (!identical(nc, "yes")) {
|
||||
return("not-sensitive")
|
||||
}
|
||||
lt <- if (length(linking_to) == 0L || is.na(linking_to)) "" else linking_to
|
||||
linked <- trimws(unlist(strsplit(lt, "[,\n]")))
|
||||
linked <- sub("\\s*\\(.*\\)$", "", linked) # strip "(>= x)" constraints
|
||||
linked <- linked[nzchar(linked)]
|
||||
if (any(linked %in% risky_deps)) {
|
||||
return("sensitive")
|
||||
}
|
||||
"ambiguous"
|
||||
}
|
||||
|
||||
parse_build_args <- function(args) {
|
||||
sensitive_only <- "--sensitive-only" %in% args
|
||||
pos <- args[!startsWith(args, "--")]
|
||||
list(
|
||||
sensitive_only = sensitive_only,
|
||||
split_into = as.integer(pos[1L]),
|
||||
split_index = as.integer(pos[2L]),
|
||||
ncpus = as.integer(pos[3L])
|
||||
)
|
||||
}
|
||||
331
local/test-package-loading.R
Normal file
331
local/test-package-loading.R
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
install.packages(
|
||||
"pak",
|
||||
repos = sprintf(
|
||||
"https://r-lib.github.io/p/pak/stable/%s/%s/%s",
|
||||
.Platform$pkgType,
|
||||
R.Version()$os,
|
||||
R.Version()$arch
|
||||
)
|
||||
)
|
||||
|
||||
Sys.setenv(PKG_SYSREQS = TRUE)
|
||||
all_pkgs <- rownames(available.packages())
|
||||
|
||||
to_skip = c("ABRSQOL", "ACA", "ACE.CoCo")
|
||||
all_pkgs = setdiff(all_pkgs, to_skip)
|
||||
|
||||
for (i in all_pkgs) {
|
||||
message(sprintf("\nInstalling %s", i))
|
||||
pak::pkg_install(i)
|
||||
library(i, character.only = TRUE)
|
||||
}
|
||||
|
||||
|
||||
# Example data
|
||||
all_pkgs <- rownames(available.packages())
|
||||
to_skip <- c(
|
||||
"ABRSQOL",
|
||||
"ACA",
|
||||
"ACE.CoCo",
|
||||
"ACEsimFit",
|
||||
"ACNE",
|
||||
"absorber",
|
||||
"adapt4pv",
|
||||
"adaptMCMC",
|
||||
"addhaz",
|
||||
"ADDT",
|
||||
"ahaz",
|
||||
"arm",
|
||||
"arules",
|
||||
"arulesCBA",
|
||||
"aster2",
|
||||
"BayesFactor",
|
||||
"bc3net",
|
||||
"bgsmtr",
|
||||
"biglasso",
|
||||
"BinNonNor",
|
||||
"BinNor",
|
||||
"bioassayR",
|
||||
"birankr",
|
||||
"BiRewire",
|
||||
"bolasso",
|
||||
"Boptbd",
|
||||
"Brobdingnag",
|
||||
"BSW",
|
||||
"BTLLasso",
|
||||
"bvartools",
|
||||
"cAIC4",
|
||||
"Category",
|
||||
"celda",
|
||||
"centiserve",
|
||||
"cjoint",
|
||||
"clinical",
|
||||
"clipper",
|
||||
"CodataGS",
|
||||
"conos",
|
||||
"CopulaInference",
|
||||
"covEB",
|
||||
"cplm",
|
||||
"CRTgeeDR",
|
||||
"cthreshER",
|
||||
"ctmcmove",
|
||||
"curephEM",
|
||||
"CVST",
|
||||
"dcGSA",
|
||||
"dclone",
|
||||
"dcsvm",
|
||||
"DelayedArray",
|
||||
"dglars",
|
||||
"dhglm",
|
||||
"disordR",
|
||||
"distrom",
|
||||
"dmm",
|
||||
"DNABarcodes",
|
||||
"DoubleCone",
|
||||
"DRR",
|
||||
"DTRlearn2",
|
||||
"DWDLargeR",
|
||||
"eds",
|
||||
"EMCluster",
|
||||
"EMMREML",
|
||||
"evalITR",
|
||||
"EventPointer",
|
||||
"evola",
|
||||
"excursions",
|
||||
"expm",
|
||||
"fanc",
|
||||
"FAS",
|
||||
"fastadi",
|
||||
"fastPLS",
|
||||
"fastRG",
|
||||
"fdaPDE",
|
||||
"flare",
|
||||
"FoReco",
|
||||
"frailtyHL",
|
||||
"freebird",
|
||||
"FSTpackage",
|
||||
"gamlr",
|
||||
"gamlss.lasso",
|
||||
"gamm4",
|
||||
"gbmt",
|
||||
"gdim",
|
||||
"gdistance",
|
||||
"GeDS",
|
||||
"geeM",
|
||||
"genlasso",
|
||||
"GenOrd",
|
||||
"GenoScan",
|
||||
"geomorph",
|
||||
"geostatsp",
|
||||
"GhostKnockoff",
|
||||
"GIGSEA",
|
||||
"GlarmaVarSel",
|
||||
"glmm",
|
||||
"glmmrBase",
|
||||
"glmmrOptim",
|
||||
"glmnet",
|
||||
"glober",
|
||||
"GPvam",
|
||||
"graphpcor",
|
||||
"gremlin",
|
||||
"growthrate",
|
||||
"grpCox",
|
||||
"GSD",
|
||||
"HelpersMG",
|
||||
"hglm",
|
||||
"hglm.data",
|
||||
"hibayes",
|
||||
"hierSDR",
|
||||
"HMTL",
|
||||
"hsem",
|
||||
"ibmdbR",
|
||||
"inca",
|
||||
"INLAspacetime",
|
||||
"INLAtools",
|
||||
"invertiforms",
|
||||
"irlba",
|
||||
"islasso",
|
||||
"ISLET",
|
||||
"isotonic.pen",
|
||||
"jordan",
|
||||
"kinship2",
|
||||
"KnockoffScreen",
|
||||
"lcpm",
|
||||
"leidenAlg",
|
||||
"lfe",
|
||||
"lingmatch",
|
||||
"LKT",
|
||||
"lme4",
|
||||
"lme4breeding",
|
||||
"lme4GS",
|
||||
"logcondiscr",
|
||||
"LPmerge",
|
||||
"LRMF3",
|
||||
"MAP",
|
||||
"marcox",
|
||||
"markovchain",
|
||||
"MatrixExtra",
|
||||
"matter",
|
||||
"MBC",
|
||||
"mcen",
|
||||
"mclogit",
|
||||
"MCMCglmm",
|
||||
"mdhglm",
|
||||
"MDPtoolbox",
|
||||
"mediation",
|
||||
"mefa4",
|
||||
"metafor",
|
||||
"mgwrsar",
|
||||
"mi",
|
||||
"midasml",
|
||||
"mind",
|
||||
"monocle",
|
||||
"msda",
|
||||
"MuData",
|
||||
"MultiGlarmaVarSel",
|
||||
"MultiOrd",
|
||||
"MultiVarSel",
|
||||
"mvglmmRank",
|
||||
"N2R",
|
||||
"nadiv",
|
||||
"NBtsVarSel",
|
||||
"NegBinBetaBinreg",
|
||||
"NetworkRiskMeasures",
|
||||
"neuroim2",
|
||||
"NOISeq",
|
||||
"numbat",
|
||||
"optbdmaeAT",
|
||||
"optimbase",
|
||||
"OptimModel",
|
||||
"optrcdmaeAT",
|
||||
"OrdNor",
|
||||
"pagoda2",
|
||||
"PCovR",
|
||||
"pedgene",
|
||||
"pedigree",
|
||||
"pedigreemm",
|
||||
"pense",
|
||||
"PERMANOVA",
|
||||
"phateR",
|
||||
"PhylogeneticEM",
|
||||
"pleio",
|
||||
"POINT",
|
||||
"PoisBinNonNor",
|
||||
"PoisBinOrd",
|
||||
"PoisBinOrdNonNor",
|
||||
"PoisBinOrdNor",
|
||||
"PoisNonNor",
|
||||
"PoisNor",
|
||||
"PRISMA",
|
||||
"ProbitSpatial",
|
||||
"prodest",
|
||||
"psqn",
|
||||
"qlcMatrix",
|
||||
"qpcR",
|
||||
"QRM",
|
||||
"quadrupen",
|
||||
"QZ",
|
||||
"ramps",
|
||||
"randnet",
|
||||
"randPedPCA",
|
||||
"rBMF",
|
||||
"RCBR",
|
||||
"RealVAMS",
|
||||
"REBayes",
|
||||
"recommenderlab",
|
||||
"Rediscover",
|
||||
"reglogit",
|
||||
"RESET",
|
||||
"RGE",
|
||||
"RGENERATEPREC",
|
||||
"riemtan",
|
||||
"RNewsflow",
|
||||
"robustlmm",
|
||||
"rsparse",
|
||||
"rSPDE",
|
||||
"rwc",
|
||||
"S4Arrays",
|
||||
"saeMSPE",
|
||||
"sbw",
|
||||
"scITD",
|
||||
"scoup",
|
||||
"sdwd",
|
||||
"SEAGLE",
|
||||
"sensory",
|
||||
"serrsBayes",
|
||||
"sglasso",
|
||||
"sharpPen",
|
||||
"SiPSiC",
|
||||
"SKAT",
|
||||
"snpReady",
|
||||
"snpStats",
|
||||
"softImpute",
|
||||
"sommer",
|
||||
"soptdmaeA",
|
||||
"SOR",
|
||||
"SparseArray",
|
||||
"SparseChol",
|
||||
"sparseLRMatrix",
|
||||
"sparsenet",
|
||||
"sparsenetgls",
|
||||
"sparsestep",
|
||||
"spatialprobit",
|
||||
"spatialreg",
|
||||
"spatstat.sparse",
|
||||
"speedglm",
|
||||
"sRDA",
|
||||
"sSDR",
|
||||
"ssfa",
|
||||
"stcos",
|
||||
"StratifiedSampling",
|
||||
"sureLDA",
|
||||
"survey",
|
||||
"surveyvoi",
|
||||
"svydiags",
|
||||
"systemfit",
|
||||
"TargetScore",
|
||||
"text2map",
|
||||
"textir",
|
||||
"textmineR",
|
||||
"textTinyR",
|
||||
"tmvtnorm",
|
||||
"TPEA",
|
||||
"triversity",
|
||||
"tsapp",
|
||||
"tvReg",
|
||||
"uwot",
|
||||
"vagam",
|
||||
"VAM",
|
||||
"WaveSampling",
|
||||
"WGScan",
|
||||
"wordspace",
|
||||
"workflowsets",
|
||||
"ACSSpack",
|
||||
"ADDT",
|
||||
"AER"
|
||||
)
|
||||
|
||||
# Find the position of the last package in to_skip within all_pkgs
|
||||
last_skip <- tail(to_skip, 1)
|
||||
|
||||
# Find its position in all_pkgs (returns NA if not found)
|
||||
start_pos <- match(last_skip, all_pkgs)
|
||||
|
||||
# If not found, start from the beginning; else, start after last_skip
|
||||
if (is.na(start_pos)) {
|
||||
to_process <- all_pkgs
|
||||
} else {
|
||||
to_process <- all_pkgs[(start_pos + 1):length(all_pkgs)]
|
||||
}
|
||||
|
||||
if (length(to_process) == 0) {
|
||||
message("All packages have been processed.")
|
||||
} else {
|
||||
for (i in to_process) {
|
||||
message(sprintf("\nInstalling %s", i))
|
||||
pak::pkg_install(i)
|
||||
library(i, character.only = TRUE)
|
||||
}
|
||||
# Update to_skip to include all up to the last processed
|
||||
to_skip <- all_pkgs[1:(last_skip_pos + length(to_process))]
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue