feat: R-minor-sensitive binary builds (full + iterative) (#84)
## Summary
Builds R-minor-sensitive CRAN packages once per installed R minor version (into per-minor S3 slots `…/contrib/<x.y>/`) and everything else once into the generic slot, driven by bincraft 4.2.0's ABI classifier. Both the full and iterative pipelines are covered.
## What's in here
**Detection / precompute**
- `local/r-minor-helpers.R` — pure `classify_from_metadata()` (NeedsCompilation / risky `LinkingTo`) + `parse_build_args()`, with unit tests.
- `local/packages-to-build.R` — adds a per-package `r_minor_sensitive` flag: cheap CRAN-metadata rules first, source download + `bincraft::needs_per_minor_recompile()` only for the ambiguous compiled subset (fail-safe to sensitive). Classified once per package, applied to all versions.
**Full build**
- `local/build-all.R` — passes the per-row `is_r_minor_sensitive` flag; new `--sensitive-only` mode builds just the risky subset.
- `.crow/build-all-versions-{amd64,arm64}.yaml` — install-deps persists the sensitive subset; build step runs a sensitive-only pass under each non-primary `/opt/R/*` minor; new step uploads the generic index plus a per-minor index for each minor.
**Iterative build**
- All 14 `.crow/process-updates-*.yaml` — primary pass uses `r_minor_detection = 'classifier'`; a sensitive-only multi-R pass builds risky updates under each other minor; per-minor index upload added.
**Tooling / housekeeping**
- Pins bincraft `v4.1.1` → `v4.2.0` across all workflows; removes the superseded standalone `build-r-minor-sensitive-packages.yaml`.
- Adds prek/pre-commit hooks (prettier, markdownlint, editorconfig-checker, yamllint, air) and applies them repo-wide; excludes the verbatim GPL `LICENSE.md` and auxiliary shell scripts.
- Design + implementation docs under `docs/superpowers/`.
## Requires before merge
- A `v4.2.0` git tag must be pushed on the bincraft repo (codefloe.com/rpkgs/bincraft) — the workflow install steps pin `@v4.2.0`. The full-build install-deps clones `main`, so it is unaffected.
Reviewed-on: #84
This commit is contained in:
Patrick Schratz
parent
f28ccaf008
commit
d558e27c11
18 changed files with 1171 additions and 413 deletions
|
|
@ -11,16 +11,34 @@ s3fs::s3_file_system(
|
|||
arch = "arm64"
|
||||
os = "alpine321"
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", arch, os))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib",
|
||||
arch,
|
||||
os
|
||||
))
|
||||
length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1])))
|
||||
non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) x[1])[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))])
|
||||
non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) {
|
||||
x[1]
|
||||
})[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))])
|
||||
# RInno is not built because it only exists for Windows
|
||||
non_archived = setdiff(non_archived, "RInno")
|
||||
# future::plan("sequential", workers = 8)
|
||||
future::plan("sequential", workers = 1)
|
||||
# future.apply::future_lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch))
|
||||
lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch, s3_region = "hel1", s3_endpoint = "https://hel1.your-objectstorage.com", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S", s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))))
|
||||
lapply(non_archived, function(x) {
|
||||
archive_package(
|
||||
x,
|
||||
codename = os,
|
||||
arch = arch,
|
||||
s3_region = "hel1",
|
||||
s3_endpoint = "https://hel1.your-objectstorage.com",
|
||||
s3_bucket = "devxy-r-package-binaries-hel1",
|
||||
s3_access_key_id = Sys.getenv(
|
||||
"HETZNER_S3_ACCESS_KEY_K3S",
|
||||
s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S")
|
||||
)
|
||||
)
|
||||
})
|
||||
# archive_package()
|
||||
|
||||
|
||||
# grep("duckdb_", files, value = T)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
sink(stdout(), type = "message")
|
||||
options(crayon.enabled = TRUE, future.globals.onReference = NULL)
|
||||
source(file.path("local", "r-minor-helpers.R"))
|
||||
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
split_into <- as.integer(args[1])
|
||||
split_index <- as.integer(args[2])
|
||||
ncpus <- as.integer(args[3])
|
||||
parsed <- parse_build_args(args)
|
||||
split_into <- parsed$split_into
|
||||
split_index <- parsed$split_index
|
||||
ncpus <- parsed$ncpus
|
||||
sensitive_only <- parsed$sensitive_only
|
||||
options(Ncpus = ncpus)
|
||||
|
||||
# Load bincraft eagerly to avoid lazy-load memory spike during first build call
|
||||
|
|
@ -12,9 +15,20 @@ library(bincraft, quietly = TRUE)
|
|||
library(future)
|
||||
plan("sequential")
|
||||
|
||||
# Read precomputed package+version pairs
|
||||
pkgs <- readRDS("/mnt/cache/packages/pkgs_to_build.rds")
|
||||
sprintf("Total# of remaining package versions: %s", nrow(pkgs))
|
||||
pkgs <- if (sensitive_only) {
|
||||
readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
||||
} else {
|
||||
readRDS("/mnt/cache/packages/pkgs_to_build.rds")
|
||||
}
|
||||
# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive)
|
||||
if (is.null(pkgs$r_minor_sensitive)) {
|
||||
pkgs$r_minor_sensitive <- FALSE
|
||||
}
|
||||
sprintf(
|
||||
"Total# of remaining package versions: %s (sensitive_only=%s)",
|
||||
nrow(pkgs),
|
||||
sensitive_only
|
||||
)
|
||||
|
||||
# Split into chunks for this worker
|
||||
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
|
||||
|
|
@ -32,26 +46,33 @@ s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
|||
sprintf("S3 cache: %s files", length(s3_cache))
|
||||
|
||||
n <- nrow(chunk)
|
||||
mapply(function(pkg, ver, i) {
|
||||
cat(sprintf("[%d/%d] %s_%s\n", i, n, pkg, ver))
|
||||
bincraft::build_binary_package(
|
||||
pkg,
|
||||
tag = ver,
|
||||
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
s3_region = "eu-central-003",
|
||||
s3_bucket = "devxy-rpkgs-binaries",
|
||||
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
s3_package_cache = s3_cache,
|
||||
metadata_db_host = "r-binaries.devxy.io",
|
||||
metadata_db_name = "build_metadata",
|
||||
metadata_db_table = "single_builds",
|
||||
metadata_db_user = "rpkgs",
|
||||
metadata_db_password = Sys.getenv("PGPASS"),
|
||||
metadata_db_sslmode = "require",
|
||||
metadata_db_port = 15432,
|
||||
archive = TRUE,
|
||||
upload = TRUE,
|
||||
store_build_metadata = TRUE
|
||||
)
|
||||
}, chunk$Package, chunk$Version, seq_len(n))
|
||||
mapply(
|
||||
function(pkg, ver, sens, i) {
|
||||
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
||||
bincraft::build_binary_package(
|
||||
pkg,
|
||||
tag = ver,
|
||||
is_r_minor_sensitive = isTRUE(sens),
|
||||
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
s3_region = "eu-central-003",
|
||||
s3_bucket = "devxy-rpkgs-binaries",
|
||||
s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
s3_package_cache = s3_cache,
|
||||
metadata_db_host = "r-binaries.devxy.io",
|
||||
metadata_db_name = "build_metadata",
|
||||
metadata_db_table = "single_builds",
|
||||
metadata_db_user = "rpkgs",
|
||||
metadata_db_password = Sys.getenv("PGPASS"),
|
||||
metadata_db_sslmode = "require",
|
||||
metadata_db_port = 15432,
|
||||
archive = TRUE,
|
||||
upload = TRUE,
|
||||
store_build_metadata = TRUE
|
||||
)
|
||||
},
|
||||
chunk$Package,
|
||||
chunk$Version,
|
||||
chunk$r_minor_sensitive,
|
||||
seq_len(n)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
### Scope: Check whether there are any incomplete entries in PACKAGES.rds. These will result in NA when calling available.packages()
|
||||
foo = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
|
|
@ -17,7 +16,9 @@ foo = available.packages("https://cran.devxy.io/amd64/rhel9/latest/src/contrib")
|
|||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/amd64/alpine320/latest/src/contrib")
|
||||
foo = available.packages(
|
||||
"https://cran.devxy.io/amd64/alpine320/latest/src/contrib"
|
||||
)
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
|
|
@ -40,6 +41,8 @@ foo = available.packages("https://cran.devxy.io/arm64/rhel9/latest/src/contrib")
|
|||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/arm64/alpine320/latest/src/contrib")
|
||||
foo = available.packages(
|
||||
"https://cran.devxy.io/arm64/alpine320/latest/src/contrib"
|
||||
)
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
|
|
|||
|
|
@ -2,42 +2,58 @@ library(future)
|
|||
future::plan(multisession)
|
||||
future::plan(sequential)
|
||||
time = Sys.time()
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive"), recurse = T)
|
||||
files <- s3fs::s3_dir_ls(
|
||||
sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive"
|
||||
),
|
||||
recurse = T
|
||||
)
|
||||
Sys.time() - time
|
||||
|
||||
|
||||
|
||||
archive <- llply(dirs, function(dir) {
|
||||
files <- list.files(dir, recursive = FALSE, full.names = TRUE, pattern = "*.tar.gz")
|
||||
files <- list.files(
|
||||
dir,
|
||||
recursive = FALSE,
|
||||
full.names = TRUE,
|
||||
pattern = "*.tar.gz"
|
||||
)
|
||||
if (length(files) == 0) {
|
||||
print(paste0("Error: Empty directory: ", dir))
|
||||
return(NULL)
|
||||
}
|
||||
info <- file.info(files)
|
||||
|
||||
tryCatch({
|
||||
rownames(info) <- paste0(basename(dirname(files)), "/", basename(files))
|
||||
}, error = function(e) {
|
||||
print(paste0("Error: Exception catched for Archived directory: ", dir))
|
||||
print(e)
|
||||
return(NULL)
|
||||
})
|
||||
tryCatch(
|
||||
{
|
||||
rownames(info) <- paste0(basename(dirname(files)), "/", basename(files))
|
||||
},
|
||||
error = function(e) {
|
||||
print(paste0("Error: Exception catched for Archived directory: ", dir))
|
||||
print(e)
|
||||
return(NULL)
|
||||
}
|
||||
)
|
||||
|
||||
info
|
||||
})
|
||||
|
||||
tryCatch({
|
||||
rownames(info) <- paste0(basename(dirname(files)), "/", basename(files))
|
||||
}, error = function(e) {
|
||||
print(paste0("Error: Exception catched for Archived directory: ", dir))
|
||||
print(e)
|
||||
return(NULL)
|
||||
})
|
||||
tryCatch(
|
||||
{
|
||||
rownames(info) <- paste0(basename(dirname(files)), "/", basename(files))
|
||||
},
|
||||
error = function(e) {
|
||||
print(paste0("Error: Exception catched for Archived directory: ", dir))
|
||||
print(e)
|
||||
return(NULL)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
curl::curl_download("https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db")
|
||||
curl::curl_download(
|
||||
"https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db",
|
||||
"PACKAGES.db"
|
||||
)
|
||||
con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db")
|
||||
df = DBI::dbReadTable(con, "packages")
|
||||
|
||||
|
|
@ -50,6 +66,14 @@ which(grepl("digest", df$Package))
|
|||
system("cat /tmp/PACKAGES | grep '^Package: digest' | wc -l")
|
||||
|
||||
|
||||
curl::curl_fetch_memory("https://cloud.r-project.org/src/contrib/Meta/archive.rds")
|
||||
remotes = readRDS(url("https://cloud.r-project.org/src/contrib/Meta/archive.rds", "rb"))
|
||||
packages = readRDS(url("https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds", "rb"))
|
||||
curl::curl_fetch_memory(
|
||||
"https://cloud.r-project.org/src/contrib/Meta/archive.rds"
|
||||
)
|
||||
remotes = readRDS(url(
|
||||
"https://cloud.r-project.org/src/contrib/Meta/archive.rds",
|
||||
"rb"
|
||||
))
|
||||
packages = readRDS(url(
|
||||
"https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds",
|
||||
"rb"
|
||||
))
|
||||
|
|
|
|||
|
|
@ -37,10 +37,13 @@ future.apply::future_lapply(
|
|||
files <- s3fs::s3_dir_ls(
|
||||
sprintf(
|
||||
"devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib",
|
||||
arch, codename
|
||||
arch,
|
||||
codename
|
||||
)
|
||||
)
|
||||
if (length(files) == 0) return(NULL)
|
||||
if (length(files) == 0) {
|
||||
return(NULL)
|
||||
}
|
||||
files_b <- basename(files)
|
||||
pkg_names <- sapply(strsplit(files_b, "_"), `[`, 1)
|
||||
dupes <- unique(pkg_names[duplicated(pkg_names)])
|
||||
|
|
|
|||
|
|
@ -1,77 +1,77 @@
|
|||
[
|
||||
{"package": "RInno", "reason": "windows-only"},
|
||||
{"package": "KeyboardSimulator", "reason": "windows-only"},
|
||||
{"package": "R2PPT", "reason": "windows-only"},
|
||||
{"package": "RWinEdt", "reason": "windows-only"},
|
||||
{"package": "blatr", "reason": "windows-only"},
|
||||
{"package": "excel.link", "reason": "windows-only"},
|
||||
{"package": "spectrino", "reason": "windows-only"},
|
||||
{"package": "taskscheduleR", "reason": "windows-only"},
|
||||
{"package": "MDSGUI", "reason": "windows-only"},
|
||||
{"package": "BiplotGUI", "reason": "windows-only"},
|
||||
{"package": "R2wd", "reason": "windows-only"},
|
||||
{"package": "rFUSION", "reason": "windows-only"},
|
||||
{"package": "MediaNews", "reason": "windows-only"},
|
||||
{"package": "doBy", "reason": "hang"},
|
||||
{"package": "IDPmisc", "reason": "hang"},
|
||||
{"package": "frailtypack", "reason": "hang"},
|
||||
{"package": "afex", "reason": "hang"},
|
||||
{"package": "FrF2", "reason": "hang"},
|
||||
{"package": "DoE.base", "reason": "hang"},
|
||||
{"package": "agricolae", "reason": "hang"},
|
||||
{"package": "doFuture", "reason": "hang"},
|
||||
{"package": "fscaret", "reason": "hang"},
|
||||
{"package": "PHYLOGR", "reason": "hang"},
|
||||
{"package": "seewave", "reason": "hang"},
|
||||
{"package": "pls", "reason": "hang"},
|
||||
{"package": "relaimpo", "reason": "hang"},
|
||||
{"package": "geepack", "reason": "hang"},
|
||||
{"package": "gggenes", "reason": "hang"},
|
||||
{"package": "NPCirc", "reason": "hang"},
|
||||
{"package": "repmis", "reason": "hang"},
|
||||
{"package": "PNDSIBGE", "reason": "hang"},
|
||||
{"package": "lidR", "reason": "hang"},
|
||||
{"package": "poismf", "reason": "hang"},
|
||||
{"package": "neonstore", "reason": "hang"},
|
||||
{"package": "MachineShop", "reason": "hang"},
|
||||
{"package": "mvst", "reason": "hang"},
|
||||
{"package": "MacBehaviour", "reason": "hang"},
|
||||
{"package": "mcmcderive", "reason": "hang"},
|
||||
{"package": "RGIFT", "reason": "hang"},
|
||||
{"package": "KnowBR", "reason": "hang"},
|
||||
{"package": "netmeta", "reason": "hang"},
|
||||
{"package": "spdep", "reason": "hang"},
|
||||
{"package": "Rfast", "reason": "hang"},
|
||||
{"package": "compareGroups", "reason": "hang"},
|
||||
{"package": "ff", "reason": "hang"},
|
||||
{"package": "GsymPoint", "reason": "hang"},
|
||||
{"package": "RcppDynProg", "reason": "hang"},
|
||||
{"package": "comtradr", "reason": "hang"},
|
||||
{"package": "FD", "reason": "hang"},
|
||||
{"package": "PearsonDS", "reason": "hang"},
|
||||
{"package": "DCluster", "reason": "hang"},
|
||||
{"package": "gRc", "reason": "hang"},
|
||||
{"package": "mixlm", "reason": "hang"},
|
||||
{"package": "geospt", "reason": "hang"},
|
||||
{"package": "fdth", "reason": "hang"},
|
||||
{"package": "ffmanova", "reason": "hang"},
|
||||
{"package": "fiery", "reason": "hang"},
|
||||
{"package": "ffscrapr", "reason": "hang"},
|
||||
{"package": "cold", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.DoE", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.NMBU", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.RiskDemo", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.ROC", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.TeachStat", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.TeachingDemos", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.UCA", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.WorldFlora", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.aRnova", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.depthTools", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.orloca", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.sos", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.survival", "reason": "hang"},
|
||||
{"package": "RcmdrPlugin.temis", "reason": "hang"},
|
||||
{"package": "GWlasso", "reason": "hang"},
|
||||
{"package": "GWmodelVis", "reason": "hang"}
|
||||
{ "package": "RInno", "reason": "windows-only" },
|
||||
{ "package": "KeyboardSimulator", "reason": "windows-only" },
|
||||
{ "package": "R2PPT", "reason": "windows-only" },
|
||||
{ "package": "RWinEdt", "reason": "windows-only" },
|
||||
{ "package": "blatr", "reason": "windows-only" },
|
||||
{ "package": "excel.link", "reason": "windows-only" },
|
||||
{ "package": "spectrino", "reason": "windows-only" },
|
||||
{ "package": "taskscheduleR", "reason": "windows-only" },
|
||||
{ "package": "MDSGUI", "reason": "windows-only" },
|
||||
{ "package": "BiplotGUI", "reason": "windows-only" },
|
||||
{ "package": "R2wd", "reason": "windows-only" },
|
||||
{ "package": "rFUSION", "reason": "windows-only" },
|
||||
{ "package": "MediaNews", "reason": "windows-only" },
|
||||
{ "package": "doBy", "reason": "hang" },
|
||||
{ "package": "IDPmisc", "reason": "hang" },
|
||||
{ "package": "frailtypack", "reason": "hang" },
|
||||
{ "package": "afex", "reason": "hang" },
|
||||
{ "package": "FrF2", "reason": "hang" },
|
||||
{ "package": "DoE.base", "reason": "hang" },
|
||||
{ "package": "agricolae", "reason": "hang" },
|
||||
{ "package": "doFuture", "reason": "hang" },
|
||||
{ "package": "fscaret", "reason": "hang" },
|
||||
{ "package": "PHYLOGR", "reason": "hang" },
|
||||
{ "package": "seewave", "reason": "hang" },
|
||||
{ "package": "pls", "reason": "hang" },
|
||||
{ "package": "relaimpo", "reason": "hang" },
|
||||
{ "package": "geepack", "reason": "hang" },
|
||||
{ "package": "gggenes", "reason": "hang" },
|
||||
{ "package": "NPCirc", "reason": "hang" },
|
||||
{ "package": "repmis", "reason": "hang" },
|
||||
{ "package": "PNDSIBGE", "reason": "hang" },
|
||||
{ "package": "lidR", "reason": "hang" },
|
||||
{ "package": "poismf", "reason": "hang" },
|
||||
{ "package": "neonstore", "reason": "hang" },
|
||||
{ "package": "MachineShop", "reason": "hang" },
|
||||
{ "package": "mvst", "reason": "hang" },
|
||||
{ "package": "MacBehaviour", "reason": "hang" },
|
||||
{ "package": "mcmcderive", "reason": "hang" },
|
||||
{ "package": "RGIFT", "reason": "hang" },
|
||||
{ "package": "KnowBR", "reason": "hang" },
|
||||
{ "package": "netmeta", "reason": "hang" },
|
||||
{ "package": "spdep", "reason": "hang" },
|
||||
{ "package": "Rfast", "reason": "hang" },
|
||||
{ "package": "compareGroups", "reason": "hang" },
|
||||
{ "package": "ff", "reason": "hang" },
|
||||
{ "package": "GsymPoint", "reason": "hang" },
|
||||
{ "package": "RcppDynProg", "reason": "hang" },
|
||||
{ "package": "comtradr", "reason": "hang" },
|
||||
{ "package": "FD", "reason": "hang" },
|
||||
{ "package": "PearsonDS", "reason": "hang" },
|
||||
{ "package": "DCluster", "reason": "hang" },
|
||||
{ "package": "gRc", "reason": "hang" },
|
||||
{ "package": "mixlm", "reason": "hang" },
|
||||
{ "package": "geospt", "reason": "hang" },
|
||||
{ "package": "fdth", "reason": "hang" },
|
||||
{ "package": "ffmanova", "reason": "hang" },
|
||||
{ "package": "fiery", "reason": "hang" },
|
||||
{ "package": "ffscrapr", "reason": "hang" },
|
||||
{ "package": "cold", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.DoE", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.NMBU", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.RiskDemo", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.ROC", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.TeachStat", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.TeachingDemos", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.UCA", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.WorldFlora", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.aRnova", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.depthTools", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.orloca", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.sos", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.survival", "reason": "hang" },
|
||||
{ "package": "RcmdrPlugin.temis", "reason": "hang" },
|
||||
{ "package": "GWlasso", "reason": "hang" },
|
||||
{ "package": "GWmodelVis", "reason": "hang" }
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,15 +1,21 @@
|
|||
library(httr2, quietly = TRUE)
|
||||
|
||||
forgejo_base <- "https://git.devxy.io/api/v1"
|
||||
repo <- "devxy/build-cran-binaries"
|
||||
platform <- Sys.getenv("PLATFORM")
|
||||
arch <- Sys.getenv("ARCH")
|
||||
token <- Sys.getenv("FORGEJO_TOKEN")
|
||||
output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt")
|
||||
repo <- "devxy/build-cran-binaries"
|
||||
platform <- Sys.getenv("PLATFORM")
|
||||
arch <- Sys.getenv("ARCH")
|
||||
token <- Sys.getenv("FORGEJO_TOKEN")
|
||||
output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt")
|
||||
|
||||
if (nchar(platform) == 0) stop("PLATFORM env var is not set")
|
||||
if (nchar(arch) == 0) stop("ARCH env var is not set")
|
||||
if (nchar(token) == 0) stop("FORGEJO_TOKEN env var is not set")
|
||||
if (nchar(platform) == 0) {
|
||||
stop("PLATFORM env var is not set")
|
||||
}
|
||||
if (nchar(arch) == 0) {
|
||||
stop("ARCH env var is not set")
|
||||
}
|
||||
if (nchar(token) == 0) {
|
||||
stop("FORGEJO_TOKEN env var is not set")
|
||||
}
|
||||
|
||||
os_family <- if (grepl("^ubuntu", platform)) {
|
||||
"Ubuntu"
|
||||
|
|
@ -21,12 +27,16 @@ os_family <- if (grepl("^ubuntu", platform)) {
|
|||
platform
|
||||
}
|
||||
|
||||
issue_title <- sprintf("Missing package binaries for latest version (%s)", os_family)
|
||||
issue_title <- sprintf(
|
||||
"Missing package binaries for latest version (%s)",
|
||||
os_family
|
||||
)
|
||||
cat(sprintf("Searching for issue: %s\n", issue_title))
|
||||
|
||||
search_url <- sprintf(
|
||||
"%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50",
|
||||
forgejo_base, repo,
|
||||
forgejo_base,
|
||||
repo,
|
||||
utils::URLencode(issue_title, reserved = TRUE)
|
||||
)
|
||||
search_resp <- request(search_url) |>
|
||||
|
|
@ -34,7 +44,9 @@ search_resp <- request(search_url) |>
|
|||
req_perform()
|
||||
|
||||
issues <- resp_body_json(search_resp, simplifyVector = FALSE)
|
||||
match_idx <- which(vapply(issues, function(x) x$title, character(1)) == issue_title)
|
||||
match_idx <- which(
|
||||
vapply(issues, function(x) x$title, character(1)) == issue_title
|
||||
)
|
||||
|
||||
if (length(match_idx) == 0) {
|
||||
cat("No matching issue found - nothing to rebuild\n")
|
||||
|
|
@ -55,7 +67,10 @@ lines <- strsplit(body, "\n", fixed = TRUE)[[1]]
|
|||
plat_header <- sprintf("## %s", platform)
|
||||
plat_idx <- which(lines == plat_header)
|
||||
if (length(plat_idx) == 0) {
|
||||
cat(sprintf("No section found for platform %s - nothing to rebuild\n", platform))
|
||||
cat(sprintf(
|
||||
"No section found for platform %s - nothing to rebuild\n",
|
||||
platform
|
||||
))
|
||||
writeLines(character(0), output_file)
|
||||
q("no")
|
||||
}
|
||||
|
|
@ -64,7 +79,11 @@ if (length(plat_idx) == 0) {
|
|||
arch_pattern <- sprintf("^### %s ", arch)
|
||||
arch_idx <- which(grepl(arch_pattern, lines) & seq_along(lines) > plat_idx[1])
|
||||
if (length(arch_idx) == 0) {
|
||||
cat(sprintf("No section found for arch %s under %s - nothing to rebuild\n", arch, platform))
|
||||
cat(sprintf(
|
||||
"No section found for arch %s under %s - nothing to rebuild\n",
|
||||
arch,
|
||||
platform
|
||||
))
|
||||
writeLines(character(0), output_file)
|
||||
q("no")
|
||||
}
|
||||
|
|
@ -82,6 +101,11 @@ pkg_lines <- section_lines[grepl("^- ", section_lines)]
|
|||
pkgs <- sub("^- ([^ ]+) \\(.*\\)$", "\\1", pkg_lines)
|
||||
pkgs <- pkgs[nchar(pkgs) > 0 & pkgs != "_None_"]
|
||||
|
||||
cat(sprintf("Found %d rebuildable packages for %s/%s\n", length(pkgs), platform, arch))
|
||||
cat(sprintf(
|
||||
"Found %d rebuildable packages for %s/%s\n",
|
||||
length(pkgs),
|
||||
platform,
|
||||
arch
|
||||
))
|
||||
writeLines(pkgs, output_file)
|
||||
cat(sprintf("Wrote package list to %s\n", output_file))
|
||||
|
|
|
|||
|
|
@ -6,9 +6,13 @@ archived <- quickcode::archivedPkg() |>
|
|||
pull(name)
|
||||
|
||||
# Query all distinct pkgs in the DB
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
con <- DBI::dbConnect(
|
||||
RPostgres::Postgres(),
|
||||
dbname = "build_metadata",
|
||||
host = "r-binaries.devxy.io",
|
||||
port = 15432,
|
||||
user = "r_binaries",
|
||||
password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
|
|
@ -22,7 +26,9 @@ to_process <- pkgs_db[pkgs_db %in% archived]
|
|||
|
||||
# for all matches, set 'removed = TRUE'
|
||||
sapply(to_process, function(.x) {
|
||||
DBI::dbExecute(con, "UPDATE single_builds SET removed = 'TRUE' where name = $1",
|
||||
DBI::dbExecute(
|
||||
con,
|
||||
"UPDATE single_builds SET removed = 'TRUE' where name = $1",
|
||||
params = list(.x)
|
||||
)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -7,4 +7,4 @@ query_metadata_table() |>
|
|||
group_by(platform, arch) |>
|
||||
arrange(desc(timestamp)) |>
|
||||
select(name, timestamp) |>
|
||||
filter(row_number()==1)
|
||||
filter(row_number() == 1)
|
||||
|
|
|
|||
|
|
@ -3,22 +3,34 @@ library(bincraft)
|
|||
library(dplyr)
|
||||
library(DBI)
|
||||
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
con <- DBI::dbConnect(
|
||||
RPostgres::Postgres(),
|
||||
dbname = "build_metadata",
|
||||
host = "r-binaries.devxy.io",
|
||||
port = 15432,
|
||||
user = "r_binaries",
|
||||
password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
`%nin%` <- Negate(`%in%`)
|
||||
|
||||
new_packages <- get_new_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
|
||||
removed_pkgs <- get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
|
||||
new_packages <- get_new_cran_packages(lubridate::interval(
|
||||
lubridate::today(),
|
||||
lubridate::today() - 2
|
||||
))$name
|
||||
removed_pkgs <- get_removed_cran_packages(lubridate::interval(
|
||||
lubridate::today(),
|
||||
lubridate::today() - 2
|
||||
))$name
|
||||
# filter windows packages
|
||||
# filter new packages from the last X days
|
||||
cran_pkgs <- unique(tools::CRAN_package_db() |>
|
||||
filter(`OS_type` != "windows" | is.na(`OS_type`)) |>
|
||||
filter(`Package` %nin% new_packages) |>
|
||||
pull(Package))
|
||||
cran_pkgs <- unique(
|
||||
tools::CRAN_package_db() |>
|
||||
filter(`OS_type` != "windows" | is.na(`OS_type`)) |>
|
||||
filter(`Package` %nin% new_packages) |>
|
||||
pull(Package)
|
||||
)
|
||||
|
||||
arch <- "arm64"
|
||||
platform <- "redhat-8"
|
||||
|
|
@ -28,13 +40,24 @@ platform <- "alpine-321"
|
|||
platform <- "ubuntu-2204"
|
||||
platform <- "ubuntu-2404"
|
||||
|
||||
data <- dbGetQuery(con, "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", params = list(platform, arch))
|
||||
removed_pkgs = get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
|
||||
data <- dbGetQuery(
|
||||
con,
|
||||
"SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;",
|
||||
params = list(platform, arch)
|
||||
)
|
||||
removed_pkgs = get_removed_cran_packages(lubridate::interval(
|
||||
lubridate::today(),
|
||||
lubridate::today() - 2
|
||||
))$name
|
||||
pkgs_db <- unique(data$name)
|
||||
pkgs_db = setdiff(pkgs_db, removed_pkgs)
|
||||
|
||||
pkgs <- setdiff(cran_pkgs, pkgs_db)
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)])))))
|
||||
formatted_pkgs <- gsub(
|
||||
'"',
|
||||
"'",
|
||||
capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)]))))
|
||||
)
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
|
|
@ -44,7 +67,6 @@ clipr::write_clip(formatted_pkgs_one_line)
|
|||
|
||||
# sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs)))
|
||||
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
|
||||
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
|
||||
|
|
|
|||
|
|
@ -10,19 +10,39 @@ s3fs::s3_file_system(
|
|||
)
|
||||
s3fs::s3_dir_ls("s3://devxy-r-package-binaries")
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"
|
||||
)) # done
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"
|
||||
)) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"
|
||||
))
|
||||
|
||||
files_b=basename(files)
|
||||
files_b = basename(files)
|
||||
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
||||
|
||||
cran_pkgs <- tools::CRAN_package_db() |>
|
||||
|
|
@ -30,7 +50,11 @@ cran_pkgs <- tools::CRAN_package_db() |>
|
|||
pull(Package)
|
||||
|
||||
pkgs = setdiff(cran_pkgs, files_b_split)
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900])))))
|
||||
formatted_pkgs <- gsub(
|
||||
'"',
|
||||
"'",
|
||||
capture.output(dput(as.character(na.omit(pkgs[1:900]))))
|
||||
)
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
|
|
|
|||
|
|
@ -7,42 +7,109 @@ s3fs::s3_file_system(
|
|||
region_name = "hel1",
|
||||
)
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"
|
||||
))
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"
|
||||
))
|
||||
|
||||
files_b=basename(files)
|
||||
files_b = basename(files)
|
||||
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
||||
files_b_split = setdiff(files_b_split, c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive"))
|
||||
files_b_split = setdiff(
|
||||
files_b_split,
|
||||
c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive")
|
||||
)
|
||||
# windows-only
|
||||
files_b_split = setdiff(files_b_split, c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive", "RInno", "KeyboardSimulator", "R2PPT", "RWinEdt", "blatr", "excel.link", "spectrino", "taskscheduleR", "MDSGUI", "BiplotGUI", "R2wd"))
|
||||
pkgs_index = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib")[, "Package"]
|
||||
pkgs_index_b=basename(pkgs_index)
|
||||
files_b_split = setdiff(
|
||||
files_b_split,
|
||||
c(
|
||||
"PACKAGES",
|
||||
"PACKAGES.gz",
|
||||
"PACKAGES.rds",
|
||||
"PACKAGES.db",
|
||||
"Archive",
|
||||
"RInno",
|
||||
"KeyboardSimulator",
|
||||
"R2PPT",
|
||||
"RWinEdt",
|
||||
"blatr",
|
||||
"excel.link",
|
||||
"spectrino",
|
||||
"taskscheduleR",
|
||||
"MDSGUI",
|
||||
"BiplotGUI",
|
||||
"R2wd"
|
||||
)
|
||||
)
|
||||
pkgs_index = available.packages(
|
||||
"https://cran.devxy.io/amd64/jammy/latest/src/contrib"
|
||||
)[, "Package"]
|
||||
pkgs_index_b = basename(pkgs_index)
|
||||
if (length(files_b_split) != length(pkgs_index_b)) {
|
||||
pkgs_missing = setdiff(files_b_split, pkgs_index_b)
|
||||
message("Packages missing in index but present in S3:")
|
||||
pkgs_missing
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs_missing[1:900])))))
|
||||
formatted_pkgs <- gsub(
|
||||
'"',
|
||||
"'",
|
||||
capture.output(dput(as.character(na.omit(pkgs_missing[1:900]))))
|
||||
)
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
}
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"
|
||||
))
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"
|
||||
))
|
||||
files <- s3fs::s3_dir_ls(sprintf(
|
||||
"devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"
|
||||
))
|
||||
|
|
|
|||
|
|
@ -17,7 +17,11 @@ suppressPackageStartupMessages(library(data.table))
|
|||
|
||||
arch = Sys.getenv("ARCH")
|
||||
# target: alpine-322, ubuntu-2404, redhat-9, etc.
|
||||
platform = paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-")
|
||||
platform = paste(
|
||||
Sys.getenv("OS"),
|
||||
gsub("[.]", "", Sys.getenv("OS_VERSION")),
|
||||
sep = "-"
|
||||
)
|
||||
# Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10")
|
||||
codename = bincraft::set_codename(NULL)
|
||||
|
||||
|
|
@ -34,7 +38,9 @@ con <- DBI::dbConnect(
|
|||
cran_archive = tools::CRAN_archive_db()
|
||||
cran_release = tools::CRAN_package_db()
|
||||
# Subset cran_archive to only those packages
|
||||
cran_archive_in_release <- cran_archive[names(cran_archive) %in% cran_release$Package]
|
||||
cran_archive_in_release <- cran_archive[
|
||||
names(cran_archive) %in% cran_release$Package
|
||||
]
|
||||
|
||||
archive_versions <- rbindlist(
|
||||
lapply(names(cran_archive), function(pkg) {
|
||||
|
|
@ -126,11 +132,15 @@ query_error <- function(pkg, ver) {
|
|||
# Fetch all relevant columns from the database
|
||||
errored_pkgs <- DBI::dbGetQuery(
|
||||
con,
|
||||
sprintf("SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", platform, arch)
|
||||
sprintf(
|
||||
"SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'",
|
||||
platform,
|
||||
arch
|
||||
)
|
||||
)
|
||||
errored_pkgs <- as.data.table(errored_pkgs)
|
||||
setkey(pkgs_to_build, Package, Version)
|
||||
setnames(errored_pkgs, c("Package","Version"))
|
||||
setnames(errored_pkgs, c("Package", "Version"))
|
||||
setkey(errored_pkgs, Package, Version)
|
||||
|
||||
### Final subsetting
|
||||
|
|
@ -140,3 +150,62 @@ pkgs <- pkgs_no_error[!s3_dt]
|
|||
# Deduplicate
|
||||
pkgs <- unique(pkgs)
|
||||
setorder(pkgs, Package, Version)
|
||||
|
||||
### R-minor sensitivity (classify once per package, applied to all versions)
|
||||
source(file.path("local", "r-minor-helpers.R"))
|
||||
risky_deps <- bincraft::abi_risky_linking_deps()
|
||||
|
||||
release_meta <- data.table(
|
||||
Package = cran_release$Package,
|
||||
NeedsCompilation = cran_release$NeedsCompilation,
|
||||
LinkingTo = cran_release$LinkingTo
|
||||
)
|
||||
|
||||
meta <- release_meta[Package %in% unique(pkgs$Package)]
|
||||
meta[,
|
||||
triage := mapply(
|
||||
classify_from_metadata,
|
||||
NeedsCompilation,
|
||||
LinkingTo,
|
||||
MoreArgs = list(risky_deps = risky_deps)
|
||||
)
|
||||
]
|
||||
|
||||
# Only the "ambiguous" compiled packages need a source grep.
|
||||
ambiguous <- meta[triage == "ambiguous", Package]
|
||||
sensitive_ambiguous <- character()
|
||||
if (length(ambiguous) > 0L) {
|
||||
tmp_src <- file.path(tempdir(), "abi_src")
|
||||
dir.create(tmp_src, showWarnings = FALSE, recursive = TRUE)
|
||||
sens <- vapply(
|
||||
ambiguous,
|
||||
function(pkg) {
|
||||
out <- tryCatch(
|
||||
{
|
||||
dl <- utils::download.packages(
|
||||
pkg,
|
||||
destdir = tmp_src,
|
||||
repos = "https://cloud.r-project.org",
|
||||
quiet = TRUE
|
||||
)
|
||||
isTRUE(as.logical(bincraft::needs_per_minor_recompile(dl[1L, 2L])))
|
||||
},
|
||||
error = function(e) TRUE
|
||||
) # fail safe: treat as sensitive
|
||||
out
|
||||
},
|
||||
logical(1L)
|
||||
)
|
||||
sensitive_ambiguous <- ambiguous[sens]
|
||||
}
|
||||
|
||||
sensitive_pkgs <- unique(c(
|
||||
meta[triage == "sensitive", Package],
|
||||
sensitive_ambiguous
|
||||
))
|
||||
pkgs[, r_minor_sensitive := Package %in% sensitive_pkgs]
|
||||
sprintf(
|
||||
"R-minor-sensitive packages: %s of %s",
|
||||
length(sensitive_pkgs),
|
||||
uniqueN(pkgs$Package)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,24 @@
|
|||
### Lists packages without any successful binaries, i.e. pkgs for which all builds errored (per platform & arch)
|
||||
library(DBI)
|
||||
library(dplyr)
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
con <- DBI::dbConnect(
|
||||
RPostgres::Postgres(),
|
||||
dbname = "build_metadata",
|
||||
host = "r-binaries.devxy.io",
|
||||
port = 15432,
|
||||
user = "r_binaries",
|
||||
password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
cran_pkgs <- tools::CRAN_package_db() |>
|
||||
filter(`Date/Publication` <= "2024-12-02") |>
|
||||
cran_pkgs <- tools::CRAN_package_db() |>
|
||||
filter(`Date/Publication` <= "2024-12-02") |>
|
||||
pull(Package)
|
||||
|
||||
data <- dbGetQuery(con, "SELECT name,platform,arch,removed,error_occurred FROM single_builds;")
|
||||
data <- dbGetQuery(
|
||||
con,
|
||||
"SELECT name,platform,arch,removed,error_occurred FROM single_builds;"
|
||||
)
|
||||
|
||||
pkgs <- data |>
|
||||
filter(platform == "alpine-320", arch == "arm64") |>
|
||||
|
|
@ -22,7 +29,11 @@ pkgs <- data |>
|
|||
|
||||
pkgs = setdiff(cran_pkgs, pkgs)
|
||||
# Format, remove line breaks, and print as a single line
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900])))))
|
||||
formatted_pkgs <- gsub(
|
||||
'"',
|
||||
"'",
|
||||
capture.output(dput(as.character(na.omit(pkgs[1:900]))))
|
||||
)
|
||||
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
|
|
|
|||
33
local/r-minor-helpers.R
Normal file
33
local/r-minor-helpers.R
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# Metadata-only ABI triage so the full build avoids downloading every source.
|
||||
# Mirrors bincraft::abi_classify rules 1-2; "ambiguous" packages still need a
|
||||
# source grep via bincraft::needs_per_minor_recompile().
|
||||
|
||||
classify_from_metadata <- function(needs_compilation, linking_to, risky_deps) {
|
||||
nc <- if (length(needs_compilation) == 0L || is.na(needs_compilation)) {
|
||||
""
|
||||
} else {
|
||||
tolower(trimws(needs_compilation))
|
||||
}
|
||||
if (!identical(nc, "yes")) {
|
||||
return("not-sensitive")
|
||||
}
|
||||
lt <- if (length(linking_to) == 0L || is.na(linking_to)) "" else linking_to
|
||||
linked <- trimws(unlist(strsplit(lt, "[,\n]")))
|
||||
linked <- sub("\\s*\\(.*\\)$", "", linked) # strip "(>= x)" constraints
|
||||
linked <- linked[nzchar(linked)]
|
||||
if (any(linked %in% risky_deps)) {
|
||||
return("sensitive")
|
||||
}
|
||||
"ambiguous"
|
||||
}
|
||||
|
||||
parse_build_args <- function(args) {
|
||||
sensitive_only <- "--sensitive-only" %in% args
|
||||
pos <- args[!startsWith(args, "--")]
|
||||
list(
|
||||
sensitive_only = sensitive_only,
|
||||
split_into = as.integer(pos[1L]),
|
||||
split_index = as.integer(pos[2L]),
|
||||
ncpus = as.integer(pos[3L])
|
||||
)
|
||||
}
|
||||
13
local/tests/test-build-all-args.R
Normal file
13
local/tests/test-build-all-args.R
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
source(file.path("..", "r-minor-helpers.R"))
|
||||
|
||||
test_that("parse_build_args splits flags from positionals", {
|
||||
a <- parse_build_args(c("--sensitive-only", "4", "2", "8"))
|
||||
expect_true(a$sensitive_only)
|
||||
expect_identical(a$split_into, 4L)
|
||||
expect_identical(a$split_index, 2L)
|
||||
expect_identical(a$ncpus, 8L)
|
||||
|
||||
b <- parse_build_args(c("4", "2", "8"))
|
||||
expect_false(b$sensitive_only)
|
||||
expect_identical(b$split_into, 4L)
|
||||
})
|
||||
28
local/tests/test-r-minor-helpers.R
Normal file
28
local/tests/test-r-minor-helpers.R
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
source(file.path("..", "r-minor-helpers.R"))
|
||||
|
||||
test_that("pure-r (NeedsCompilation != yes) is not sensitive", {
|
||||
expect_identical(classify_from_metadata("no", NA, c("Rcpp")), "not-sensitive")
|
||||
expect_identical(
|
||||
classify_from_metadata("", "Rcpp", c("Rcpp")),
|
||||
"not-sensitive"
|
||||
)
|
||||
})
|
||||
|
||||
test_that("LinkingTo a risky dep is sensitive (version constraints stripped)", {
|
||||
expect_identical(
|
||||
classify_from_metadata("yes", "Rcpp (>= 1.0)", c("Rcpp")),
|
||||
"sensitive"
|
||||
)
|
||||
expect_identical(
|
||||
classify_from_metadata("yes", "R6,\n cpp11", c("Rcpp", "cpp11")),
|
||||
"sensitive"
|
||||
)
|
||||
})
|
||||
|
||||
test_that("compiled but no risky LinkingTo is ambiguous (needs source)", {
|
||||
expect_identical(
|
||||
classify_from_metadata("yes", "R6", c("Rcpp", "cpp11")),
|
||||
"ambiguous"
|
||||
)
|
||||
expect_identical(classify_from_metadata("yes", NA, c("Rcpp")), "ambiguous")
|
||||
})
|
||||
|
|
@ -15,10 +15,14 @@ library(httr2, quietly = TRUE)
|
|||
# Environment / config
|
||||
# ---------------------------------------------------------------------------
|
||||
platform <- Sys.getenv("PLATFORM")
|
||||
arch <- Sys.getenv("ARCH")
|
||||
arch <- Sys.getenv("ARCH")
|
||||
|
||||
if (nchar(platform) == 0) stop("PLATFORM env var is not set")
|
||||
if (nchar(arch) == 0) stop("ARCH env var is not set")
|
||||
if (nchar(platform) == 0) {
|
||||
stop("PLATFORM env var is not set")
|
||||
}
|
||||
if (nchar(arch) == 0) {
|
||||
stop("ARCH env var is not set")
|
||||
}
|
||||
|
||||
# Map platform names to S3 codenames
|
||||
s3_codename <- gsub("-", "", platform)
|
||||
|
|
@ -37,8 +41,13 @@ os_family <- if (grepl("^ubuntu", platform)) {
|
|||
platform
|
||||
}
|
||||
|
||||
cat(sprintf("Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n",
|
||||
platform, arch, s3_codename, os_family))
|
||||
cat(sprintf(
|
||||
"Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n",
|
||||
platform,
|
||||
arch,
|
||||
s3_codename,
|
||||
os_family
|
||||
))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Query PostgreSQL for known build failures (before s3fs init to avoid
|
||||
|
|
@ -47,19 +56,20 @@ cat(sprintf("Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n",
|
|||
cat("Connecting to PostgreSQL...\n")
|
||||
con <- DBI::dbConnect(
|
||||
RPostgres::Postgres(),
|
||||
dbname = "build_metadata",
|
||||
host = "r-binaries.devxy.io",
|
||||
port = 15432,
|
||||
user = "rpkgs",
|
||||
dbname = "build_metadata",
|
||||
host = "r-binaries.devxy.io",
|
||||
port = 15432,
|
||||
user = "rpkgs",
|
||||
password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
errored_pkgs <- DBI::dbGetQuery(
|
||||
con,
|
||||
sprintf(
|
||||
"SELECT name, tag FROM single_builds WHERE error_occurred = TRUE AND platform = '%s' AND arch = '%s'",
|
||||
platform, arch
|
||||
platform,
|
||||
arch
|
||||
)
|
||||
)
|
||||
DBI::dbDisconnect(con)
|
||||
|
|
@ -69,7 +79,12 @@ if (nrow(errored_dt) > 0) {
|
|||
setnames(errored_dt, c("Package", "Version"))
|
||||
setkey(errored_dt, Package, Version)
|
||||
}
|
||||
cat(sprintf("Found %d known build failures for %s/%s\n", nrow(errored_dt), platform, arch))
|
||||
cat(sprintf(
|
||||
"Found %d known build failures for %s/%s\n",
|
||||
nrow(errored_dt),
|
||||
platform,
|
||||
arch
|
||||
))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. CRAN release packages (uses curl internally - must come after PG)
|
||||
|
|
@ -86,28 +101,36 @@ cran_dt <- data.table(
|
|||
# ---------------------------------------------------------------------------
|
||||
cat("Connecting to S3...\n")
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region_name = "eu-central-003",
|
||||
refresh = TRUE
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region_name = "eu-central-003",
|
||||
refresh = TRUE
|
||||
)
|
||||
|
||||
s3_path <- sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, s3_codename)
|
||||
s3_path <- sprintf(
|
||||
"devxy-rpkgs-binaries/%s/%s/latest/src/contrib",
|
||||
arch,
|
||||
s3_codename
|
||||
)
|
||||
cat(sprintf("Listing S3 path: %s\n", s3_path))
|
||||
|
||||
s3_pkgs <- tryCatch(
|
||||
s3fs::s3_dir_ls(s3_path, recurse = FALSE),
|
||||
error = function(e) {
|
||||
cat(sprintf("WARNING: Could not list S3 path %s: %s\n", s3_path, conditionMessage(e)))
|
||||
cat(sprintf(
|
||||
"WARNING: Could not list S3 path %s: %s\n",
|
||||
s3_path,
|
||||
conditionMessage(e)
|
||||
))
|
||||
character(0)
|
||||
}
|
||||
)
|
||||
|
||||
file_names <- basename(s3_pkgs)
|
||||
matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names)
|
||||
parts <- regmatches(file_names, matches)
|
||||
parts <- parts[sapply(parts, length) == 3]
|
||||
matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names)
|
||||
parts <- regmatches(file_names, matches)
|
||||
parts <- parts[sapply(parts, length) == 3]
|
||||
if (length(parts) == 0) {
|
||||
s3_dt <- data.table(Package = character(0), Version = character(0))
|
||||
} else {
|
||||
|
|
@ -117,13 +140,18 @@ if (length(parts) == 0) {
|
|||
)
|
||||
}
|
||||
|
||||
cat(sprintf("S3 contains %d tarballs for %s/%s\n", nrow(s3_dt), arch, s3_codename))
|
||||
cat(sprintf(
|
||||
"S3 contains %d tarballs for %s/%s\n",
|
||||
nrow(s3_dt),
|
||||
arch,
|
||||
s3_codename
|
||||
))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Find missing packages (CRAN release version not in S3)
|
||||
# ---------------------------------------------------------------------------
|
||||
setkey(cran_dt, Package, Version)
|
||||
setkey(s3_dt, Package, Version)
|
||||
setkey(s3_dt, Package, Version)
|
||||
missing_dt <- cran_dt[!s3_dt]
|
||||
cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt)))
|
||||
|
||||
|
|
@ -132,9 +160,15 @@ cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt)))
|
|||
# ---------------------------------------------------------------------------
|
||||
script_dir <- tryCatch(
|
||||
dirname(normalizePath(
|
||||
if (exists("ofile", envir = sys.frame(1), inherits = FALSE)) sys.frame(1)$ofile
|
||||
else commandArgs(trailingOnly = FALSE)[grepl("--file=", commandArgs(trailingOnly = FALSE))] |>
|
||||
sub("--file=", "", x = _),
|
||||
if (exists("ofile", envir = sys.frame(1), inherits = FALSE)) {
|
||||
sys.frame(1)$ofile
|
||||
} else {
|
||||
commandArgs(trailingOnly = FALSE)[grepl(
|
||||
"--file=",
|
||||
commandArgs(trailingOnly = FALSE)
|
||||
)] |>
|
||||
sub("--file=", "", x = _)
|
||||
},
|
||||
mustWork = FALSE
|
||||
)),
|
||||
error = function(e) "local"
|
||||
|
|
@ -143,13 +177,23 @@ excluded_path <- file.path(script_dir, "excluded-packages.json")
|
|||
|
||||
excluded_pkgs <- data.table(package = character(0), reason = character(0))
|
||||
if (file.exists(excluded_path)) {
|
||||
excluded_raw <- jsonlite::fromJSON(excluded_path)
|
||||
excluded_raw <- jsonlite::fromJSON(excluded_path)
|
||||
excluded_pkgs <- as.data.table(excluded_raw)
|
||||
cat(sprintf("Loaded %d excluded packages from %s\n", nrow(excluded_pkgs), excluded_path))
|
||||
cat(sprintf(
|
||||
"Loaded %d excluded packages from %s\n",
|
||||
nrow(excluded_pkgs),
|
||||
excluded_path
|
||||
))
|
||||
missing_dt <- missing_dt[!Package %in% excluded_pkgs$package]
|
||||
cat(sprintf("%d packages remain after removing exclusions\n", nrow(missing_dt)))
|
||||
cat(sprintf(
|
||||
"%d packages remain after removing exclusions\n",
|
||||
nrow(missing_dt)
|
||||
))
|
||||
} else {
|
||||
cat(sprintf("No excluded-packages.json found at %s -- skipping exclusion step\n", excluded_path))
|
||||
cat(sprintf(
|
||||
"No excluded-packages.json found at %s -- skipping exclusion step\n",
|
||||
excluded_path
|
||||
))
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -157,24 +201,34 @@ if (file.exists(excluded_path)) {
|
|||
# ---------------------------------------------------------------------------
|
||||
if (nrow(errored_dt) > 0) {
|
||||
known_failures_dt <- missing_dt[errored_dt, nomatch = 0]
|
||||
rebuildable_dt <- missing_dt[!errored_dt]
|
||||
rebuildable_dt <- missing_dt[!errored_dt]
|
||||
} else {
|
||||
known_failures_dt <- missing_dt[0]
|
||||
rebuildable_dt <- missing_dt
|
||||
rebuildable_dt <- missing_dt
|
||||
}
|
||||
|
||||
cat(sprintf("Rebuildable: %d | Known failures: %d\n",
|
||||
nrow(rebuildable_dt), nrow(known_failures_dt)))
|
||||
cat(sprintf(
|
||||
"Rebuildable: %d | Known failures: %d\n",
|
||||
nrow(rebuildable_dt),
|
||||
nrow(known_failures_dt)
|
||||
))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 7. Write rebuildable package names to cache RDS
|
||||
# ---------------------------------------------------------------------------
|
||||
cache_dir <- "/mnt/cache/packages"
|
||||
cache_file <- file.path(cache_dir, sprintf("weekly_rebuild_%s_%s.rds", platform, arch))
|
||||
cache_dir <- "/mnt/cache/packages"
|
||||
cache_file <- file.path(
|
||||
cache_dir,
|
||||
sprintf("weekly_rebuild_%s_%s.rds", platform, arch)
|
||||
)
|
||||
|
||||
if (dir.exists(cache_dir)) {
|
||||
saveRDS(rebuildable_dt$Package, cache_file)
|
||||
cat(sprintf("Wrote %d rebuildable packages to %s\n", nrow(rebuildable_dt), cache_file))
|
||||
cat(sprintf(
|
||||
"Wrote %d rebuildable packages to %s\n",
|
||||
nrow(rebuildable_dt),
|
||||
cache_file
|
||||
))
|
||||
} else {
|
||||
cat(sprintf("Cache dir %s does not exist -- skipping RDS write\n", cache_dir))
|
||||
}
|
||||
|
|
@ -186,29 +240,47 @@ forgejo_token <- Sys.getenv("FORGEJO_TOKEN")
|
|||
if (nchar(forgejo_token) == 0) {
|
||||
cat("FORGEJO_TOKEN not set -- skipping issue update\n")
|
||||
} else {
|
||||
issue_title <- sprintf("Missing package binaries for latest version (%s)", os_family)
|
||||
issue_title <- sprintf(
|
||||
"Missing package binaries for latest version (%s)",
|
||||
os_family
|
||||
)
|
||||
|
||||
n_missing <- nrow(missing_dt)
|
||||
n_rebuild <- nrow(rebuildable_dt)
|
||||
n_missing <- nrow(missing_dt)
|
||||
n_rebuild <- nrow(rebuildable_dt)
|
||||
|
||||
arch_lines <- sprintf("### %s (%d missing, %d to rebuild)", arch, n_missing, n_rebuild)
|
||||
arch_lines <- sprintf(
|
||||
"### %s (%d missing, %d to rebuild)",
|
||||
arch,
|
||||
n_missing,
|
||||
n_rebuild
|
||||
)
|
||||
if (nrow(rebuildable_dt) > 0) {
|
||||
arch_lines <- c(arch_lines,
|
||||
arch_lines <- c(
|
||||
arch_lines,
|
||||
paste0("- ", rebuildable_dt$Package, " (", rebuildable_dt$Version, ")")
|
||||
)
|
||||
} else {
|
||||
arch_lines <- c(arch_lines, "_None_")
|
||||
}
|
||||
if (nrow(known_failures_dt) > 0) {
|
||||
arch_lines <- c(arch_lines,
|
||||
arch_lines <- c(
|
||||
arch_lines,
|
||||
"",
|
||||
"#### Known build failures",
|
||||
paste0("- ", known_failures_dt$Package, " (", known_failures_dt$Version, ")")
|
||||
paste0(
|
||||
"- ",
|
||||
known_failures_dt$Package,
|
||||
" (",
|
||||
known_failures_dt$Version,
|
||||
")"
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
build_excluded_footer <- function() {
|
||||
if (nrow(excluded_pkgs) == 0) return(character(0))
|
||||
if (nrow(excluded_pkgs) == 0) {
|
||||
return(character(0))
|
||||
}
|
||||
entries <- paste(
|
||||
paste0(excluded_pkgs$package, " (", excluded_pkgs$reason, ")"),
|
||||
collapse = ", "
|
||||
|
|
@ -217,11 +289,12 @@ if (nchar(forgejo_token) == 0) {
|
|||
}
|
||||
|
||||
forgejo_base <- "https://git.devxy.io/api/v1"
|
||||
repo <- "devxy/build-cran-binaries"
|
||||
repo <- "devxy/build-cran-binaries"
|
||||
|
||||
search_url <- sprintf(
|
||||
"%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50",
|
||||
forgejo_base, repo,
|
||||
forgejo_base,
|
||||
repo,
|
||||
utils::URLencode(issue_title, reserved = TRUE)
|
||||
)
|
||||
search_resp <- httr2::request(search_url) |>
|
||||
|
|
@ -230,15 +303,19 @@ if (nchar(forgejo_token) == 0) {
|
|||
|
||||
existing_issues <- httr2::resp_body_json(search_resp, simplifyVector = FALSE)
|
||||
|
||||
match_idx <- which(sapply(existing_issues, function(x) x$title) == issue_title)
|
||||
match_idx <- which(
|
||||
sapply(existing_issues, function(x) x$title) == issue_title
|
||||
)
|
||||
|
||||
today_str <- format(Sys.Date(), "%Y-%m-%d")
|
||||
|
||||
if (length(match_idx) > 0) {
|
||||
# ---- Update existing issue ----
|
||||
issue_number <- existing_issues[[match_idx[1]]]$number
|
||||
old_body <- existing_issues[[match_idx[1]]]$body
|
||||
if (is.null(old_body)) old_body <- ""
|
||||
old_body <- existing_issues[[match_idx[1]]]$body
|
||||
if (is.null(old_body)) {
|
||||
old_body <- ""
|
||||
}
|
||||
|
||||
lines <- strsplit(old_body, "\n", fixed = TRUE)[[1]]
|
||||
|
||||
|
|
@ -258,7 +335,11 @@ if (nchar(forgejo_token) == 0) {
|
|||
if (length(plat_idx) == 0) {
|
||||
# Platform section missing -- insert before --- footer
|
||||
footer_idx <- which(lines == "---")
|
||||
insert_at <- if (length(footer_idx) > 0) footer_idx[length(footer_idx)] else length(lines) + 1
|
||||
insert_at <- if (length(footer_idx) > 0) {
|
||||
footer_idx[length(footer_idx)]
|
||||
} else {
|
||||
length(lines) + 1
|
||||
}
|
||||
|
||||
new_plat_block <- c(sprintf("## %s", platform), "", arch_lines, "")
|
||||
lines <- c(
|
||||
|
|
@ -271,10 +352,14 @@ if (nchar(forgejo_token) == 0) {
|
|||
|
||||
# End of platform section: next ## or --- at a higher level, or EOF
|
||||
next_section <- which(grepl("^## |^---", lines) & seq_along(lines) > pi)
|
||||
plat_end <- if (length(next_section) > 0) next_section[1] - 1 else length(lines)
|
||||
plat_end <- if (length(next_section) > 0) {
|
||||
next_section[1] - 1
|
||||
} else {
|
||||
length(lines)
|
||||
}
|
||||
|
||||
plat_lines <- lines[seq(pi, plat_end)]
|
||||
arch_local_idx <- which(plat_lines == arch_header)
|
||||
plat_lines <- lines[seq(pi, plat_end)]
|
||||
arch_local_idx <- which(plat_lines == arch_header)
|
||||
|
||||
if (length(arch_local_idx) == 0) {
|
||||
# Append arch subsection at end of platform block
|
||||
|
|
@ -285,11 +370,13 @@ if (nchar(forgejo_token) == 0) {
|
|||
lines[seq(plat_end + 1, length(lines))]
|
||||
)
|
||||
} else {
|
||||
ai <- pi + arch_local_idx[1] - 1 # absolute line index
|
||||
ai <- pi + arch_local_idx[1] - 1 # absolute line index
|
||||
|
||||
# End of arch subsection
|
||||
next_arch <- which(grepl("^### |^## |^---", lines) & seq_along(lines) > ai)
|
||||
arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end
|
||||
next_arch <- which(
|
||||
grepl("^### |^## |^---", lines) & seq_along(lines) > ai
|
||||
)
|
||||
arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end
|
||||
|
||||
lines <- c(
|
||||
lines[seq_len(ai - 1)],
|
||||
|
|
@ -302,8 +389,12 @@ if (nchar(forgejo_token) == 0) {
|
|||
# Rebuild excluded footer
|
||||
excl_hdr_idx <- which(lines == "## Excluded packages")
|
||||
if (length(excl_hdr_idx) > 0) {
|
||||
pre_dash <- which(lines == "---" & seq_along(lines) < excl_hdr_idx[1])
|
||||
remove_from <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_hdr_idx[1]
|
||||
pre_dash <- which(lines == "---" & seq_along(lines) < excl_hdr_idx[1])
|
||||
remove_from <- if (length(pre_dash) > 0) {
|
||||
pre_dash[length(pre_dash)]
|
||||
} else {
|
||||
excl_hdr_idx[1]
|
||||
}
|
||||
lines <- lines[seq_len(remove_from - 1)]
|
||||
}
|
||||
footer <- build_excluded_footer()
|
||||
|
|
@ -313,10 +404,15 @@ if (nchar(forgejo_token) == 0) {
|
|||
|
||||
new_body <- paste(lines, collapse = "\n")
|
||||
|
||||
patch_url <- sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, issue_number)
|
||||
patch_url <- sprintf(
|
||||
"%s/repos/%s/issues/%d",
|
||||
forgejo_base,
|
||||
repo,
|
||||
issue_number
|
||||
)
|
||||
httr2::request(patch_url) |>
|
||||
httr2::req_headers(
|
||||
Authorization = paste("token", forgejo_token),
|
||||
Authorization = paste("token", forgejo_token),
|
||||
`Content-Type` = "application/json"
|
||||
) |>
|
||||
httr2::req_body_json(list(body = new_body)) |>
|
||||
|
|
@ -324,7 +420,6 @@ if (nchar(forgejo_token) == 0) {
|
|||
httr2::req_perform()
|
||||
|
||||
cat(sprintf("Updated Forgejo issue #%d: %s\n", issue_number, issue_title))
|
||||
|
||||
} else {
|
||||
# ---- Create new issue ----
|
||||
body_lines <- c(
|
||||
|
|
@ -342,17 +437,21 @@ if (nchar(forgejo_token) == 0) {
|
|||
post_url <- sprintf("%s/repos/%s/issues", forgejo_base, repo)
|
||||
create_resp <- httr2::request(post_url) |>
|
||||
httr2::req_headers(
|
||||
Authorization = paste("token", forgejo_token),
|
||||
Authorization = paste("token", forgejo_token),
|
||||
`Content-Type` = "application/json"
|
||||
) |>
|
||||
httr2::req_body_json(list(
|
||||
title = issue_title,
|
||||
body = paste(body_lines, collapse = "\n")
|
||||
body = paste(body_lines, collapse = "\n")
|
||||
)) |>
|
||||
httr2::req_perform()
|
||||
|
||||
new_issue <- httr2::resp_body_json(create_resp)
|
||||
cat(sprintf("Created Forgejo issue #%d: %s\n", new_issue$number, issue_title))
|
||||
cat(sprintf(
|
||||
"Created Forgejo issue #%d: %s\n",
|
||||
new_issue$number,
|
||||
issue_title
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue