update local/ scripts
This commit is contained in:
parent
fcf2b3fed2
commit
2588b2512f
GPG key ID:
3C6318841EF78925
8 changed files with 130 additions and 106 deletions
|
|
@ -1,3 +1,4 @@
|
|||
### Scope: Archive packages that have more than one entry in repository root
|
||||
library(bincraftR)
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
|
||||
|
|
@ -6,8 +7,8 @@ s3fs::s3_file_system(
|
|||
region_name = "eu-central-003",
|
||||
)
|
||||
|
||||
arch = "arm64"
|
||||
os = "jammy"
|
||||
arch = "amd64"
|
||||
os = "noble"
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/%s/%s/latest/src/contrib", arch, os))
|
||||
length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1])))
|
||||
|
|
|
|||
45
local/check-NA.R
Normal file
45
local/check-NA.R
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
|
||||
### Scope: Check whether there are any incomplete entries in PACKAGES.rds. These will result in NA when calling available.packages()
|
||||
foo = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/amd64/noble/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/amd64/rhel8/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/amd64/rhel9/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/amd64/alpine320/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
# arm64
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/arm64/jammy/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/arm64/noble/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/arm64/rhel8/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/arm64/rhel9/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
|
||||
foo = available.packages("https://cran.devxy.io/arm64/alpine320/latest/src/contrib")
|
||||
sum((is.na(foo[, "Version"])))
|
||||
which((is.na(foo[, "Version"])))
|
||||
18
local/detect-duplicates.R
Normal file
18
local/detect-duplicates.R
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
### Scope: Shows if there are duplicated packages at the S3 repository root (e.g. due to missed/aborted archiving)
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib"))
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/alpine320/latest/src/contrib"))
|
||||
|
||||
files_b=basename(files)
|
||||
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
||||
files_b_split[duplicated(files_b_split)]
|
||||
|
||||
archive_package("Rglpk", codename = "alpine320", arch = "arm64")
|
||||
25
local/label-removed-cran-packages.R
Normal file
25
local/label-removed-cran-packages.R
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
### Scope: List decomisisoned CRAN packages and updates them in the metadata DB
|
||||
archived <- quickcode::archivedPkg() |>
|
||||
filter(arch.status == "decom") |>
|
||||
pull(name)
|
||||
|
||||
# Query all distinct pkgs in the DB
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
pkgs_db <- query_metadata_table() |>
|
||||
distinct(name) |>
|
||||
pull(name)
|
||||
|
||||
# Check which pkgs in the DB match with the decomissioned ones
|
||||
to_process <- pkgs_db[pkgs_db %in% archived]
|
||||
|
||||
# for all matches, set 'removed = TRUE'
|
||||
sapply(to_process, function(.x) {
|
||||
DBI::dbExecute(con, "UPDATE single_builds SET removed = 'TRUE' where name = $1",
|
||||
params = list(.x)
|
||||
)
|
||||
})
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
devtools::load_all()
|
||||
### Scope: Show last processed pkgs and timestamp by platform/OS
|
||||
library(dplyr)
|
||||
library(bincraftR)
|
||||
|
||||
### Show last processed package by each platform/arch
|
||||
query_metadata_table() |>
|
||||
|
|
|
|||
36
local/missing-cran-packages.R
Normal file
36
local/missing-cran-packages.R
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
### Scope: Lists packages which are missing any tarball in S3 compared to the CRAN pkg listing
|
||||
library(bincraftR)
|
||||
library(dplyr)
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region_name = "eu-central-003",
|
||||
)
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib"))
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel8/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib")) # done
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/alpine320/latest/src/contrib"))
|
||||
|
||||
files_b=basename(files)
|
||||
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
||||
|
||||
cran_pkgs <- tools::CRAN_package_db() |>
|
||||
filter(`Date/Publication` <= "2024-11-21") |>
|
||||
pull(Package)
|
||||
|
||||
pkgs = setdiff(cran_pkgs, files_b_split)
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:400])))))
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
|
||||
clipr::write_clip(formatted_pkgs_one_line)
|
||||
22
local/packages-without-any-binary.R
Normal file
22
local/packages-without-any-binary.R
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
### Lists packages without any successful binaries, i.e. pkgs for which all builds errored (per platform & arch)
|
||||
library(DBI)
|
||||
library(dplyr)
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
data <- dbGetQuery(con, "SELECT name,platform,arch FROM single_builds group by name,platform,arch HAVING COUNT(*) = SUM(CASE WHEN error_occurred = 'TRUE' AND removed = 'FALSE' THEN 1 ELSE 0 END);")
|
||||
|
||||
pkgs <- data |>
|
||||
filter(platform == "ubuntu-2204", arch == "arm64") |>
|
||||
pull(name)
|
||||
|
||||
# Format, remove line breaks, and print as a single line
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(pkgs[1:300])))
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
|
||||
clipr::write_clip(formatted_pkgs_one_line)
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
###
|
||||
# Scope: List Packages for which the most recent CRAN release is not available
|
||||
###
|
||||
library(furrr)
|
||||
library(cli)
|
||||
library(dplyr)
|
||||
library(data.table)
|
||||
|
||||
last_versions_cran <- available.packages("https://cloud.r-project.org/src/contrib")[, c("Package", "Version")]
|
||||
|
||||
arch <- c("amd64", "arm64")
|
||||
os_ids <- c("rhel8", "rhel9", "noble", "jammy", "alpine320")
|
||||
grid <- tidyr::expand_grid(arch = arch, os_ids = os_ids)
|
||||
|
||||
# Define the function to check for missing packages
|
||||
check_package <- function(pkg_name, available_pkgs, version_cran, arch, os_id) {
|
||||
if (pkg_name %in% available_pkgs$Package) {
|
||||
version_devxy <- available_pkgs %>%
|
||||
filter(Package == pkg_name) %>%
|
||||
pull(Version)
|
||||
|
||||
if (is.na(version_devxy) || version_cran != version_devxy) {
|
||||
# Package is missing or version mismatch
|
||||
return(data.frame(pkg_name = pkg_name, arch = arch, ID = os_id))
|
||||
}
|
||||
} else {
|
||||
# Package completely missing in devxy
|
||||
return(data.frame(pkg_name = pkg_name, arch = arch, ID = os_id))
|
||||
}
|
||||
return(NULL) # No mismatch, no entry added
|
||||
}
|
||||
|
||||
# Run the mapping with parallel processing for each (arch, os_id) combination
|
||||
future::plan("multisession", workers = 5)
|
||||
|
||||
|
||||
pkgs_missing <- future_map2(grid$arch, grid$os_ids, ~ {
|
||||
# Prepare the URL and fetch available packages only once per (arch, os_id)
|
||||
repo_url <- sprintf("https://cran.devxy.io/%s/%s/latest/src/contrib", .x, .y)
|
||||
available_pkgs <- as.data.table(available.packages(contriburl = repo_url)[, c("Package", "Version")])
|
||||
|
||||
# Define the packages to check once
|
||||
cran_pkgs <- as.data.table(last_versions_cran)
|
||||
|
||||
# Accumulate results in a list
|
||||
pkg_entries <- list()
|
||||
|
||||
# Loop through each package name and check if it's missing
|
||||
for (pkg_name in cran_pkgs$Package) {
|
||||
version_cran <- cran_pkgs[Package == pkg_name, Version]
|
||||
result <- check_package(pkg_name, available_pkgs, version_cran, .x, .y)
|
||||
if (!is.null(result)) {
|
||||
pkg_entries[[length(pkg_entries) + 1]] <- result
|
||||
}
|
||||
}
|
||||
|
||||
# Combine results for the current (arch, os_id)
|
||||
if (length(pkg_entries) > 0) {
|
||||
return(rbindlist(pkg_entries))
|
||||
} else {
|
||||
return(data.table(pkg_name = character(), arch = character(), ID = character()))
|
||||
}
|
||||
})
|
||||
|
||||
df_missing_pkgs <- rbindlist(pkgs_missing, fill = TRUE)
|
||||
|
||||
# Group and count missing packages by architecture and ID
|
||||
df_summary <- df_missing_pkgs %>%
|
||||
group_by(arch, ID) %>%
|
||||
arrange(arch, ID) %>%
|
||||
count()
|
||||
|
||||
# Extract and copy missing package names for each architecture and ID combination
|
||||
extract_and_copy_pkgs <- function(df, arch, range = 300, id, clip = FALSE) {
|
||||
pkgs <- df %>%
|
||||
filter(arch == arch, ID == id) %>%
|
||||
pull(pkg_name)
|
||||
|
||||
# Format, remove line breaks, and print as a single line
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(pkgs[1:range])))
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
|
||||
if (clip) {
|
||||
clipr::write_clip(formatted_pkgs_one_line)
|
||||
}
|
||||
|
||||
return(pkgs)
|
||||
}
|
||||
|
||||
# Usage for different architecture and ID combinations
|
||||
pkgs_missing_amd64_rhel8 <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "rhel8", range = 20, clip = TRUE)
|
||||
pkgs_missing_amd64_rhel9 <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "rhel9", clip = TRUE)
|
||||
pkgs_missing_amd64_jammy <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "jammy", clip = TRUE)
|
||||
pkgs_missing_amd64_noble <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "noble", clip = TRUE)
|
||||
pkgs_missing_amd64_alpine320 <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "alpine320", clip = TRUE)
|
||||
|
||||
pkgs_missing_arm64_rhel8 <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "rhel8", clip = TRUE)
|
||||
pkgs_missing_arm64_rhel9 <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "rhel9", clip = TRUE)
|
||||
pkgs_missing_arm64_jammy <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "jammy", clip = TRUE)
|
||||
pkgs_missing_arm64_noble <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "noble", clip = TRUE)
|
||||
pkgs_missing_arm64_alpine320 <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "alpine320", clip = TRUE)
|
||||
Loading…
Reference in a new issue