refactor: bincraftR -> bincraft
Some checks failed
ci/crow/cron/process-updates-alpine-320-arm64 Pipeline failed
ci/crow/cron/process-updates-redhat-8-arm64 Pipeline failed
ci/crow/cron/process-updates-redhat-8-amd64 Pipeline failed
ci/crow/cron/process-updates-alpine-320-amd64 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2404-amd64 Pipeline failed
ci/crow/cron/process-updates-redhat-9-arm64 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2204-arm64 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2204-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-322-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-322-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-9-amd64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2404-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-321-amd64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-8-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-320-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-9-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-8-arm64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2204-arm64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2404-arm64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-320-amd64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2204-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-321-arm64 Pipeline was successful
ci/crow/cron/process-updates-redhat-9-amd64 Pipeline was successful
ci/crow/cron/process-updates-ubuntu-2404-arm64 Pipeline failed
ci/crow/cron/process-updates-alpine-321-amd64 Pipeline was successful
ci/crow/cron/process-updates-alpine-321-arm64 Pipeline was successful

This commit is contained in:
Patrick Schratz 2025-06-08 15:39:54 +02:00
commit 321e3d3205
Signed by: pat-s
GPG key ID: 3C6318841EF78925
8 changed files with 285 additions and 243 deletions

View file

@ -1,5 +1,5 @@
### Scope: Archive packages that have more than one entry in repository root
library(bincraftR)
library(bincraft)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),

View file

@ -1,27 +1,63 @@
library(future.apply)
access_key <- Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S")
secret_key <- Sys.getenv("HETZNER_S3_SECRET_KEY_K3S")
# Set up S3 connection
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
aws_access_key_id = access_key,
aws_secret_access_key = secret_key,
endpoint = "https://hel1.your-objectstorage.com",
region_name = "hel1",
region_name = "hel1"
)
### Scope: Shows if there are duplicated packages at the S3 repository root (e.g. due to missed/aborted archiving)
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine321/latest/src/contrib"))
# Define architectures and codenames
arches <- c("amd64", "arm64")
codenames <- c("jammy", "noble", "rhel8", "rhel9", "alpine320", "alpine321")
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
# Create all combinations as a data frame
combos <- expand.grid(arch = arches, codename = codenames)
files_b=basename(files)
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
files_b_split[duplicated(files_b_split)]
bincraftR::archive_package("Keng", codename = "alpine320", arch = "arm64")
# Set up parallel plan
future::plan(multisession) # or multicore
# Parallel processing
future.apply::future_lapply(
seq_len(nrow(combos)),
function(i) {
s3fs::s3_file_system(
aws_access_key_id = access_key,
aws_secret_access_key = secret_key,
endpoint = "https://hel1.your-objectstorage.com",
region_name = "hel1",
refresh = TRUE
)
arch <- combos$arch[i]
codename <- combos$codename[i]
files <- s3fs::s3_dir_ls(
sprintf(
"devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib",
arch, codename
)
)
if (length(files) == 0) return(NULL)
files_b <- basename(files)
pkg_names <- sapply(strsplit(files_b, "_"), `[`, 1)
dupes <- unique(pkg_names[duplicated(pkg_names)])
for (pkg in dupes) {
bincraft::archive_package(
pkg,
codename = codename,
arch = arch,
s3_region = "hel1",
s3_endpoint = "https://hel1.your-objectstorage.com",
s3_bucket = "devxy-r-package-binaries-hel1",
s3_access_key_id = access_key,
s3_secret_access_key = secret_key
)
}
NULL
},
future.globals = TRUE,
future.seed = TRUE
)

View file

@ -1,6 +1,6 @@
### Scope: List decomisisoned CRAN packages and updates them in the metadata DB
library(dplyr)
library(bincraftR)
library(bincraft)
archived <- quickcode::archivedPkg() |>
filter(arch.status == "decom") |>
pull(name)
@ -13,7 +13,7 @@ con <- DBI::dbConnect(RPostgres::Postgres(),
)
pkgs_db <- query_metadata_table() |>
filter(removed == FALSE) |>
filter(removed == FALSE) |>
distinct(name) |>
pull(name)

View file

@ -1,10 +1,10 @@
### Scope: Show last processed pkgs and timestamp by platform/OS
library(dplyr)
library(bincraftR)
library(bincraft)
### Show last processed package by each platform/arch
query_metadata_table() |>
group_by(platform, arch) |>
arrange(desc(timestamp)) |>
select(name, timestamp) |>
group_by(platform, arch) |>
arrange(desc(timestamp)) |>
select(name, timestamp) |>
filter(row_number()==1)

View file

@ -1,7 +1,7 @@
# R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")'
# remotes::install_gitlab("devxy/r-package-binaries/bincraftR")
# R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")'
# remotes::install_gitlab("devxy/r-package-binaries/bincraft")
library(bincraftR)
library(bincraft)
library(dplyr)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),

View file

@ -1,5 +1,5 @@
### Scope: Lists packages which are missing any DB entry compared to what is available on CRAN
library(bincraftR)
library(bincraft)
library(dplyr)
library(DBI)
@ -20,7 +20,7 @@ cran_pkgs <- unique(tools::CRAN_package_db() |>
filter(`Package` %nin% new_packages) |>
pull(Package))
arch <- "amd64"
arch <- "arm64"
platform <- "redhat-8"
platform <- "redhat-9"
platform <- "alpine-320"
@ -42,4 +42,14 @@ length(pkgs)
clipr::write_clip(formatted_pkgs_one_line)
# sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs)))
# sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs)))
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
endpoint = "https://hel1.your-objectstorage.com",
region_name = "hel1",
)
s3fs::s3_dir_delete("devxy-r-package-binaries-docs/_site")

View file

@ -1,5 +1,5 @@
### Scope: Lists packages which are missing any tarball in S3 compared to the CRAN pkg listing
library(bincraftR)
library(bincraft)
library(dplyr)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
@ -25,8 +25,8 @@ files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/
files_b=basename(files)
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
cran_pkgs <- tools::CRAN_package_db() |>
filter(`Date/Publication` <= "2025-01-25") |>
cran_pkgs <- tools::CRAN_package_db() |>
filter(`Date/Publication` <= "2025-01-25") |>
pull(Package)
pkgs = setdiff(cran_pkgs, files_b_split)
@ -36,4 +36,3 @@ formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
cat(formatted_pkgs_one_line, "\n", sep = "")
clipr::write_clip(formatted_pkgs_one_line)

View file

@ -1,4 +1,4 @@
library(bincraftR)
library(bincraft)
library(dplyr)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
@ -32,20 +32,17 @@ if (length(files_b_split) != length(pkgs_index_b)) {
pkgs_missing
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs_missing[1:900])))))
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
cat(formatted_pkgs_one_line, "\n", sep = "")
}
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))