refactor: bincraftR -> bincraft
Some checks failed
ci/crow/cron/process-updates-alpine-320-arm64 Pipeline failed
ci/crow/cron/process-updates-redhat-8-arm64 Pipeline failed
ci/crow/cron/process-updates-redhat-8-amd64 Pipeline failed
ci/crow/cron/process-updates-alpine-320-amd64 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2404-amd64 Pipeline failed
ci/crow/cron/process-updates-redhat-9-arm64 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2204-arm64 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2204-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-322-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-322-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-9-amd64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2404-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-321-amd64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-8-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-320-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-9-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-8-arm64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2204-arm64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2404-arm64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-320-amd64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2204-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-321-arm64 Pipeline was successful
ci/crow/cron/process-updates-redhat-9-amd64 Pipeline was successful
ci/crow/cron/process-updates-ubuntu-2404-arm64 Pipeline failed
ci/crow/cron/process-updates-alpine-321-amd64 Pipeline was successful
ci/crow/cron/process-updates-alpine-321-arm64 Pipeline was successful
Some checks failed
ci/crow/cron/process-updates-alpine-320-arm64 Pipeline failed
ci/crow/cron/process-updates-redhat-8-arm64 Pipeline failed
ci/crow/cron/process-updates-redhat-8-amd64 Pipeline failed
ci/crow/cron/process-updates-alpine-320-amd64 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2404-amd64 Pipeline failed
ci/crow/cron/process-updates-redhat-9-arm64 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2204-arm64 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2204-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-322-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-322-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-9-amd64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2404-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-321-amd64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-8-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-320-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-9-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-8-arm64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2204-arm64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2404-arm64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-320-amd64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2204-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-321-arm64 Pipeline was successful
ci/crow/cron/process-updates-redhat-9-amd64 Pipeline was successful
ci/crow/cron/process-updates-ubuntu-2404-arm64 Pipeline failed
ci/crow/cron/process-updates-alpine-321-amd64 Pipeline was successful
ci/crow/cron/process-updates-alpine-321-arm64 Pipeline was successful
This commit is contained in:
parent
bdece25754
commit
321e3d3205
8 changed files with 285 additions and 243 deletions
|
|
@ -1,5 +1,5 @@
|
|||
### Scope: Archive packages that have more than one entry in repository root
|
||||
library(bincraftR)
|
||||
library(bincraft)
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
|
||||
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
|
||||
|
|
|
|||
|
|
@ -1,27 +1,63 @@
|
|||
library(future.apply)
|
||||
|
||||
access_key <- Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S")
|
||||
secret_key <- Sys.getenv("HETZNER_S3_SECRET_KEY_K3S")
|
||||
|
||||
# Set up S3 connection
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
|
||||
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
|
||||
aws_access_key_id = access_key,
|
||||
aws_secret_access_key = secret_key,
|
||||
endpoint = "https://hel1.your-objectstorage.com",
|
||||
region_name = "hel1",
|
||||
region_name = "hel1"
|
||||
)
|
||||
|
||||
### Scope: Shows if there are duplicated packages at the S3 repository root (e.g. due to missed/aborted archiving)
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine321/latest/src/contrib"))
|
||||
# Define architectures and codenames
|
||||
arches <- c("amd64", "arm64")
|
||||
codenames <- c("jammy", "noble", "rhel8", "rhel9", "alpine320", "alpine321")
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
|
||||
# Create all combinations as a data frame
|
||||
combos <- expand.grid(arch = arches, codename = codenames)
|
||||
|
||||
files_b=basename(files)
|
||||
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
||||
files_b_split[duplicated(files_b_split)]
|
||||
|
||||
bincraftR::archive_package("Keng", codename = "alpine320", arch = "arm64")
|
||||
# Set up parallel plan
|
||||
future::plan(multisession) # or multicore
|
||||
|
||||
# Parallel processing
|
||||
future.apply::future_lapply(
|
||||
seq_len(nrow(combos)),
|
||||
function(i) {
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = access_key,
|
||||
aws_secret_access_key = secret_key,
|
||||
endpoint = "https://hel1.your-objectstorage.com",
|
||||
region_name = "hel1",
|
||||
refresh = TRUE
|
||||
)
|
||||
arch <- combos$arch[i]
|
||||
codename <- combos$codename[i]
|
||||
files <- s3fs::s3_dir_ls(
|
||||
sprintf(
|
||||
"devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib",
|
||||
arch, codename
|
||||
)
|
||||
)
|
||||
if (length(files) == 0) return(NULL)
|
||||
files_b <- basename(files)
|
||||
pkg_names <- sapply(strsplit(files_b, "_"), `[`, 1)
|
||||
dupes <- unique(pkg_names[duplicated(pkg_names)])
|
||||
for (pkg in dupes) {
|
||||
bincraft::archive_package(
|
||||
pkg,
|
||||
codename = codename,
|
||||
arch = arch,
|
||||
s3_region = "hel1",
|
||||
s3_endpoint = "https://hel1.your-objectstorage.com",
|
||||
s3_bucket = "devxy-r-package-binaries-hel1",
|
||||
s3_access_key_id = access_key,
|
||||
s3_secret_access_key = secret_key
|
||||
)
|
||||
}
|
||||
NULL
|
||||
},
|
||||
future.globals = TRUE,
|
||||
future.seed = TRUE
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
### Scope: List decomisisoned CRAN packages and updates them in the metadata DB
|
||||
library(dplyr)
|
||||
library(bincraftR)
|
||||
library(bincraft)
|
||||
archived <- quickcode::archivedPkg() |>
|
||||
filter(arch.status == "decom") |>
|
||||
pull(name)
|
||||
|
|
@ -13,7 +13,7 @@ con <- DBI::dbConnect(RPostgres::Postgres(),
|
|||
)
|
||||
|
||||
pkgs_db <- query_metadata_table() |>
|
||||
filter(removed == FALSE) |>
|
||||
filter(removed == FALSE) |>
|
||||
distinct(name) |>
|
||||
pull(name)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
### Scope: Show last processed pkgs and timestamp by platform/OS
|
||||
library(dplyr)
|
||||
library(bincraftR)
|
||||
library(bincraft)
|
||||
|
||||
### Show last processed package by each platform/arch
|
||||
query_metadata_table() |>
|
||||
group_by(platform, arch) |>
|
||||
arrange(desc(timestamp)) |>
|
||||
select(name, timestamp) |>
|
||||
group_by(platform, arch) |>
|
||||
arrange(desc(timestamp)) |>
|
||||
select(name, timestamp) |>
|
||||
filter(row_number()==1)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")'
|
||||
# remotes::install_gitlab("devxy/r-package-binaries/bincraftR")
|
||||
# R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")'
|
||||
# remotes::install_gitlab("devxy/r-package-binaries/bincraft")
|
||||
|
||||
library(bincraftR)
|
||||
library(bincraft)
|
||||
library(dplyr)
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
### Scope: Lists packages which are missing any DB entry compared to what is available on CRAN
|
||||
library(bincraftR)
|
||||
library(bincraft)
|
||||
library(dplyr)
|
||||
library(DBI)
|
||||
|
||||
|
|
@ -20,7 +20,7 @@ cran_pkgs <- unique(tools::CRAN_package_db() |>
|
|||
filter(`Package` %nin% new_packages) |>
|
||||
pull(Package))
|
||||
|
||||
arch <- "amd64"
|
||||
arch <- "arm64"
|
||||
platform <- "redhat-8"
|
||||
platform <- "redhat-9"
|
||||
platform <- "alpine-320"
|
||||
|
|
@ -42,4 +42,14 @@ length(pkgs)
|
|||
|
||||
clipr::write_clip(formatted_pkgs_one_line)
|
||||
|
||||
# sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs)))
|
||||
# sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs)))
|
||||
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
|
||||
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
|
||||
endpoint = "https://hel1.your-objectstorage.com",
|
||||
region_name = "hel1",
|
||||
)
|
||||
|
||||
s3fs::s3_dir_delete("devxy-r-package-binaries-docs/_site")
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
### Scope: Lists packages which are missing any tarball in S3 compared to the CRAN pkg listing
|
||||
library(bincraftR)
|
||||
library(bincraft)
|
||||
library(dplyr)
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
|
||||
|
|
@ -25,8 +25,8 @@ files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/
|
|||
files_b=basename(files)
|
||||
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
|
||||
|
||||
cran_pkgs <- tools::CRAN_package_db() |>
|
||||
filter(`Date/Publication` <= "2025-01-25") |>
|
||||
cran_pkgs <- tools::CRAN_package_db() |>
|
||||
filter(`Date/Publication` <= "2025-01-25") |>
|
||||
pull(Package)
|
||||
|
||||
pkgs = setdiff(cran_pkgs, files_b_split)
|
||||
|
|
@ -36,4 +36,3 @@ formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
|||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
|
||||
clipr::write_clip(formatted_pkgs_one_line)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
library(bincraftR)
|
||||
library(bincraft)
|
||||
library(dplyr)
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
|
||||
|
|
@ -32,20 +32,17 @@ if (length(files_b_split) != length(pkgs_index_b)) {
|
|||
pkgs_missing
|
||||
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs_missing[1:900])))))
|
||||
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
|
||||
|
||||
|
||||
cat(formatted_pkgs_one_line, "\n", sep = "")
|
||||
}
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
|
||||
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
|
||||
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue