devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 10 Pull requests Releases Activity

feat: migrate to hel1 location
Some checks failed
ci/crow/cron/process-updates-ubuntu-2204-arm64 Pipeline failed
Details
ci/crow/cron/update-package-index-redhat-9-amd64 Pipeline was successful
Details
ci/crow/cron/update-package-index-redhat-8-amd64 Pipeline was successful
Details
ci/crow/cron/update-package-index-alpine-320-amd64 Pipeline was successful
Details
ci/crow/cron/update-package-index-ubuntu-2204-amd64 Pipeline was successful
Details
ci/crow/cron/update-package-index-ubuntu-2404-amd64 Pipeline was successful
Details
ci/crow/cron/update-package-index-alpine-321-amd64 Pipeline was successful
Details
ci/crow/cron/update-package-index-redhat-9-arm64 Pipeline was successful
Details
ci/crow/cron/update-package-index-redhat-8-arm64 Pipeline was successful
Details
ci/crow/cron/update-package-index-ubuntu-2204-arm64 Pipeline was successful
Details
ci/crow/cron/update-package-index-alpine-320-arm64 Pipeline was successful
Details
ci/crow/cron/update-package-index-alpine-321-arm64 Pipeline was successful
Details
ci/crow/cron/update-package-index-ubuntu-2404-arm64 Pipeline was successful
Details
ci/crow/cron/process-updates-alpine-320-arm64 Pipeline was successful
Details
ci/crow/cron/process-updates-redhat-8-arm64 Pipeline was successful
Details
ci/crow/cron/process-updates-alpine-321-arm64 Pipeline was successful
Details
ci/crow/cron/process-updates-redhat-8-amd64 Pipeline was successful
Details
ci/crow/cron/process-updates-alpine-320-amd64 Pipeline was successful
Details
ci/crow/cron/process-updates-ubuntu-2404-amd64 Pipeline was successful
Details
ci/crow/cron/process-updates-redhat-9-arm64 Pipeline was successful
Details
ci/crow/cron/process-updates-ubuntu-2204-amd64 Pipeline was successful
Details
ci/crow/cron/process-updates-redhat-9-amd64 Pipeline was successful
Details
ci/crow/cron/process-updates-alpine-321-amd64 Pipeline was successful
Details
ci/crow/cron/process-updates-ubuntu-2404-arm64 Pipeline was successful
Details

Browse source
This commit is contained in:
Patrick Schratz 2025-04-29 09:13:50 +02:00
commit 16e773b655
Signed by: pat-s
GPG key ID: 3C6318841EF78925
6 changed files with 204 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

22
local/archive-missed-pkgs.R
View file

@ -3,18 +3,24 @@ library(bincraftR)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
endpoint = "https://fsn1.your-objectstorage.com",
region_name = "fsn1",
endpoint = "https://hel1.your-objectstorage.com",
region_name = "hel1",
refresh = T
)
arch = "amd64"
os = "noble"
arch = "arm64"
os = "alpine320"
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/%s/%s/latest/src/contrib", arch, os))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", arch, os))
length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1])))
non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) x[1])[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))])
# RInno is not built because it only exists for Windows
non_archived = setdiff(non_archived, "RInno")
future::plan("sequential", workers = 8)
future.apply::future_lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch))
archive_package()
# future::plan("sequential", workers = 8)
future::plan("sequential", workers = 1)
# future.apply::future_lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch))
lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch))
# archive_package()
grep("duckdb_", files, value = T)

2
local/debug-packages-writing.R
View file

@ -2,7 +2,7 @@ library(future)
future::plan(multisession)
future::plan(sequential)
time = Sys.time()
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel9/latest/src/contrib/Archive"), recurse = T)
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive"), recurse = T)
Sys.time() - time

26
local/detect-duplicates.R
View file

@ -1,23 +1,23 @@
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
endpoint = "https://fsn1.your-objectstorage.com",
region_name = "fsn1",
endpoint = "https://hel1.your-objectstorage.com",
region_name = "hel1",
)
### Scope: Shows if there are duplicated packages at the S3 repository root (e.g. due to missed/aborted archiving)
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/alpine321/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine321/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
files_b=basename(files)
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])

39
local/manual-package-index-update.R Normal file
View file

@ -0,0 +1,39 @@
# R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")'
# remotes::install_gitlab("devxy/r-package-binaries/bincraftR")
library(bincraftR)
library(dplyr)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
endpoint = "https://hel1.your-objectstorage.com",
region_name = "hel1",
refresh = TRUE
)
s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES.db")
s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES.gz")
s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES.rds")
s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES")
s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/duckdbfs_0.0.9.tar.gz")$etag
s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/Archive/duckplyr/duckplyr_0.4.1.tar.gz")$etag
s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/duckplyr_1.0.0.tar.gz")$etag
s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/duckduckr_1.0.0.tar.gz")$etag
s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/getDTeval_0.0.2.tar.gz")$etag
s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/dualtrees_0.1.5.tar.gz")$etag
s3fs::s3_file_info("devxy-r-package-binaries-hel1/arm64/alpine321/latest/src/contrib/tidyselect_1.2.1.tar.gz")
s3fs::s3_file_info("devxy-r-package-binaries-hel1/arm64/alpine321/latest/src/contrib/tidysq_1.2.3.tar.gz")
# s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib/Archive/curl/curl_6.0.1.tar.gz")
s3fs::s3_file_download("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db")
upload_package_index(codename = "noble", arch = "arm64")
fs::file_delete("PACKAGES.db")
con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES-debug.db")
DBI::dbListTables(con)
df = DBI::dbReadTable(con, "packages")

26
local/missing-cran-packages.R
View file

@ -4,29 +4,29 @@ library(dplyr)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
endpoint = "https://fsn1.your-objectstorage.com",
region_name = "fsn1",
endpoint = "https://hel1.your-objectstorage.com",
region_name = "hel1",
refresh = T
)
s3fs::s3_dir_ls("s3://devxy-r-package-binaries")
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/jammy/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/noble/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel8/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel9/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/alpine320/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/jammy/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/noble/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel8/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel9/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
files_b=basename(files)
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
cran_pkgs <- tools::CRAN_package_db() |>
filter(`Date/Publication` <= "2024-12-02") |>
filter(`Date/Publication` <= "2025-01-25") |>
pull(Package)
pkgs = setdiff(cran_pkgs, files_b_split)

42
local/missing-packages-in-index.R
View file

@ -3,21 +3,21 @@ library(dplyr)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
endpoint = "https://fsn1.your-objectstorage.com",
region_name = "fsn1",
endpoint = "https://hel1.your-objectstorage.com",
region_name = "hel1",
)
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))
files_b=basename(files)
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
@ -36,16 +36,16 @@ if (length(files_b_split) != length(pkgs_index_b)) {
cat(formatted_pkgs_one_line, "\n", sep = "")
}
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib"))