devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 10 Pull requests Releases Activity

chore: update local/ files

Browse source
This commit is contained in:
Patrick Schratz 2024-12-13 14:34:53 +01:00
commit a78d489aac
Signed by: pat-s
GPG key ID: 3C6318841EF78925
7 changed files with 78 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

3
local/check-cdn-cache.sh
View file

@ -6,7 +6,6 @@ curl -s https://cran.devxy.io/arm64/jammy/latest/src/contrib/PACKAGES | grep '^P
curl -s https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.gz | gzip -cd | grep '^Package:' | wc -l
curl -s https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l
curl -s https://cran.devxy.io/arm64/rhel8/latest/src/contrib/PACKAGES.gz | gzip -cd | grep '^Package:' | wc -l
curl -s https://cran.devxy.io/arm64/rhel8/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l
@ -31,4 +30,4 @@ curl -s https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.gz | gzip
curl -s https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l
curl -s https://cran.devxy.io/amd64/alpine320/latest/src/contrib/PACKAGES.gz | gzip -cd | grep '^Package:' | wc -l
curl -s https://cran.devxy.io/amd64/alpine320/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l
curl -s https://cran.devxy.io/amd64/alpine320/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l

14
local/debug-packages-writing.R Normal file
View file

@ -0,0 +1,14 @@
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib"))
curl::curl_download("https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db")
con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db")
df = DBI::dbReadTable(con, "packages")
cranlike:::write_packages_files("/tmp", "PACKAGES.db")
# check
which(grepl("digest", df$Package))
which(grepl("digest", df$Package))
system("cat /tmp/PACKAGES | grep '^Package: digest' | wc -l")

3
local/detect-duplicates.R
View file

@ -15,4 +15,5 @@ files_b=basename(files)
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
files_b_split[duplicated(files_b_split)]
archive_package("Rglpk", codename = "alpine320", arch = "arm64")
archive_package("Rglpk", codename = "alpine320", arch = "arm64")

3
local/label-removed-cran-packages.R
View file

@ -1,4 +1,6 @@
### Scope: List decomisisoned CRAN packages and updates them in the metadata DB
library(dplyr)
library(bincraftR)
archived <- quickcode::archivedPkg() |>
filter(arch.status == "decom") |>
pull(name)
@ -11,6 +13,7 @@ con <- DBI::dbConnect(RPostgres::Postgres(),
)
pkgs_db <- query_metadata_table() |>
filter(removed == FALSE) |>
distinct(name) |>
pull(name)

42
local/missing-cran-packages-db.R Normal file
View file

@ -0,0 +1,42 @@
### Scope: Lists packages which are missing any DB entry compared to what is available on CRAN
library(bincraftR)
library(dplyr)
library(DBI)
con <- DBI::dbConnect(RPostgres::Postgres(),
dbname = "build_metadata", host = "r-binaries.devxy.io",
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
sslmode = "require"
)
`%nin%` <- Negate(`%in%`)
new_packages <- get_new_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
removed_pkgs <- get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
# filter windows packages
# filter new packages from the last X days
cran_pkgs <- unique(tools::CRAN_package_db() |>
filter(`OS_type` != "windows" | is.na(`OS_type`)) |>
filter(`Package` %nin% new_packages) |>
pull(Package))
arch <- "amd64"
platform <- "redhat-8"
platform <- "redhat-9"
platform <- "alpine-320"
platform <- "ubuntu-2204"
platform <- "ubuntu-2404"
data <- dbGetQuery(con, "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", params = list(platform, arch))
removed_pkgs = get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
pkgs_db <- unique(data$name)
pkgs_db = setdiff(pkgs_db, removed_pkgs)
pkgs <- setdiff(cran_pkgs, pkgs_db)
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900])))))
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
cat(formatted_pkgs_one_line, "\n", sep = "")
length(formatted_pkgs)
clipr::write_clip(formatted_pkgs_one_line)

7
local/missing-cran-packages.R
View file

@ -12,7 +12,7 @@ files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib")) # done
@ -24,13 +24,14 @@ files_b=basename(files)
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
cran_pkgs <- tools::CRAN_package_db() |>
filter(`Date/Publication` <= "2024-11-21") |>
filter(`Date/Publication` <= "2024-12-02") |>
pull(Package)
pkgs = setdiff(cran_pkgs, files_b_split)
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:400])))))
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900])))))
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
cat(formatted_pkgs_one_line, "\n", sep = "")
clipr::write_clip(formatted_pkgs_one_line)

15
local/packages-without-any-binary.R
View file

@ -7,16 +7,25 @@ con <- DBI::dbConnect(RPostgres::Postgres(),
sslmode = "require"
)
data <- dbGetQuery(con, "SELECT name,platform,arch FROM single_builds group by name,platform,arch HAVING COUNT(*) = SUM(CASE WHEN error_occurred = 'TRUE' AND removed = 'FALSE' THEN 1 ELSE 0 END);")
cran_pkgs <- tools::CRAN_package_db() |>
filter(`Date/Publication` <= "2024-12-02") |>
pull(Package)
data <- dbGetQuery(con, "SELECT name,platform,arch,removed,error_occurred FROM single_builds;")
pkgs <- data |>
filter(platform == "ubuntu-2204", arch == "arm64") |>
filter(platform == "redhat-8", arch == "arm64") |>
filter(removed == FALSE) |>
filter(error_occurred == FALSE) |>
distinct(name) |>
pull(name)
pkgs = setdiff(cran_pkgs, pkgs)
# Format, remove line breaks, and print as a single line
formatted_pkgs <- gsub('"', "'", capture.output(dput(pkgs[1:300])))
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:300])))))
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
cat(formatted_pkgs_one_line, "\n", sep = "")
length(pkgs)
clipr::write_clip(formatted_pkgs_one_line)