chore: update local/ files

local/check-cdn-cache.sh

@@ -6,7 +6,6 @@ curl -s https://cran.devxy.io/arm64/jammy/latest/src/contrib/PACKAGES | grep '^P
 curl -s https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.gz | gzip -cd | grep '^Package:' | wc -l
 curl -s https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l
 
-
 curl -s https://cran.devxy.io/arm64/rhel8/latest/src/contrib/PACKAGES.gz | gzip -cd | grep '^Package:' | wc -l
 curl -s https://cran.devxy.io/arm64/rhel8/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l
 

local/debug-packages-writing.R

@@ -0,0 +1,14 @@
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib"))
+
+curl::curl_download("https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db")
+con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db")
+df = DBI::dbReadTable(con, "packages")
+
+cranlike:::write_packages_files("/tmp", "PACKAGES.db")
+
+# check
+which(grepl("digest", df$Package))
+which(grepl("digest", df$Package))
+
+system("cat /tmp/PACKAGES | grep '^Package: digest' | wc -l")
+

local/detect-duplicates.R

@@ -16,3 +16,4 @@ files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
 files_b_split[duplicated(files_b_split)]
 
 archive_package("Rglpk", codename = "alpine320", arch = "arm64")
+

local/label-removed-cran-packages.R

@@ -1,4 +1,6 @@
 ### Scope: List decomisisoned CRAN packages and updates them in the metadata DB
+library(dplyr)
+library(bincraftR)
 archived <- quickcode::archivedPkg() |>
   filter(arch.status == "decom") |>
   pull(name)
@@ -11,6 +13,7 @@ con <- DBI::dbConnect(RPostgres::Postgres(),
 )
 
 pkgs_db <- query_metadata_table() |>
+  filter(removed == FALSE) |> 
   distinct(name) |>
   pull(name)
 

local/missing-cran-packages-db.R

@@ -0,0 +1,42 @@
+### Scope: Lists packages which are missing any DB entry compared to what is available on CRAN
+library(bincraftR)
+library(dplyr)
+library(DBI)
+
+con <- DBI::dbConnect(RPostgres::Postgres(),
+  dbname = "build_metadata", host = "r-binaries.devxy.io",
+  port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
+  sslmode = "require"
+)
+
+`%nin%` <- Negate(`%in%`)
+
+new_packages <- get_new_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
+removed_pkgs <- get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
+# filter windows packages
+# filter new packages from the last X days
+cran_pkgs <- unique(tools::CRAN_package_db() |>
+  filter(`OS_type` != "windows" | is.na(`OS_type`)) |>
+  filter(`Package` %nin% new_packages) |>
+  pull(Package))
+
+arch <- "amd64"
+platform <- "redhat-8"
+platform <- "redhat-9"
+platform <- "alpine-320"
+platform <- "ubuntu-2204"
+platform <- "ubuntu-2404"
+
+data <- dbGetQuery(con, "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", params = list(platform, arch))
+removed_pkgs = get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name
+pkgs_db <- unique(data$name)
+pkgs_db = setdiff(pkgs_db, removed_pkgs)
+
+pkgs <- setdiff(cran_pkgs, pkgs_db)
+formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900])))))
+formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
+
+cat(formatted_pkgs_one_line, "\n", sep = "")
+length(formatted_pkgs)
+
+clipr::write_clip(formatted_pkgs_one_line)

local/missing-cran-packages.R

@@ -12,7 +12,7 @@ files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/
 files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib")) # done
 files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib")) # done
 files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib")) # done
-files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib"))
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib")) # done
 
 files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib")) # done
 files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib")) # done
@@ -24,13 +24,14 @@ files_b=basename(files)
 files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
 
 cran_pkgs <- tools::CRAN_package_db() |> 
-  filter(`Date/Publication` <= "2024-11-21") |> 
+  filter(`Date/Publication` <= "2024-12-02") |> 
   pull(Package)
 
 pkgs = setdiff(cran_pkgs, files_b_split)
-formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:400])))))
+formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900])))))
 formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
 
 cat(formatted_pkgs_one_line, "\n", sep = "")
 
 clipr::write_clip(formatted_pkgs_one_line)
+

local/packages-without-any-binary.R

@@ -7,16 +7,25 @@ con <- DBI::dbConnect(RPostgres::Postgres(),
   sslmode = "require"
 )
 
-data <- dbGetQuery(con, "SELECT name,platform,arch FROM single_builds group by name,platform,arch HAVING COUNT(*) = SUM(CASE WHEN error_occurred = 'TRUE' AND removed = 'FALSE' THEN 1 ELSE 0 END);")
+cran_pkgs <- tools::CRAN_package_db() |> 
+  filter(`Date/Publication` <= "2024-12-02") |> 
+  pull(Package)
+
+data <- dbGetQuery(con, "SELECT name,platform,arch,removed,error_occurred FROM single_builds;")
 
 pkgs <- data |>
-  filter(platform == "ubuntu-2204", arch == "arm64") |>
+  filter(platform == "redhat-8", arch == "arm64") |>
+  filter(removed == FALSE) |>
+  filter(error_occurred == FALSE) |>
+  distinct(name) |>
   pull(name)
 
+pkgs = setdiff(cran_pkgs, pkgs)
 # Format, remove line breaks, and print as a single line
-formatted_pkgs <- gsub('"', "'", capture.output(dput(pkgs[1:300])))
+formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:300])))))
 formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
 
 cat(formatted_pkgs_one_line, "\n", sep = "")
+length(pkgs)
 
 clipr::write_clip(formatted_pkgs_one_line)