From a78d489aac93b1f73f3ca43908163d21fbaa05d1 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 13 Dec 2024 14:34:53 +0100 Subject: [PATCH] chore: update local/ files --- local/check-cdn-cache.sh | 3 +-- local/debug-packages-writing.R | 14 ++++++++++ local/detect-duplicates.R | 3 ++- local/label-removed-cran-packages.R | 3 +++ local/missing-cran-packages-db.R | 42 +++++++++++++++++++++++++++++ local/missing-cran-packages.R | 7 ++--- local/packages-without-any-binary.R | 15 ++++++++--- 7 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 local/debug-packages-writing.R create mode 100644 local/missing-cran-packages-db.R diff --git a/local/check-cdn-cache.sh b/local/check-cdn-cache.sh index 73a813a..e56e3e2 100644 --- a/local/check-cdn-cache.sh +++ b/local/check-cdn-cache.sh @@ -6,7 +6,6 @@ curl -s https://cran.devxy.io/arm64/jammy/latest/src/contrib/PACKAGES | grep '^P curl -s https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.gz | gzip -cd | grep '^Package:' | wc -l curl -s https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l - curl -s https://cran.devxy.io/arm64/rhel8/latest/src/contrib/PACKAGES.gz | gzip -cd | grep '^Package:' | wc -l curl -s https://cran.devxy.io/arm64/rhel8/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l @@ -31,4 +30,4 @@ curl -s https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.gz | gzip curl -s https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l curl -s https://cran.devxy.io/amd64/alpine320/latest/src/contrib/PACKAGES.gz | gzip -cd | grep '^Package:' | wc -l -curl -s https://cran.devxy.io/amd64/alpine320/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l \ No newline at end of file +curl -s https://cran.devxy.io/amd64/alpine320/latest/src/contrib/PACKAGES | grep '^Package:' | wc -l diff --git a/local/debug-packages-writing.R b/local/debug-packages-writing.R new file mode 100644 index 0000000..d1a0789 --- /dev/null +++ b/local/debug-packages-writing.R @@ -0,0 +1,14 @@ +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib")) + +curl::curl_download("https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db") +con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db") +df = DBI::dbReadTable(con, "packages") + +cranlike:::write_packages_files("/tmp", "PACKAGES.db") + +# check +which(grepl("digest", df$Package)) +which(grepl("digest", df$Package)) + +system("cat /tmp/PACKAGES | grep '^Package: digest' | wc -l") + diff --git a/local/detect-duplicates.R b/local/detect-duplicates.R index 5c57dbc..0024c15 100644 --- a/local/detect-duplicates.R +++ b/local/detect-duplicates.R @@ -15,4 +15,5 @@ files_b=basename(files) files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) files_b_split[duplicated(files_b_split)] -archive_package("Rglpk", codename = "alpine320", arch = "arm64") \ No newline at end of file +archive_package("Rglpk", codename = "alpine320", arch = "arm64") + diff --git a/local/label-removed-cran-packages.R b/local/label-removed-cran-packages.R index 445f23d..6fc9f68 100644 --- a/local/label-removed-cran-packages.R +++ b/local/label-removed-cran-packages.R @@ -1,4 +1,6 @@ ### Scope: List decomisisoned CRAN packages and updates them in the metadata DB +library(dplyr) +library(bincraftR) archived <- quickcode::archivedPkg() |> filter(arch.status == "decom") |> pull(name) @@ -11,6 +13,7 @@ con <- DBI::dbConnect(RPostgres::Postgres(), ) pkgs_db <- query_metadata_table() |> + filter(removed == FALSE) |> distinct(name) |> pull(name) diff --git a/local/missing-cran-packages-db.R b/local/missing-cran-packages-db.R new file mode 100644 index 0000000..cda7d64 --- /dev/null +++ b/local/missing-cran-packages-db.R @@ -0,0 +1,42 @@ +### Scope: Lists packages which are missing any DB entry compared to what is available on CRAN +library(bincraftR) +library(dplyr) +library(DBI) + +con <- DBI::dbConnect(RPostgres::Postgres(), + dbname = "build_metadata", host = "r-binaries.devxy.io", + port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), + sslmode = "require" +) + +`%nin%` <- Negate(`%in%`) + +new_packages <- get_new_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name +removed_pkgs <- get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name +# filter windows packages +# filter new packages from the last X days +cran_pkgs <- unique(tools::CRAN_package_db() |> + filter(`OS_type` != "windows" | is.na(`OS_type`)) |> + filter(`Package` %nin% new_packages) |> + pull(Package)) + +arch <- "amd64" +platform <- "redhat-8" +platform <- "redhat-9" +platform <- "alpine-320" +platform <- "ubuntu-2204" +platform <- "ubuntu-2404" + +data <- dbGetQuery(con, "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", params = list(platform, arch)) +removed_pkgs = get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name +pkgs_db <- unique(data$name) +pkgs_db = setdiff(pkgs_db, removed_pkgs) + +pkgs <- setdiff(cran_pkgs, pkgs_db) +formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900]))))) +formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") + +cat(formatted_pkgs_one_line, "\n", sep = "") +length(formatted_pkgs) + +clipr::write_clip(formatted_pkgs_one_line) diff --git a/local/missing-cran-packages.R b/local/missing-cran-packages.R index ad4b69e..17d7baa 100644 --- a/local/missing-cran-packages.R +++ b/local/missing-cran-packages.R @@ -12,7 +12,7 @@ files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/ files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib")) # done files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib")) # done files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib")) # done files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib")) # done files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib")) # done @@ -24,13 +24,14 @@ files_b=basename(files) files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) cran_pkgs <- tools::CRAN_package_db() |> - filter(`Date/Publication` <= "2024-11-21") |> + filter(`Date/Publication` <= "2024-12-02") |> pull(Package) pkgs = setdiff(cran_pkgs, files_b_split) -formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:400]))))) +formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900]))))) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") cat(formatted_pkgs_one_line, "\n", sep = "") clipr::write_clip(formatted_pkgs_one_line) + diff --git a/local/packages-without-any-binary.R b/local/packages-without-any-binary.R index 88a95f4..af1cad3 100644 --- a/local/packages-without-any-binary.R +++ b/local/packages-without-any-binary.R @@ -7,16 +7,25 @@ con <- DBI::dbConnect(RPostgres::Postgres(), sslmode = "require" ) -data <- dbGetQuery(con, "SELECT name,platform,arch FROM single_builds group by name,platform,arch HAVING COUNT(*) = SUM(CASE WHEN error_occurred = 'TRUE' AND removed = 'FALSE' THEN 1 ELSE 0 END);") +cran_pkgs <- tools::CRAN_package_db() |> + filter(`Date/Publication` <= "2024-12-02") |> + pull(Package) + +data <- dbGetQuery(con, "SELECT name,platform,arch,removed,error_occurred FROM single_builds;") pkgs <- data |> - filter(platform == "ubuntu-2204", arch == "arm64") |> + filter(platform == "redhat-8", arch == "arm64") |> + filter(removed == FALSE) |> + filter(error_occurred == FALSE) |> + distinct(name) |> pull(name) +pkgs = setdiff(cran_pkgs, pkgs) # Format, remove line breaks, and print as a single line -formatted_pkgs <- gsub('"', "'", capture.output(dput(pkgs[1:300]))) +formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:300]))))) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") cat(formatted_pkgs_one_line, "\n", sep = "") +length(pkgs) clipr::write_clip(formatted_pkgs_one_line)