From 2588b2512f9f2ed9a2b988a89cd356a915c7d25f Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 25 Nov 2024 10:04:23 +0100 Subject: [PATCH] update local/ scripts --- local/archive-missed-pkgs.R | 5 +- local/check-NA.R | 45 ++++++++ local/detect-duplicates.R | 18 +++ ...m-cran.R => label-removed-cran-packages.R} | 4 + local/last-processed-by-platform.R | 3 +- local/missing-cran-packages.R | 36 ++++++ local/packages-without-any-binary.R | 22 ++++ local/packages-without-binaries.R | 103 ------------------ 8 files changed, 130 insertions(+), 106 deletions(-) create mode 100644 local/check-NA.R create mode 100644 local/detect-duplicates.R rename local/{removed-from-cran.R => label-removed-cran-packages.R} (71%) create mode 100644 local/missing-cran-packages.R create mode 100644 local/packages-without-any-binary.R delete mode 100644 local/packages-without-binaries.R diff --git a/local/archive-missed-pkgs.R b/local/archive-missed-pkgs.R index 1d8706e..129e0b5 100644 --- a/local/archive-missed-pkgs.R +++ b/local/archive-missed-pkgs.R @@ -1,3 +1,4 @@ +### Scope: Archive packages that have more than one entry in repository root library(bincraftR) s3fs::s3_file_system( aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"), @@ -6,8 +7,8 @@ s3fs::s3_file_system( region_name = "eu-central-003", ) -arch = "arm64" -os = "jammy" +arch = "amd64" +os = "noble" files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/%s/%s/latest/src/contrib", arch, os)) length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1]))) diff --git a/local/check-NA.R b/local/check-NA.R new file mode 100644 index 0000000..7405db4 --- /dev/null +++ b/local/check-NA.R @@ -0,0 +1,45 @@ + +### Scope: Check whether there are any incomplete entries in PACKAGES.rds. These will result in NA when calling available.packages() +foo = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib") +sum((is.na(foo[, "Version"]))) +which((is.na(foo[, "Version"]))) + +foo = available.packages("https://cran.devxy.io/amd64/noble/latest/src/contrib") +sum((is.na(foo[, "Version"]))) +which((is.na(foo[, "Version"]))) + + +foo = available.packages("https://cran.devxy.io/amd64/rhel8/latest/src/contrib") +sum((is.na(foo[, "Version"]))) +which((is.na(foo[, "Version"]))) + +foo = available.packages("https://cran.devxy.io/amd64/rhel9/latest/src/contrib") +sum((is.na(foo[, "Version"]))) +which((is.na(foo[, "Version"]))) + +foo = available.packages("https://cran.devxy.io/amd64/alpine320/latest/src/contrib") +sum((is.na(foo[, "Version"]))) +which((is.na(foo[, "Version"]))) + +# arm64 + +foo = available.packages("https://cran.devxy.io/arm64/jammy/latest/src/contrib") +sum((is.na(foo[, "Version"]))) +which((is.na(foo[, "Version"]))) + +foo = available.packages("https://cran.devxy.io/arm64/noble/latest/src/contrib") +sum((is.na(foo[, "Version"]))) +which((is.na(foo[, "Version"]))) + + +foo = available.packages("https://cran.devxy.io/arm64/rhel8/latest/src/contrib") +sum((is.na(foo[, "Version"]))) +which((is.na(foo[, "Version"]))) + +foo = available.packages("https://cran.devxy.io/arm64/rhel9/latest/src/contrib") +sum((is.na(foo[, "Version"]))) +which((is.na(foo[, "Version"]))) + +foo = available.packages("https://cran.devxy.io/arm64/alpine320/latest/src/contrib") +sum((is.na(foo[, "Version"]))) +which((is.na(foo[, "Version"]))) diff --git a/local/detect-duplicates.R b/local/detect-duplicates.R new file mode 100644 index 0000000..5c57dbc --- /dev/null +++ b/local/detect-duplicates.R @@ -0,0 +1,18 @@ +### Scope: Shows if there are duplicated packages at the S3 repository root (e.g. due to missed/aborted archiving) +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib")) + +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel8/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/alpine320/latest/src/contrib")) + +files_b=basename(files) +files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) +files_b_split[duplicated(files_b_split)] + +archive_package("Rglpk", codename = "alpine320", arch = "arm64") \ No newline at end of file diff --git a/local/removed-from-cran.R b/local/label-removed-cran-packages.R similarity index 71% rename from local/removed-from-cran.R rename to local/label-removed-cran-packages.R index 8e57581..445f23d 100644 --- a/local/removed-from-cran.R +++ b/local/label-removed-cran-packages.R @@ -1,7 +1,9 @@ +### Scope: List decomisisoned CRAN packages and updates them in the metadata DB archived <- quickcode::archivedPkg() |> filter(arch.status == "decom") |> pull(name) +# Query all distinct pkgs in the DB con <- DBI::dbConnect(RPostgres::Postgres(), dbname = "build_metadata", host = "r-binaries.devxy.io", port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), @@ -12,8 +14,10 @@ pkgs_db <- query_metadata_table() |> distinct(name) |> pull(name) +# Check which pkgs in the DB match with the decomissioned ones to_process <- pkgs_db[pkgs_db %in% archived] +# for all matches, set 'removed = TRUE' sapply(to_process, function(.x) { DBI::dbExecute(con, "UPDATE single_builds SET removed = 'TRUE' where name = $1", params = list(.x) diff --git a/local/last-processed-by-platform.R b/local/last-processed-by-platform.R index 7e1f9a2..1a0b2b1 100644 --- a/local/last-processed-by-platform.R +++ b/local/last-processed-by-platform.R @@ -1,5 +1,6 @@ -devtools::load_all() +### Scope: Show last processed pkgs and timestamp by platform/OS library(dplyr) +library(bincraftR) ### Show last processed package by each platform/arch query_metadata_table() |> diff --git a/local/missing-cran-packages.R b/local/missing-cran-packages.R new file mode 100644 index 0000000..ad4b69e --- /dev/null +++ b/local/missing-cran-packages.R @@ -0,0 +1,36 @@ +### Scope: Lists packages which are missing any tarball in S3 compared to the CRAN pkg listing +library(bincraftR) +library(dplyr) +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"), + aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", +) + +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib")) + +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel8/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/alpine320/latest/src/contrib")) + +files_b=basename(files) +files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) + +cran_pkgs <- tools::CRAN_package_db() |> + filter(`Date/Publication` <= "2024-11-21") |> + pull(Package) + +pkgs = setdiff(cran_pkgs, files_b_split) +formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:400]))))) +formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") + +cat(formatted_pkgs_one_line, "\n", sep = "") + +clipr::write_clip(formatted_pkgs_one_line) diff --git a/local/packages-without-any-binary.R b/local/packages-without-any-binary.R new file mode 100644 index 0000000..88a95f4 --- /dev/null +++ b/local/packages-without-any-binary.R @@ -0,0 +1,22 @@ +### Lists packages without any successful binaries, i.e. pkgs for which all builds errored (per platform & arch) +library(DBI) +library(dplyr) +con <- DBI::dbConnect(RPostgres::Postgres(), + dbname = "build_metadata", host = "r-binaries.devxy.io", + port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), + sslmode = "require" +) + +data <- dbGetQuery(con, "SELECT name,platform,arch FROM single_builds group by name,platform,arch HAVING COUNT(*) = SUM(CASE WHEN error_occurred = 'TRUE' AND removed = 'FALSE' THEN 1 ELSE 0 END);") + +pkgs <- data |> + filter(platform == "ubuntu-2204", arch == "arm64") |> + pull(name) + +# Format, remove line breaks, and print as a single line +formatted_pkgs <- gsub('"', "'", capture.output(dput(pkgs[1:300]))) +formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") + +cat(formatted_pkgs_one_line, "\n", sep = "") + +clipr::write_clip(formatted_pkgs_one_line) diff --git a/local/packages-without-binaries.R b/local/packages-without-binaries.R deleted file mode 100644 index 8709056..0000000 --- a/local/packages-without-binaries.R +++ /dev/null @@ -1,103 +0,0 @@ -### -# Scope: List Packages for which the most recent CRAN release is not available -### -library(furrr) -library(cli) -library(dplyr) -library(data.table) - -last_versions_cran <- available.packages("https://cloud.r-project.org/src/contrib")[, c("Package", "Version")] - -arch <- c("amd64", "arm64") -os_ids <- c("rhel8", "rhel9", "noble", "jammy", "alpine320") -grid <- tidyr::expand_grid(arch = arch, os_ids = os_ids) - -# Define the function to check for missing packages -check_package <- function(pkg_name, available_pkgs, version_cran, arch, os_id) { - if (pkg_name %in% available_pkgs$Package) { - version_devxy <- available_pkgs %>% - filter(Package == pkg_name) %>% - pull(Version) - - if (is.na(version_devxy) || version_cran != version_devxy) { - # Package is missing or version mismatch - return(data.frame(pkg_name = pkg_name, arch = arch, ID = os_id)) - } - } else { - # Package completely missing in devxy - return(data.frame(pkg_name = pkg_name, arch = arch, ID = os_id)) - } - return(NULL) # No mismatch, no entry added -} - -# Run the mapping with parallel processing for each (arch, os_id) combination -future::plan("multisession", workers = 5) - - -pkgs_missing <- future_map2(grid$arch, grid$os_ids, ~ { - # Prepare the URL and fetch available packages only once per (arch, os_id) - repo_url <- sprintf("https://cran.devxy.io/%s/%s/latest/src/contrib", .x, .y) - available_pkgs <- as.data.table(available.packages(contriburl = repo_url)[, c("Package", "Version")]) - - # Define the packages to check once - cran_pkgs <- as.data.table(last_versions_cran) - - # Accumulate results in a list - pkg_entries <- list() - - # Loop through each package name and check if it's missing - for (pkg_name in cran_pkgs$Package) { - version_cran <- cran_pkgs[Package == pkg_name, Version] - result <- check_package(pkg_name, available_pkgs, version_cran, .x, .y) - if (!is.null(result)) { - pkg_entries[[length(pkg_entries) + 1]] <- result - } - } - - # Combine results for the current (arch, os_id) - if (length(pkg_entries) > 0) { - return(rbindlist(pkg_entries)) - } else { - return(data.table(pkg_name = character(), arch = character(), ID = character())) - } -}) - -df_missing_pkgs <- rbindlist(pkgs_missing, fill = TRUE) - -# Group and count missing packages by architecture and ID -df_summary <- df_missing_pkgs %>% - group_by(arch, ID) %>% - arrange(arch, ID) %>% - count() - -# Extract and copy missing package names for each architecture and ID combination -extract_and_copy_pkgs <- function(df, arch, range = 300, id, clip = FALSE) { - pkgs <- df %>% - filter(arch == arch, ID == id) %>% - pull(pkg_name) - - # Format, remove line breaks, and print as a single line - formatted_pkgs <- gsub('"', "'", capture.output(dput(pkgs[1:range]))) - formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") - - cat(formatted_pkgs_one_line, "\n", sep = "") - - if (clip) { - clipr::write_clip(formatted_pkgs_one_line) - } - - return(pkgs) -} - -# Usage for different architecture and ID combinations -pkgs_missing_amd64_rhel8 <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "rhel8", range = 20, clip = TRUE) -pkgs_missing_amd64_rhel9 <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "rhel9", clip = TRUE) -pkgs_missing_amd64_jammy <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "jammy", clip = TRUE) -pkgs_missing_amd64_noble <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "noble", clip = TRUE) -pkgs_missing_amd64_alpine320 <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "alpine320", clip = TRUE) - -pkgs_missing_arm64_rhel8 <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "rhel8", clip = TRUE) -pkgs_missing_arm64_rhel9 <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "rhel9", clip = TRUE) -pkgs_missing_arm64_jammy <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "jammy", clip = TRUE) -pkgs_missing_arm64_noble <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "noble", clip = TRUE) -pkgs_missing_arm64_alpine320 <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "alpine320", clip = TRUE)