update local/ scripts

local/archive-missed-pkgs.R

@@ -1,3 +1,4 @@
+### Scope: Archive packages that have more than one entry in repository root
 library(bincraftR)
 s3fs::s3_file_system(
   aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
@@ -6,8 +7,8 @@ s3fs::s3_file_system(
   region_name = "eu-central-003",
 )
 
-arch = "arm64"
+arch = "amd64"
-os = "jammy"
+os = "noble"
 
 files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/%s/%s/latest/src/contrib", arch, os))
 length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1])))

local/check-NA.R

@@ -0,0 +1,45 @@
+
+### Scope: Check whether there are any incomplete entries in PACKAGES.rds. These will result in NA when calling available.packages()
+foo = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib")
+sum((is.na(foo[, "Version"])))
+which((is.na(foo[, "Version"])))
+
+foo = available.packages("https://cran.devxy.io/amd64/noble/latest/src/contrib")
+sum((is.na(foo[, "Version"])))
+which((is.na(foo[, "Version"])))
+
+
+foo = available.packages("https://cran.devxy.io/amd64/rhel8/latest/src/contrib")
+sum((is.na(foo[, "Version"])))
+which((is.na(foo[, "Version"])))
+
+foo = available.packages("https://cran.devxy.io/amd64/rhel9/latest/src/contrib")
+sum((is.na(foo[, "Version"])))
+which((is.na(foo[, "Version"])))
+
+foo = available.packages("https://cran.devxy.io/amd64/alpine320/latest/src/contrib")
+sum((is.na(foo[, "Version"])))
+which((is.na(foo[, "Version"])))
+
+# arm64
+
+foo = available.packages("https://cran.devxy.io/arm64/jammy/latest/src/contrib")
+sum((is.na(foo[, "Version"])))
+which((is.na(foo[, "Version"])))
+
+foo = available.packages("https://cran.devxy.io/arm64/noble/latest/src/contrib")
+sum((is.na(foo[, "Version"])))
+which((is.na(foo[, "Version"])))
+
+
+foo = available.packages("https://cran.devxy.io/arm64/rhel8/latest/src/contrib")
+sum((is.na(foo[, "Version"])))
+which((is.na(foo[, "Version"])))
+
+foo = available.packages("https://cran.devxy.io/arm64/rhel9/latest/src/contrib")
+sum((is.na(foo[, "Version"])))
+which((is.na(foo[, "Version"])))
+
+foo = available.packages("https://cran.devxy.io/arm64/alpine320/latest/src/contrib")
+sum((is.na(foo[, "Version"])))
+which((is.na(foo[, "Version"])))

local/detect-duplicates.R

@@ -0,0 +1,18 @@
+### Scope: Shows if there are duplicated packages at the S3 repository root (e.g. due to missed/aborted archiving)
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/contrib"))
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib"))
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib"))
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib"))
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib"))
+
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib"))
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib"))
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel8/latest/src/contrib"))
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib"))
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/alpine320/latest/src/contrib"))
+
+files_b=basename(files)
+files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
+files_b_split[duplicated(files_b_split)]
+
+archive_package("Rglpk", codename = "alpine320", arch = "arm64")

local/label-removed-cran-packages.R

@@ -0,0 +1,25 @@
+### Scope: List decomisisoned CRAN packages and updates them in the metadata DB
+archived <- quickcode::archivedPkg() |>
+  filter(arch.status == "decom") |>
+  pull(name)
+
+# Query all distinct pkgs in the DB
+con <- DBI::dbConnect(RPostgres::Postgres(),
+  dbname = "build_metadata", host = "r-binaries.devxy.io",
+  port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
+  sslmode = "require"
+)
+
+pkgs_db <- query_metadata_table() |>
+  distinct(name) |>
+  pull(name)
+
+# Check which pkgs in the DB match with the decomissioned ones
+to_process <- pkgs_db[pkgs_db %in% archived]
+
+# for all matches, set 'removed = TRUE'
+sapply(to_process, function(.x) {
+  DBI::dbExecute(con, "UPDATE single_builds SET removed = 'TRUE' where name = $1",
+    params = list(.x)
+  )
+})

local/last-processed-by-platform.R

@@ -1,5 +1,6 @@
-devtools::load_all()
+### Scope: Show last processed pkgs and timestamp by platform/OS
 library(dplyr)
+library(bincraftR)
 
 ### Show last processed package by each platform/arch
 query_metadata_table() |>

local/missing-cran-packages.R

@@ -0,0 +1,36 @@
+### Scope: Lists packages which are missing any tarball in S3 compared to the CRAN pkg listing
+library(bincraftR)
+library(dplyr)
+s3fs::s3_file_system(
+  aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
+  aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
+  endpoint = "https://s3.eu-central-003.backblazeb2.com",
+  region_name = "eu-central-003",
+)
+
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/contrib")) # done
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib")) # done
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib")) # done
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib")) # done
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib"))
+
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib")) # done
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib")) # done
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel8/latest/src/contrib")) # done
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib")) # done
+files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/alpine320/latest/src/contrib"))
+
+files_b=basename(files)
+files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
+
+cran_pkgs <- tools::CRAN_package_db() |> 
+  filter(`Date/Publication` <= "2024-11-21") |> 
+  pull(Package)
+
+pkgs = setdiff(cran_pkgs, files_b_split)
+formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:400])))))
+formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
+
+cat(formatted_pkgs_one_line, "\n", sep = "")
+
+clipr::write_clip(formatted_pkgs_one_line)

local/packages-without-any-binary.R

@@ -0,0 +1,22 @@
+### Lists packages without any successful binaries, i.e. pkgs for which all builds errored (per platform & arch)
+library(DBI)
+library(dplyr)
+con <- DBI::dbConnect(RPostgres::Postgres(),
+  dbname = "build_metadata", host = "r-binaries.devxy.io",
+  port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
+  sslmode = "require"
+)
+
+data <- dbGetQuery(con, "SELECT name,platform,arch FROM single_builds group by name,platform,arch HAVING COUNT(*) = SUM(CASE WHEN error_occurred = 'TRUE' AND removed = 'FALSE' THEN 1 ELSE 0 END);")
+
+pkgs <- data |>
+  filter(platform == "ubuntu-2204", arch == "arm64") |>
+  pull(name)
+
+# Format, remove line breaks, and print as a single line
+formatted_pkgs <- gsub('"', "'", capture.output(dput(pkgs[1:300])))
+formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
+
+cat(formatted_pkgs_one_line, "\n", sep = "")
+
+clipr::write_clip(formatted_pkgs_one_line)

local/packages-without-binaries.R

@@ -1,103 +0,0 @@
-###
-# Scope: List Packages for which the most recent CRAN release is not available
-###
-library(furrr)
-library(cli)
-library(dplyr)
-library(data.table)
-
-last_versions_cran <- available.packages("https://cloud.r-project.org/src/contrib")[, c("Package", "Version")]
-
-arch <- c("amd64", "arm64")
-os_ids <- c("rhel8", "rhel9", "noble", "jammy", "alpine320")
-grid <- tidyr::expand_grid(arch = arch, os_ids = os_ids)
-
-# Define the function to check for missing packages
-check_package <- function(pkg_name, available_pkgs, version_cran, arch, os_id) {
-  if (pkg_name %in% available_pkgs$Package) {
-    version_devxy <- available_pkgs %>%
-      filter(Package == pkg_name) %>%
-      pull(Version)
-
-    if (is.na(version_devxy) || version_cran != version_devxy) {
-      # Package is missing or version mismatch
-      return(data.frame(pkg_name = pkg_name, arch = arch, ID = os_id))
-    }
-  } else {
-    # Package completely missing in devxy
-    return(data.frame(pkg_name = pkg_name, arch = arch, ID = os_id))
-  }
-  return(NULL) # No mismatch, no entry added
-}
-
-# Run the mapping with parallel processing for each (arch, os_id) combination
-future::plan("multisession", workers = 5)
-
-
-pkgs_missing <- future_map2(grid$arch, grid$os_ids, ~ {
-  # Prepare the URL and fetch available packages only once per (arch, os_id)
-  repo_url <- sprintf("https://cran.devxy.io/%s/%s/latest/src/contrib", .x, .y)
-  available_pkgs <- as.data.table(available.packages(contriburl = repo_url)[, c("Package", "Version")])
-
-  # Define the packages to check once
-  cran_pkgs <- as.data.table(last_versions_cran)
-
-  # Accumulate results in a list
-  pkg_entries <- list()
-
-  # Loop through each package name and check if it's missing
-  for (pkg_name in cran_pkgs$Package) {
-    version_cran <- cran_pkgs[Package == pkg_name, Version]
-    result <- check_package(pkg_name, available_pkgs, version_cran, .x, .y)
-    if (!is.null(result)) {
-      pkg_entries[[length(pkg_entries) + 1]] <- result
-    }
-  }
-
-  # Combine results for the current (arch, os_id)
-  if (length(pkg_entries) > 0) {
-    return(rbindlist(pkg_entries))
-  } else {
-    return(data.table(pkg_name = character(), arch = character(), ID = character()))
-  }
-})
-
-df_missing_pkgs <- rbindlist(pkgs_missing, fill = TRUE)
-
-# Group and count missing packages by architecture and ID
-df_summary <- df_missing_pkgs %>%
-  group_by(arch, ID) %>%
-  arrange(arch, ID) %>%
-  count()
-
-# Extract and copy missing package names for each architecture and ID combination
-extract_and_copy_pkgs <- function(df, arch, range = 300, id, clip = FALSE) {
-  pkgs <- df %>%
-    filter(arch == arch, ID == id) %>%
-    pull(pkg_name)
-
-  # Format, remove line breaks, and print as a single line
-  formatted_pkgs <- gsub('"', "'", capture.output(dput(pkgs[1:range])))
-  formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
-  
-  cat(formatted_pkgs_one_line, "\n", sep = "")
-  
-  if (clip) {
-    clipr::write_clip(formatted_pkgs_one_line)
-  }
-
-  return(pkgs)
-}
-
-# Usage for different architecture and ID combinations
-pkgs_missing_amd64_rhel8 <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "rhel8", range = 20, clip = TRUE)
-pkgs_missing_amd64_rhel9 <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "rhel9", clip = TRUE)
-pkgs_missing_amd64_jammy <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "jammy", clip = TRUE)
-pkgs_missing_amd64_noble <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "noble", clip = TRUE)
-pkgs_missing_amd64_alpine320 <- extract_and_copy_pkgs(df_missing_pkgs, "amd64", "alpine320", clip = TRUE)
-
-pkgs_missing_arm64_rhel8 <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "rhel8", clip = TRUE)
-pkgs_missing_arm64_rhel9 <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "rhel9", clip = TRUE)
-pkgs_missing_arm64_jammy <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "jammy", clip = TRUE)
-pkgs_missing_arm64_noble <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "noble", clip = TRUE)
-pkgs_missing_arm64_alpine320 <- extract_and_copy_pkgs(df_missing_pkgs, "arm64", "alpine320", clip = TRUE)