local/weekly-missing-binaries-audit.R

@@ -7,7 +7,6 @@ options(error = function() {
 suppressPackageStartupMessages(library(data.table))
 library(DBI, quietly = TRUE)
 library(RPostgres, quietly = TRUE)
-library(s3fs, quietly = TRUE)
 library(jsonlite, quietly = TRUE)
 library(httr2, quietly = TRUE)
 
@@ -50,8 +49,8 @@ cat(sprintf(
 ))
 
 # ---------------------------------------------------------------------------
-# 1. Query PostgreSQL for known build failures (before s3fs init to avoid
+# 1. Query PostgreSQL for known build failures (before anything that uses curl,
-#    C++ pointer conflicts between s3fs/curl and RPostgres/libpq)
+#    to avoid C++ pointer conflicts between curl and RPostgres/libpq)
 # ---------------------------------------------------------------------------
 cat("Connecting to PostgreSQL...\n")
 con <- DBI::dbConnect(
@@ -97,62 +96,71 @@ cran_dt <- data.table(
 )
 
 # ---------------------------------------------------------------------------
-# 3. S3 tarballs
+# 3. Published binaries
 # ---------------------------------------------------------------------------
-cat("Connecting to S3...\n")
+# Read the slot's own index rather than listing the bucket. An object being
-s3fs::s3_file_system(
+# present does not mean a binary was built: when a build fails, bincraft
-  aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
+# publishes the CRAN source tarball in its place so the package stays
-  aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
+# installable, and a bucket listing cannot tell the two apart. That is how
-  endpoint = "https://s3.eu-central-003.backblazeb2.com",
+# amd64/alpine324 came to hold 13,547 CRAN sources that this audit never
-  region_name = "eu-central-003",
+# reported. bincraft stamps `Built` only on records it actually built, so the
-  refresh = TRUE
+# index answers the question a listing cannot.
-)
+#
-
+# A slot last indexed by a bincraft that predates the source-fallback fix
-s3_path <- sprintf(
+# stamps `Built` on every record, including the fallbacks, so this reports
-  "devxy-rpkgs-binaries/%s/%s/latest/src/contrib",
+# exactly what it used to until that slot is re-indexed.
+index_url <- sprintf(
+  "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz",
   arch,
   s3_codename
 )
-cat(sprintf("Listing S3 path: %s\n", s3_path))
+cat(sprintf("Reading package index: %s\n", index_url))
 
-s3_pkgs <- tryCatch(
+index <- tryCatch(
-  s3fs::s3_dir_ls(s3_path, recurse = FALSE),
+  {
+    con <- gzcon(url(index_url, open = "rb"))
+    on.exit(close(con), add = TRUE)
+    read.dcf(con, fields = c("Package", "Version", "Built"))
+  },
   error = function(e) {
     cat(sprintf(
-      "WARNING: Could not list S3 path %s: %s\n",
+      "WARNING: Could not read %s: %s\n",
-      s3_path,
+      index_url,
       conditionMessage(e)
     ))
-    character(0)
+    NULL
   }
 )
 
-file_names <- basename(s3_pkgs)
+if (is.null(index) || nrow(index) == 0) {
-matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names)
+  binary_dt <- data.table(Package = character(0), Version = character(0))
-parts <- regmatches(file_names, matches)
-parts <- parts[sapply(parts, length) == 3]
-if (length(parts) == 0) {
-  s3_dt <- data.table(Package = character(0), Version = character(0))
 } else {
-  s3_dt <- data.table(
+  built <- !is.na(index[, "Built"])
-    Package = sapply(parts, `[`, 2),
+  binary_dt <- data.table(
-    Version = sapply(parts, `[`, 3)
+    Package = as.character(index[built, "Package"]),
+    Version = as.character(index[built, "Version"])
   )
+  cat(sprintf(
+    "Index holds %d records, %d of them built binaries (%d served as CRAN source)\n",
+    nrow(index),
+    sum(built),
+    sum(!built)
+  ))
 }
 
 cat(sprintf(
-  "S3 contains %d tarballs for %s/%s\n",
+  "S3 contains %d binaries for %s/%s\n",
-  nrow(s3_dt),
+  nrow(binary_dt),
   arch,
   s3_codename
 ))
 
 # ---------------------------------------------------------------------------
-# 4. Find missing packages (CRAN release version not in S3)
+# 4. Find missing packages (CRAN release version without a binary in S3)
 # ---------------------------------------------------------------------------
 setkey(cran_dt, Package, Version)
-setkey(s3_dt, Package, Version)
+setkey(binary_dt, Package, Version)
-missing_dt <- cran_dt[!s3_dt]
+missing_dt <- cran_dt[!binary_dt]
 cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt)))
 
 # ---------------------------------------------------------------------------