From 5e58eb5c9137c80f922c28bde2f6428085cdd98d Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 9 Aug 2026 10:31:19 +0000 Subject: [PATCH] fix(audit): count a source fallback as a missing binary An object being present in the slot does not mean a binary was built: when a build fails, bincraft publishes the CRAN source tarball in its place, and the bucket listing this used could not tell the two apart. That is how amd64/alpine324 came to hold 13547 CRAN sources that this audit never reported and weekly-rebuild-missing therefore never retried. - read the slot's own PACKAGES index instead of listing the bucket, and count only records carrying a Built stamp, which bincraft writes only for what it actually built - report how many records the index holds and how many are served as source - drop the now-unused s3fs dependency, so the audit needs no B2 credentials A slot last indexed by a bincraft that predates the matching fix stamps Built on every record, so this reports exactly what it used to until re-indexed. --- local/weekly-missing-binaries-audit.R | 78 +++++++++++++++------------ 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index 7b0b65d..e7ca776 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -7,7 +7,6 @@ options(error = function() { suppressPackageStartupMessages(library(data.table)) library(DBI, quietly = TRUE) library(RPostgres, quietly = TRUE) -library(s3fs, quietly = TRUE) library(jsonlite, quietly = TRUE) library(httr2, quietly = TRUE) @@ -50,8 +49,8 @@ cat(sprintf( )) # --------------------------------------------------------------------------- -# 1. Query PostgreSQL for known build failures (before s3fs init to avoid -# C++ pointer conflicts between s3fs/curl and RPostgres/libpq) +# 1. Query PostgreSQL for known build failures (before anything that uses curl, +# to avoid C++ pointer conflicts between curl and RPostgres/libpq) # --------------------------------------------------------------------------- cat("Connecting to PostgreSQL...\n") con <- DBI::dbConnect( @@ -97,62 +96,71 @@ cran_dt <- data.table( ) # --------------------------------------------------------------------------- -# 3. S3 tarballs +# 3. Published binaries # --------------------------------------------------------------------------- -cat("Connecting to S3...\n") -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE -) - -s3_path <- sprintf( - "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", +# Read the slot's own index rather than listing the bucket. An object being +# present does not mean a binary was built: when a build fails, bincraft +# publishes the CRAN source tarball in its place so the package stays +# installable, and a bucket listing cannot tell the two apart. That is how +# amd64/alpine324 came to hold 13,547 CRAN sources that this audit never +# reported. bincraft stamps `Built` only on records it actually built, so the +# index answers the question a listing cannot. +# +# A slot last indexed by a bincraft that predates the source-fallback fix +# stamps `Built` on every record, including the fallbacks, so this reports +# exactly what it used to until that slot is re-indexed. +index_url <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", arch, s3_codename ) -cat(sprintf("Listing S3 path: %s\n", s3_path)) +cat(sprintf("Reading package index: %s\n", index_url)) -s3_pkgs <- tryCatch( - s3fs::s3_dir_ls(s3_path, recurse = FALSE), +index <- tryCatch( + { + con <- gzcon(url(index_url, open = "rb")) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "Built")) + }, error = function(e) { cat(sprintf( - "WARNING: Could not list S3 path %s: %s\n", - s3_path, + "WARNING: Could not read %s: %s\n", + index_url, conditionMessage(e) )) - character(0) + NULL } ) -file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) -parts <- parts[sapply(parts, length) == 3] -if (length(parts) == 0) { - s3_dt <- data.table(Package = character(0), Version = character(0)) +if (is.null(index) || nrow(index) == 0) { + binary_dt <- data.table(Package = character(0), Version = character(0)) } else { - s3_dt <- data.table( - Package = sapply(parts, `[`, 2), - Version = sapply(parts, `[`, 3) + built <- !is.na(index[, "Built"]) + binary_dt <- data.table( + Package = as.character(index[built, "Package"]), + Version = as.character(index[built, "Version"]) ) + cat(sprintf( + "Index holds %d records, %d of them built binaries (%d served as CRAN source)\n", + nrow(index), + sum(built), + sum(!built) + )) } cat(sprintf( - "S3 contains %d tarballs for %s/%s\n", - nrow(s3_dt), + "S3 contains %d binaries for %s/%s\n", + nrow(binary_dt), arch, s3_codename )) # --------------------------------------------------------------------------- -# 4. Find missing packages (CRAN release version not in S3) +# 4. Find missing packages (CRAN release version without a binary in S3) # --------------------------------------------------------------------------- setkey(cran_dt, Package, Version) -setkey(s3_dt, Package, Version) -missing_dt <- cran_dt[!s3_dt] +setkey(binary_dt, Package, Version) +missing_dt <- cran_dt[!binary_dt] cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) # --------------------------------------------------------------------------- -- 2.54.0