fix(audit): count a source fallback as a missing binary (#157)
Some checks failed
ci/crow/cron/process-updates/17 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/manual/weekly-audit-missing/1 Pipeline was successful
ci/crow/manual/weekly-audit-missing/2 Pipeline was successful
ci/crow/manual/weekly-audit-missing/3 Pipeline was successful
ci/crow/manual/weekly-audit-missing/4 Pipeline was successful
ci/crow/manual/weekly-audit-missing/7 Pipeline was successful
ci/crow/manual/weekly-audit-missing/8 Pipeline was successful
ci/crow/manual/weekly-audit-missing/9 Pipeline was successful
ci/crow/manual/weekly-audit-missing/11 Pipeline was successful
ci/crow/manual/weekly-audit-missing/12 Pipeline was successful
ci/crow/manual/weekly-audit-missing/13 Pipeline was successful
ci/crow/manual/weekly-audit-missing/16 Pipeline was successful
ci/crow/manual/weekly-audit-missing/17 Pipeline was successful
ci/crow/manual/weekly-audit-missing/15 Pipeline was successful
ci/crow/manual/weekly-audit-missing/10 Pipeline was successful
ci/crow/manual/weekly-audit-missing/18 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/manual/weekly-audit-missing/14 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/1 Pipeline was successful
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/manual/build-all-versions/2 Pipeline is pending
ci/crow/manual/build-all-versions/3 Pipeline is pending
ci/crow/manual/build-all-versions/4 Pipeline is pending
ci/crow/manual/process-updates/12 Pipeline is pending
ci/crow/manual/process-updates/14 Pipeline is pending
ci/crow/manual/process-updates/16 Pipeline is pending
ci/crow/manual/process-updates/18 Pipeline is pending
ci/crow/manual/repair-built-stamp/2 Pipeline is pending
ci/crow/manual/weekly-audit-missing/6 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/1 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/2 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/3 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/4 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/5 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/6 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/7 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/8 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/9 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/10 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/11 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/12 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/14 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/15 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/16 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/18 Pipeline is pending
ci/crow/manual/process-updates/10 Pipeline was canceled
ci/crow/manual/build-all-versions/1 Pipeline was canceled
ci/crow/manual/process-updates/17 Pipeline was canceled
ci/crow/manual/weekly-audit-missing/5 Pipeline was canceled
ci/crow/manual/process-updates/15 Pipeline was canceled
ci/crow/manual/process-updates/7 Pipeline failed
ci/crow/manual/process-updates/9 Pipeline failed
ci/crow/manual/process-updates/6 Pipeline failed
ci/crow/manual/process-updates/13 Pipeline failed
ci/crow/manual/process-updates/8 Pipeline failed
ci/crow/manual/process-updates/1 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/13 Pipeline failed
ci/crow/manual/build-all-versions-install-deps/1 Pipeline was canceled
ci/crow/manual/process-updates/11 Pipeline failed
ci/crow/manual/process-updates/3 Pipeline was canceled
ci/crow/manual/process-updates/5 Pipeline was canceled
ci/crow/manual/process-updates/4 Pipeline was canceled
ci/crow/manual/process-updates/2 Pipeline was canceled
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/manual/weekly-rebuild-missing/17 Pipeline was successful
Some checks failed
ci/crow/cron/process-updates/17 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/manual/weekly-audit-missing/1 Pipeline was successful
ci/crow/manual/weekly-audit-missing/2 Pipeline was successful
ci/crow/manual/weekly-audit-missing/3 Pipeline was successful
ci/crow/manual/weekly-audit-missing/4 Pipeline was successful
ci/crow/manual/weekly-audit-missing/7 Pipeline was successful
ci/crow/manual/weekly-audit-missing/8 Pipeline was successful
ci/crow/manual/weekly-audit-missing/9 Pipeline was successful
ci/crow/manual/weekly-audit-missing/11 Pipeline was successful
ci/crow/manual/weekly-audit-missing/12 Pipeline was successful
ci/crow/manual/weekly-audit-missing/13 Pipeline was successful
ci/crow/manual/weekly-audit-missing/16 Pipeline was successful
ci/crow/manual/weekly-audit-missing/17 Pipeline was successful
ci/crow/manual/weekly-audit-missing/15 Pipeline was successful
ci/crow/manual/weekly-audit-missing/10 Pipeline was successful
ci/crow/manual/weekly-audit-missing/18 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/manual/weekly-audit-missing/14 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/1 Pipeline was successful
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/manual/build-all-versions/2 Pipeline is pending
ci/crow/manual/build-all-versions/3 Pipeline is pending
ci/crow/manual/build-all-versions/4 Pipeline is pending
ci/crow/manual/process-updates/12 Pipeline is pending
ci/crow/manual/process-updates/14 Pipeline is pending
ci/crow/manual/process-updates/16 Pipeline is pending
ci/crow/manual/process-updates/18 Pipeline is pending
ci/crow/manual/repair-built-stamp/2 Pipeline is pending
ci/crow/manual/weekly-audit-missing/6 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/1 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/2 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/3 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/4 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/5 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/6 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/7 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/8 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/9 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/10 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/11 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/12 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/14 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/15 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/16 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/18 Pipeline is pending
ci/crow/manual/process-updates/10 Pipeline was canceled
ci/crow/manual/build-all-versions/1 Pipeline was canceled
ci/crow/manual/process-updates/17 Pipeline was canceled
ci/crow/manual/weekly-audit-missing/5 Pipeline was canceled
ci/crow/manual/process-updates/15 Pipeline was canceled
ci/crow/manual/process-updates/7 Pipeline failed
ci/crow/manual/process-updates/9 Pipeline failed
ci/crow/manual/process-updates/6 Pipeline failed
ci/crow/manual/process-updates/13 Pipeline failed
ci/crow/manual/process-updates/8 Pipeline failed
ci/crow/manual/process-updates/1 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/13 Pipeline failed
ci/crow/manual/build-all-versions-install-deps/1 Pipeline was canceled
ci/crow/manual/process-updates/11 Pipeline failed
ci/crow/manual/process-updates/3 Pipeline was canceled
ci/crow/manual/process-updates/5 Pipeline was canceled
ci/crow/manual/process-updates/4 Pipeline was canceled
ci/crow/manual/process-updates/2 Pipeline was canceled
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/manual/weekly-rebuild-missing/17 Pipeline was successful
## Problem The audit's definition of "missing" was `missing_dt <- cran_dt[!s3_dt]`, where `s3_dt` came from listing the bucket. But an object being present does not mean a binary was built: when a build fails, bincraft publishes the CRAN source tarball in its place so the package stays installable, and a listing cannot tell the two apart. So every failed build looked done. On `amd64/alpine324` that is 13 547 of 24 134 comparable objects (56%) — all CRAN sources from the June 2026 bootstrap, none ever reported here, so `weekly-rebuild-missing` never retried any of them. `amd64/noble` sits at 4.6% for comparison. ## What this changes Reads the slot's own `PACKAGES` index instead of listing the bucket, and counts only records carrying a `Built` stamp. bincraft writes that stamp only for what it actually built (rpkgs/bincraft#105), so the index answers the question a listing cannot. Side effects of reading the index rather than the bucket: - the audit reports how many records the index holds and how many are served as source, so the gap is visible in the log - `s3fs` is no longer used, so the audit needs no B2 credentials - an object present but absent from the index is now counted as missing, which is correct: clients only see the index ## Sequencing This is inert until the matching bincraft change ships and a slot is re-indexed. A slot last indexed by an older bincraft carries `Built` on every record, including the fallbacks, so the audit reports exactly what it does today. Verified against the live indices: ``` amd64/alpine324: records=24152 built=24152 source-served=0 amd64/noble: records=24681 built=24681 source-served=0 ``` After a re-index those `source-served` counts become the real ones, and the packages behind them start appearing in issue #63 and getting rebuilt. Reviewed-on: #157
This commit is contained in:
parent
0c86b2692d
commit
0703969088
1 changed files with 44 additions and 36 deletions
|
|
@ -7,7 +7,6 @@ options(error = function() {
|
|||
suppressPackageStartupMessages(library(data.table))
|
||||
library(DBI, quietly = TRUE)
|
||||
library(RPostgres, quietly = TRUE)
|
||||
library(s3fs, quietly = TRUE)
|
||||
library(jsonlite, quietly = TRUE)
|
||||
library(httr2, quietly = TRUE)
|
||||
|
||||
|
|
@ -50,8 +49,8 @@ cat(sprintf(
|
|||
))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Query PostgreSQL for known build failures (before s3fs init to avoid
|
||||
# C++ pointer conflicts between s3fs/curl and RPostgres/libpq)
|
||||
# 1. Query PostgreSQL for known build failures (before anything that uses curl,
|
||||
# to avoid C++ pointer conflicts between curl and RPostgres/libpq)
|
||||
# ---------------------------------------------------------------------------
|
||||
cat("Connecting to PostgreSQL...\n")
|
||||
con <- DBI::dbConnect(
|
||||
|
|
@ -97,62 +96,71 @@ cran_dt <- data.table(
|
|||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. S3 tarballs
|
||||
# 3. Published binaries
|
||||
# ---------------------------------------------------------------------------
|
||||
cat("Connecting to S3...\n")
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region_name = "eu-central-003",
|
||||
refresh = TRUE
|
||||
)
|
||||
|
||||
s3_path <- sprintf(
|
||||
"devxy-rpkgs-binaries/%s/%s/latest/src/contrib",
|
||||
# Read the slot's own index rather than listing the bucket. An object being
|
||||
# present does not mean a binary was built: when a build fails, bincraft
|
||||
# publishes the CRAN source tarball in its place so the package stays
|
||||
# installable, and a bucket listing cannot tell the two apart. That is how
|
||||
# amd64/alpine324 came to hold 13,547 CRAN sources that this audit never
|
||||
# reported. bincraft stamps `Built` only on records it actually built, so the
|
||||
# index answers the question a listing cannot.
|
||||
#
|
||||
# A slot last indexed by a bincraft that predates the source-fallback fix
|
||||
# stamps `Built` on every record, including the fallbacks, so this reports
|
||||
# exactly what it used to until that slot is re-indexed.
|
||||
index_url <- sprintf(
|
||||
"https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz",
|
||||
arch,
|
||||
s3_codename
|
||||
)
|
||||
cat(sprintf("Listing S3 path: %s\n", s3_path))
|
||||
cat(sprintf("Reading package index: %s\n", index_url))
|
||||
|
||||
s3_pkgs <- tryCatch(
|
||||
s3fs::s3_dir_ls(s3_path, recurse = FALSE),
|
||||
index <- tryCatch(
|
||||
{
|
||||
con <- gzcon(url(index_url, open = "rb"))
|
||||
on.exit(close(con), add = TRUE)
|
||||
read.dcf(con, fields = c("Package", "Version", "Built"))
|
||||
},
|
||||
error = function(e) {
|
||||
cat(sprintf(
|
||||
"WARNING: Could not list S3 path %s: %s\n",
|
||||
s3_path,
|
||||
"WARNING: Could not read %s: %s\n",
|
||||
index_url,
|
||||
conditionMessage(e)
|
||||
))
|
||||
character(0)
|
||||
NULL
|
||||
}
|
||||
)
|
||||
|
||||
file_names <- basename(s3_pkgs)
|
||||
matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names)
|
||||
parts <- regmatches(file_names, matches)
|
||||
parts <- parts[sapply(parts, length) == 3]
|
||||
if (length(parts) == 0) {
|
||||
s3_dt <- data.table(Package = character(0), Version = character(0))
|
||||
if (is.null(index) || nrow(index) == 0) {
|
||||
binary_dt <- data.table(Package = character(0), Version = character(0))
|
||||
} else {
|
||||
s3_dt <- data.table(
|
||||
Package = sapply(parts, `[`, 2),
|
||||
Version = sapply(parts, `[`, 3)
|
||||
built <- !is.na(index[, "Built"])
|
||||
binary_dt <- data.table(
|
||||
Package = as.character(index[built, "Package"]),
|
||||
Version = as.character(index[built, "Version"])
|
||||
)
|
||||
cat(sprintf(
|
||||
"Index holds %d records, %d of them built binaries (%d served as CRAN source)\n",
|
||||
nrow(index),
|
||||
sum(built),
|
||||
sum(!built)
|
||||
))
|
||||
}
|
||||
|
||||
cat(sprintf(
|
||||
"S3 contains %d tarballs for %s/%s\n",
|
||||
nrow(s3_dt),
|
||||
"S3 contains %d binaries for %s/%s\n",
|
||||
nrow(binary_dt),
|
||||
arch,
|
||||
s3_codename
|
||||
))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Find missing packages (CRAN release version not in S3)
|
||||
# 4. Find missing packages (CRAN release version without a binary in S3)
|
||||
# ---------------------------------------------------------------------------
|
||||
setkey(cran_dt, Package, Version)
|
||||
setkey(s3_dt, Package, Version)
|
||||
missing_dt <- cran_dt[!s3_dt]
|
||||
setkey(binary_dt, Package, Version)
|
||||
missing_dt <- cran_dt[!binary_dt]
|
||||
cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt)))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Reference in a new issue