From 7e0f3d0ac614c7d6cfdb9b3c5e18ddf6523f34d8 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 2 Sep 2026 09:13:54 +0000 Subject: [PATCH] fix(build): judge each index by its own source fallbacks (#193) ## Why #192 made the per-minor objects visible to their pass, and that part works: every slot now reports a populated per-minor cache where it previously reported `0 of 119053`. But the cache-driven skip rate stayed near **zero**, and the builds kept recompiling packages that already exist. Sampling what pipeline 12027 called missing, against the live CDN: ``` orthoDr_0.6.7.tar.gz 4.4/=404 flat=404 <- genuinely missing osmdata_0.2.4.tar.gz 4.4/=200 flat=404 <- already built, rebuilt anyway osmdata_0.2.5.tar.gz 4.4/=200 flat=404 osmdata_0.3.0.tar.gz 4.4/=200 flat=404 osqp_0.6.3.3.tar.gz 4.4/=200 flat=404 outbreaker2_1.1.0.tar.gz 4.4/=200 flat=404 ``` `source_served` is read from the **flat** slot's `PACKAGES.gz` and lists the packages that index serves as CRAN source instead of a binary. It was applied to every path in the listing: ```r existence_cache <- relative_paths[!basename(relative_paths) %in% source_basenames] ``` A package the flat slot serves as source still has a real `4.4/.tar.gz`, so dropping that basename deleted a genuine binary from the cache. The overlap with the work list is total rather than partial, which is why the skip rate was ~0% and not merely low: the sensitive candidate list is exactly "sensitive packages with no flat binary", which is the same set this was removing. amd64/resolute listed 21212 per-minor objects and cached only 13572; the missing ~7600 are these. ## What changed - Read each per-minor `PACKAGES.gz` and exclude only that minor's own source fallbacks. - Keep judging flat-slot objects by the flat index. - Log the fallback count per index, so a future collapse is visible rather than silent. All per-minor indexes were confirmed reachable (`4.4`, `4.5`, `4.6` return 200 on the slots checked); an unreadable index falls back to keeping that scope's cache in full, which is the safe direction. ## Verification ``` 4.4 pass sees -> OLD: osqp_0.6.3.3 NEW: osmdata_0.2.4, osqp_0.6.3.3 all assertions PASS ``` Correct in both directions: the old code dropped `4.4/osmdata` (a real binary) *and* kept `4.6/broken` (a real source fallback in the 4.6 index). The new code restores the first and still excludes the second. `prek run -a` passes. Pipelines 12025-12029 were stopped rather than left recompiling what they already have; their uploads persist, so a fresh run inherits them. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/193 --- local/packages-to-build.R | 106 +++++++++++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 23 deletions(-) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 9e17f4d..0654a43 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -154,26 +154,30 @@ index_url <- sprintf( arch, codename ) -source_served <- tryCatch( - { - con_idx <- gzcon(url(index_url, open = "rb")) - on.exit(close(con_idx), add = TRUE) - idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) - sprintf( - "%s_%s.tar.gz", - idx[is.na(idx[, "Built"]), "Package"], - idx[is.na(idx[, "Built"]), "Version"] - ) - }, - error = function(e) { - cat(sprintf( - "WARNING: could not read %s (%s); keeping the full S3 cache\n", - index_url, - conditionMessage(e) - )) - character(0) - } -) +read_source_served <- function(index) { + tryCatch( + { + con_idx <- gzcon(url(index, open = "rb")) + on.exit(close(con_idx), add = TRUE) + idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) + sprintf( + "%s_%s.tar.gz", + idx[is.na(idx[, "Built"]), "Package"], + idx[is.na(idx[, "Built"]), "Version"] + ) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read %s (%s); keeping that index's cache in full\n", + index, + conditionMessage(e) + )) + character(0) + } + ) +} + +source_served <- read_source_served(index_url) binary_cache <- setdiff(file_names, source_served) @@ -225,9 +229,65 @@ if (sum(stripped_per_minor) != sum(per_minor_object)) { )) } source_basenames <- source_served -existence_cache <- relative_paths[ - !basename(relative_paths) %in% source_basenames -] + +# A source-fallback list describes ONE index, so it may only be applied to that +# index's objects. Applied to every path it deletes real per-minor binaries from +# the cache: a package the flat slot serves as CRAN source still has a genuine +# `4.4/.tar.gz`, and dropping that name makes every per-minor pass rebuild +# it, every run, forever. +# +# The overlap is total rather than partial, which is why this pinned the skip +# rate near 0%: the sensitive candidate list is exactly "sensitive packages with +# no flat binary", which is the same set this was removing. amd64/resolute +# listed 21212 per-minor objects but cached only 13572, then recompiled osmdata, +# osqp and outbreaker2 while their 4.4 binaries sat in the bucket. +# +# So judge each minor by its own index, and the flat slot by the flat index. +is_per_minor_path <- grepl("^[0-9]+\\.[0-9]+/", relative_paths) +path_minor <- ifelse( + is_per_minor_path, + sub("^([0-9]+\\.[0-9]+)/.*$", "\\1", relative_paths), + "" +) +minors_present <- sort(unique(path_minor[is_per_minor_path])) +per_minor_source <- lapply(minors_present, function(m) { + read_source_served(sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/%s/PACKAGES.gz", + arch, + codename, + m + )) +}) +names(per_minor_source) <- minors_present +cat(sprintf( + "Source fallbacks per index: flat=%d%s\n", + length(source_basenames), + if (length(minors_present)) { + paste0( + ", ", + paste( + sprintf("%s=%d", minors_present, lengths(per_minor_source)), + collapse = ", " + ) + ) + } else { + "" + } +)) + +served_as_source <- vapply( + seq_along(relative_paths), + function(i) { + scope <- if (nzchar(path_minor[i])) { + per_minor_source[[path_minor[i]]] + } else { + source_basenames + } + basename(relative_paths[i]) %in% scope + }, + logical(1L) +) +existence_cache <- relative_paths[!served_as_source] cat(sprintf( "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", length(file_names),