fix(build): judge each index by its own source fallbacks #193

Merged
pat-s merged 1 commit from fix/scope-source-fallback-per-minor into main 2026-09-02 09:13:55 +00:00
Showing only changes of commit 0940f004d4 - Show all commits

fix(build): judge each index by its own source fallbacks

The source-fallback list is read from the flat slot's PACKAGES.gz and names the
packages that index serves as CRAN source rather than as a binary. It was then
applied to every path in the listing, including per-minor ones.

A package the flat slot serves as source still has a genuine `4.4/<pkg>.tar.gz`,
so removing that basename deleted a real binary from the cache and made every
per-minor pass rebuild it, every run, forever.

The overlap with the work list is total, not partial: the sensitive candidate
list is exactly "sensitive packages with no flat binary", which is the same set
this removed. That pinned the cache-driven skip rate near zero even after #192
made the per-minor objects visible. amd64/resolute listed 21212 per-minor
objects but cached only 13572, then recompiled osmdata, osqp and outbreaker2
while their 4.4 binaries sat in the bucket returning HTTP 200.

This change will:

- read each per-minor index and exclude only that minor's own source fallbacks
- keep judging flat-slot objects by the flat index
- report the fallback count per index, so a collapse is visible in the log
Patrick Schratz 2026-09-02 09:12:57 +00:00
No known key found for this signature in database
GPG key ID: 62050D5BC68AB6DC

View file

@ -154,26 +154,30 @@ index_url <- sprintf(
arch, arch,
codename codename
) )
source_served <- tryCatch( read_source_served <- function(index) {
{ tryCatch(
con_idx <- gzcon(url(index_url, open = "rb")) {
on.exit(close(con_idx), add = TRUE) con_idx <- gzcon(url(index, open = "rb"))
idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) on.exit(close(con_idx), add = TRUE)
sprintf( idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built"))
"%s_%s.tar.gz", sprintf(
idx[is.na(idx[, "Built"]), "Package"], "%s_%s.tar.gz",
idx[is.na(idx[, "Built"]), "Version"] idx[is.na(idx[, "Built"]), "Package"],
) idx[is.na(idx[, "Built"]), "Version"]
}, )
error = function(e) { },
cat(sprintf( error = function(e) {
"WARNING: could not read %s (%s); keeping the full S3 cache\n", cat(sprintf(
index_url, "WARNING: could not read %s (%s); keeping that index's cache in full\n",
conditionMessage(e) index,
)) conditionMessage(e)
character(0) ))
} character(0)
) }
)
}
source_served <- read_source_served(index_url)
binary_cache <- setdiff(file_names, source_served) binary_cache <- setdiff(file_names, source_served)
@ -225,9 +229,65 @@ if (sum(stripped_per_minor) != sum(per_minor_object)) {
)) ))
} }
source_basenames <- source_served source_basenames <- source_served
existence_cache <- relative_paths[
!basename(relative_paths) %in% source_basenames # A source-fallback list describes ONE index, so it may only be applied to that
] # index's objects. Applied to every path it deletes real per-minor binaries from
# the cache: a package the flat slot serves as CRAN source still has a genuine
# `4.4/<pkg>.tar.gz`, and dropping that name makes every per-minor pass rebuild
# it, every run, forever.
#
# The overlap is total rather than partial, which is why this pinned the skip
# rate near 0%: the sensitive candidate list is exactly "sensitive packages with
# no flat binary", which is the same set this was removing. amd64/resolute
# listed 21212 per-minor objects but cached only 13572, then recompiled osmdata,
# osqp and outbreaker2 while their 4.4 binaries sat in the bucket.
#
# So judge each minor by its own index, and the flat slot by the flat index.
is_per_minor_path <- grepl("^[0-9]+\\.[0-9]+/", relative_paths)
path_minor <- ifelse(
is_per_minor_path,
sub("^([0-9]+\\.[0-9]+)/.*$", "\\1", relative_paths),
""
)
minors_present <- sort(unique(path_minor[is_per_minor_path]))
per_minor_source <- lapply(minors_present, function(m) {
read_source_served(sprintf(
"https://cran.rpkgs.com/%s/%s/latest/src/contrib/%s/PACKAGES.gz",
arch,
codename,
m
))
})
names(per_minor_source) <- minors_present
cat(sprintf(
"Source fallbacks per index: flat=%d%s\n",
length(source_basenames),
if (length(minors_present)) {
paste0(
", ",
paste(
sprintf("%s=%d", minors_present, lengths(per_minor_source)),
collapse = ", "
)
)
} else {
""
}
))
served_as_source <- vapply(
seq_along(relative_paths),
function(i) {
scope <- if (nzchar(path_minor[i])) {
per_minor_source[[path_minor[i]]]
} else {
source_basenames
}
basename(relative_paths[i]) %in% scope
},
logical(1L)
)
existence_cache <- relative_paths[!served_as_source]
cat(sprintf( cat(sprintf(
"S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n",
length(file_names), length(file_names),