diff --git a/local/build-all.R b/local/build-all.R index c4d7f97..eda92e9 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -189,8 +189,42 @@ sprintf( # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, # saving significant memory for the dependency-installer subprocesses -s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") -sprintf("S3 cache: %s files", length(s3_cache)) +s3_cache_paths <- readRDS("/mnt/cache/packages/s3_cache.rds") + +# The cache is stored as paths relative to the slot, so a pass can select the +# objects that belong to it. `build_binary_package()` compares basenames, which +# cannot distinguish `4.4/curl_1.0.tar.gz` from `curl_1.0.tar.gz`, so the choice +# has to be made here where the running R minor is known. +# +# Getting this wrong is expensive in both directions: hand it everything and a +# per-minor pass believes the flat slot's binaries are its own and builds +# nothing; hand it nothing and it rebuilds what it already has. amd64/resolute +# recompiled 8683 packages that way. +select_cache_for_pass <- function(paths, sensitive_only, r_minor) { + # A cache written before this change holds bare basenames. Filtering those by + # path would select nothing and trigger a full rebuild, so use them as they + # are; #190's staleness check replaces it on the next pipeline anyway. + if (!any(grepl("/", paths, fixed = TRUE))) { + message("S3 cache is in the legacy basename format; using it unfiltered.") + return(paths) + } + in_minor <- grepl(sprintf("^%s/", r_minor), paths) + if (sensitive_only) { + basename(paths[in_minor]) + } else { + # The primary pass writes the flat slot. Archive/ counts as present there: + # those are versions built and later superseded. + basename(paths[!grepl("^[0-9]+\\.[0-9]+/", paths)]) + } +} + +s3_cache <- select_cache_for_pass(s3_cache_paths, sensitive_only, r_minor) +sprintf( + "S3 cache: %s files (%s of %s objects apply to this pass)", + length(s3_cache), + length(s3_cache), + length(s3_cache_paths) +) n <- nrow(chunk) mapply( diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 9cc1ed6..64a2f5f 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -176,6 +176,28 @@ source_served <- tryCatch( ) binary_cache <- setdiff(file_names, source_served) + +# The existence cache and the candidate list need different views of the same +# listing, and conflating them is what made this wrong in both directions. +# +# The candidate list must ignore per-minor objects, or a package present under +# one minor prunes itself from every other minor's work (#189). The existence +# cache must NOT ignore them, or `build_binary_package()` is told nothing is +# present under any minor and recompiles the lot: amd64/resolute recompiled +# 8683 packages it had already built, reporting "already exists in S3" three +# times. +# +# So the cache keeps the path relative to the slot, and `build-all.R` selects +# the part that matches the pass it is running: the flat slot for the primary, +# `/` for a per-minor pass. +contrib_prefix <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib/", + arch, + codename +) +relative_paths <- sub(contrib_prefix, "", s3_pkgs, fixed = TRUE) +source_basenames <- source_served +existence_cache <- relative_paths[!basename(relative_paths) %in% source_basenames] cat(sprintf( "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", length(file_names), @@ -186,7 +208,7 @@ cat(sprintf( # Save the S3 file listing for the build step to use as s3_package_cache. # This avoids loading s3fs/reticulate in the build container, saving memory for # the dependency-installer subprocesses -saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") +saveRDS(existence_cache, "/mnt/cache/packages/s3_cache.rds") # Built from the filtered listing, not the raw one: `s3_dt` is subtracted from # the build list below, so a source fallback left in here would exclude the very # package that needs building.