diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 887952d..3b508fb 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -89,14 +89,61 @@ s3_pkgs <- s3fs::s3_dir_ls( recurse = TRUE ) -# Save the raw S3 file listing for the build step to use as s3_package_cache +file_names <- basename(s3_pkgs) + +# An object occupying a key is not proof a binary was built: a package whose +# build failed has its CRAN source published under exactly that name. Left in +# the cache, `build_binary_package()` reads it as "already built" and skips the +# package forever, which is how alpine324 accumulated ~13.5k source tarballs. +# +# bincraft stamps `Built` only on records it actually built, so the slot's own +# index distinguishes them. A slot last indexed by a bincraft that predates that +# fix stamps `Built` on everything, so the cache is then unchanged from before. +# Archived objects have no index record and are kept: unknown means binary, +# never "rebuild it". +index_url <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", + arch, + codename +) +source_served <- tryCatch( + { + con_idx <- gzcon(url(index_url, open = "rb")) + on.exit(close(con_idx), add = TRUE) + idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) + sprintf( + "%s_%s.tar.gz", + idx[is.na(idx[, "Built"]), "Package"], + idx[is.na(idx[, "Built"]), "Version"] + ) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read %s (%s); keeping the full S3 cache\n", + index_url, + conditionMessage(e) + )) + character(0) + } +) + +binary_cache <- setdiff(file_names, source_served) +cat(sprintf( + "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", + length(file_names), + length(file_names) - length(binary_cache), + length(binary_cache) +)) + +# Save the S3 file listing for the build step to use as s3_package_cache. # This avoids loading s3fs/reticulate in the build container, saving memory for # the dependency-installer subprocesses -saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds") - -file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) +saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") +# Built from the filtered listing, not the raw one: `s3_dt` is subtracted from +# the build list below, so a source fallback left in here would exclude the very +# package that needs building. +matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", binary_cache) +parts <- regmatches(binary_cache, matches) parts <- parts[sapply(parts, length) == 3] s3_dt <- data.table( Package = sapply(parts, `[`, 2),