fix(build): strip the s3 scheme so per-minor objects reach their pass #192
3 changed files with 50 additions and 9 deletions
|
|
@ -201,10 +201,15 @@ s3_cache_paths <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
|||
# nothing; hand it nothing and it rebuilds what it already has. amd64/resolute
|
||||
# recompiled 8683 packages that way.
|
||||
select_cache_for_pass <- function(paths, sensitive_only, r_minor) {
|
||||
# A cache written before this change holds bare basenames. Filtering those by
|
||||
# path would select nothing and trigger a full rebuild, so use them as they
|
||||
# are; #190's staleness check replaces it on the next pipeline anyway.
|
||||
if (!any(grepl("/", paths, fixed = TRUE))) {
|
||||
# A cache written before #191 holds bare basenames. Filtering those by path
|
||||
# would select nothing and trigger a full rebuild, so use them as they are;
|
||||
# #190's staleness check replaces it on the next pipeline anyway.
|
||||
#
|
||||
# The format is read from the marker `packages-to-build.R` sets, not guessed
|
||||
# from the content: a slot-relative cache with no per-minor or Archive object
|
||||
# holds bare names too, and treating that as legacy would hand a per-minor
|
||||
# pass the flat slot's binaries and build nothing.
|
||||
if (!isTRUE(attr(paths, "slot_relative"))) {
|
||||
message("S3 cache is in the legacy basename format; using it unfiltered.")
|
||||
return(paths)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -190,14 +190,44 @@ binary_cache <- setdiff(file_names, source_served)
|
|||
# So the cache keeps the path relative to the slot, and `build-all.R` selects
|
||||
# the part that matches the pass it is running: the flat slot for the primary,
|
||||
# `<minor>/` for a per-minor pass.
|
||||
#
|
||||
# `s3_dir_ls()` returns keys with the `s3://` scheme attached, so stripping the
|
||||
# prefix as a fixed substring takes it out of the middle and leaves
|
||||
# `s3://4.4/curl_1.0.tar.gz`. That leading scheme defeats the `^<minor>/` test
|
||||
# downstream, so every per-minor object is read as a flat-slot object: the
|
||||
# sensitive passes see an empty cache and recompile everything they already
|
||||
# have. Anchor the pattern and swallow the scheme with it.
|
||||
contrib_prefix <- sprintf(
|
||||
"devxy-rpkgs-binaries/%s/%s/latest/src/contrib/",
|
||||
"^(s3://)?devxy-rpkgs-binaries/%s/%s/latest/src/contrib/",
|
||||
arch,
|
||||
codename
|
||||
)
|
||||
relative_paths <- sub(contrib_prefix, "", s3_pkgs, fixed = TRUE)
|
||||
relative_paths <- sub(contrib_prefix, "", s3_pkgs)
|
||||
|
||||
# The strip is load-bearing and fails silently, so assert it. Both counts are
|
||||
# derived from the same listing and use the same shape of pattern, so they must
|
||||
# agree exactly; a mismatch means the prefix no longer describes the keys.
|
||||
stripped_per_minor <- grepl("^[0-9]+\\.[0-9]+/[^/]+$", relative_paths)
|
||||
if (sum(stripped_per_minor) != sum(per_minor_object)) {
|
||||
stop(sprintf(
|
||||
paste0(
|
||||
"S3 prefix strip failed: %d per-minor objects in the listing, %d after ",
|
||||
"stripping /%s/. Example key: %s"
|
||||
),
|
||||
sum(per_minor_object),
|
||||
sum(stripped_per_minor),
|
||||
contrib_prefix,
|
||||
if (any(per_minor_object)) {
|
||||
s3_pkgs[which(per_minor_object)[1L]]
|
||||
} else {
|
||||
"<none>"
|
||||
}
|
||||
))
|
||||
}
|
||||
source_basenames <- source_served
|
||||
existence_cache <- relative_paths[!basename(relative_paths) %in% source_basenames]
|
||||
existence_cache <- relative_paths[
|
||||
!basename(relative_paths) %in% source_basenames
|
||||
]
|
||||
cat(sprintf(
|
||||
"S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n",
|
||||
length(file_names),
|
||||
|
|
@ -208,6 +238,12 @@ cat(sprintf(
|
|||
# Save the S3 file listing for the build step to use as s3_package_cache.
|
||||
# This avoids loading s3fs/reticulate in the build container, saving memory for
|
||||
# the dependency-installer subprocesses
|
||||
# Mark the format explicitly. `build-all.R` has to tell a slot-relative cache
|
||||
# from a pre-#191 basename one, and sniffing for a "/" cannot: a new-format
|
||||
# cache for a slot with no per-minor or Archive objects holds bare names too,
|
||||
# and would be read as legacy and used unfiltered, which makes a per-minor pass
|
||||
# believe the flat slot's binaries are its own and build nothing.
|
||||
attr(existence_cache, "slot_relative") <- TRUE
|
||||
saveRDS(existence_cache, "/mnt/cache/packages/s3_cache.rds")
|
||||
# Built from the filtered listing, not the raw one: `s3_dt` is subtracted from
|
||||
# the build list below, so a source fallback left in here would exclude the very
|
||||
|
|
|
|||
|
|
@ -70,8 +70,8 @@ resolve_zone_id() {
|
|||
zone_id=$(
|
||||
jq -r --arg hostname "${zone}" \
|
||||
'(if type == "object" then (.Items // []) else . end)[]
|
||||
| select(any(.Hostnames[]?; .Value == $hostname))
|
||||
| .Id' \
|
||||
| select(any(.Hostnames[]?; .Value == $hostname))
|
||||
| .Id' \
|
||||
"${response_file}"
|
||||
)
|
||||
rm -f "${response_file}"
|
||||
|
|
|
|||
Loading…
Reference in a new issue