diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index fb1229d..cbbc70d 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -35,6 +35,14 @@ variables: - 'resolute' default: '3.24' R_VERSION: + # The slot's *primary* R minor: what `local/build-all.R` builds into the + # generic slot. The loop below already runs `--sensitive-only` for every + # other installed minor, so filling a non-primary minor's gap needs this + # left alone, not changed. + # + # 4.6.0 was briefly offered here (#183) and removed: selecting it for a + # slot whose generic binaries are 4.5-built would publish 4.6 binaries + # into the generic slot and break every 4.5 client. description: 'Primary R version under /opt/R.' options: - 4.5.3 diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml index b8f0d86..b13245a 100644 --- a/.crow/reindex.yaml +++ b/.crow/reindex.yaml @@ -172,7 +172,12 @@ steps: from_secret: BUNNYNET_API_KEY # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + # The staging zone is listed too. It shares the B2 origin, so an index + # it still holds is a stale copy of the same object, and its + # cache_expiration_time is the same ~370 days: without a purge here it + # serves pre-reindex indexes indefinitely and any verification run + # against it measures the past. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' commands: - apk add --no-cache -q bash curl jq # Crow carries the checkout from the re-index step into this step. diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index d764ba5..9639792 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -20,19 +20,6 @@ variables: # Gates this pipeline. A manual pipeline creation instantiates every file in # .crow/, and a declared default is applied even when the run never passed # this variable, so the default must be a value that matches no matrix row. - # Which R minor to rebuild for. The matrix pins each slot's primary minor, - # which is the only one the weekly cron ever builds - that is why non-primary - # minors accumulate a backlog and why an ABI-risky package can exist for 4.4 - # and 4.5 but not 4.6. Override this to work that backlog off. - rebuild_r_version: - description: "R version to rebuild with: 'matrix' for each slot's primary, or an explicit x.y.z." - options: - - matrix - - 4.6.0 - - 4.5.3 - - 4.4.3 - default: matrix - weekly_rebuild_missing: description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." options: @@ -423,7 +410,6 @@ steps: UVR_PACKAGES_DIR: /mnt/cache/uvr/packages R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} - REBUILD_R_VERSION: ${rebuild_r_version} CCACHE_DIR: /mnt/cache/ccache PLATFORM: ${OS} ARCH: ${ARCH} @@ -438,26 +424,12 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - # `matrix` keeps the slot's primary minor; anything else overrides it. - # rebuild-missing.R derives its target slot from the R it runs under, so - # selecting the interpreter is all that is needed to fill a minor's gap. - - | - if [ "$REBUILD_R_VERSION" = "matrix" ] || [ -z "$REBUILD_R_VERSION" ]; then - RV="$R_VERSION" - else - RV="$REBUILD_R_VERSION" - fi - if [ ! -x "/opt/R/$RV/bin/R" ]; then - echo "R $RV is not installed in this image; available: $(ls /opt/R)" >&2 - exit 1 - fi - echo "Rebuilding with R $RV" - - /opt/R/$RV/bin/Rscript local/install-bincraft.R - - /opt/R/$RV/bin/R -q -e 'packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - UVR_R_BIN=/opt/R/$RV/bin/R local/uvr-install.sh httr2 - - /opt/R/$RV/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$RV/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 + - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 backend_options: docker: resources: diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 2e0f2a0..c9f9a8a 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -175,7 +175,12 @@ steps: from_secret: BUNNYNET_API_KEY # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + # The staging zone is listed too. It shares the B2 origin, so an index + # it still holds is a stale copy of the same object, and its + # cache_expiration_time is the same ~370 days: without a purge here it + # serves pre-reindex indexes indefinitely and any verification run + # against it measures the past. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' commands: - apk add --no-cache -q bash curl jq # Crow carries the checkout from the re-index step into this step. diff --git a/cdn.tf b/cdn.tf index 349486e..df30273 100644 --- a/cdn.tf +++ b/cdn.tf @@ -89,9 +89,19 @@ resource "bunnynet_compute_script" "rpkgs_router" { # slot that is not listed here would hide every package the per-minor index does # not carry, so this stays empty until a slot has been backfilled. resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { - script = bunnynet_compute_script.rpkgs_router.id - name = "UNION_SLOTS" - default_value = "" + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + # Enabled. Until this was set, every client resolved against the generic + # index and never reached a per-minor binary: an R 4.6.1 client on resolute + # downloaded the 4.5-built rlang (2079570 bytes) while the correct 4.6 build + # (2075106 bytes) sat unused one directory away, and died at load with + # `undefined symbol: SETLENGTH`. + # + # Verified before enabling, against the staging zone with the same script and + # the same origin: all 16 slots report zero regressions against the generic + # slot, an excluded R minor is sent to CRAN, a client without an R minor + # still gets the generic index, and tarball requests are never rewritten. + default_value = join(",", local.rpkgs_slots) required = false } diff --git a/local/build-all.R b/local/build-all.R index 37fc593..349092f 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -26,9 +26,50 @@ package_cache_files <- c( "/mnt/cache/packages/r_minor_sensitive_pkgs.rds", "/mnt/cache/packages/s3_cache.rds" ) -if (!all(file.exists(package_cache_files))) { +# Existence is not freshness. The snapshot describes S3 and CRAN state at the +# moment it was written, and the volume is per-agent, so an agent that ran an +# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324 +# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43 +# sensitive packages, while the install-deps step in the very same pipeline had +# just computed 6871 on another agent. +# +# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes +# once per agent and its shards then share the result. Off CI, or when no such +# variable is set, fall back to an age check. +snapshot_id_path <- "/mnt/cache/packages/snapshot.id" +snapshot_ttl_hours <- as.numeric( + Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2") +) +current_snapshot_id <- "" +for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) { + val <- Sys.getenv(v, unset = "") + if (nzchar(val)) { + current_snapshot_id <- paste(v, val, sep = "=") + break + } +} + +snapshot_is_stale <- function() { + if (!all(file.exists(package_cache_files))) { + return(TRUE) + } + if (nzchar(current_snapshot_id)) { + cached <- tryCatch( + readLines(snapshot_id_path, warn = FALSE)[1L], + error = function(e) NA_character_, + warning = function(w) NA_character_ + ) + return(!identical(cached, current_snapshot_id)) + } + age_hours <- as.numeric( + difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours") + ) + isTRUE(age_hours > snapshot_ttl_hours) +} + +if (snapshot_is_stale()) { message( - "Package snapshot missing from cache; recomputing via packages-to-build.R" + "Package snapshot missing or stale; recomputing via packages-to-build.R" ) dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { @@ -42,6 +83,9 @@ if (!all(file.exists(package_cache_files))) { pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" ) + if (nzchar(current_snapshot_id)) { + writeLines(current_snapshot_id, snapshot_id_path) + } message("Package snapshot recomputed.") } @@ -110,10 +154,25 @@ con <- DBI::dbConnect( password = Sys.getenv("PGPASS"), sslmode = "require" ) +# Scope the skip to the R minor this pass is running under. `single_builds` +# records `r_version` per attempt, but querying without it made a non-primary +# pass skip everything the primary pass had already attempted under a different +# minor - so `--sensitive-only` under 4.6 skipped packages that had only ever +# been built for 4.5, and the per-minor slots never filled. That is why +# amd64/resolute served 4000 fewer packages to a 4.6 client than to a 4.5 one. +r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) built <- DBI::dbGetQuery( con, - "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2", - params = list(platform, arch) + paste( + "SELECT name, tag FROM single_builds", + "WHERE platform = $1 AND arch = $2", + "AND substring(r_version from '^[0-9]+[.][0-9]+') = $3" + ), + params = list(platform, arch, r_minor) ) DBI::dbDisconnect(con) before <- nrow(chunk) @@ -121,16 +180,56 @@ chunk <- chunk[ !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] sprintf( - "Skipped %d already-attempted package versions; %d remaining for this job", + "Skipped %d package versions already attempted under R %s; %d remaining for this job", before - nrow(chunk), + r_minor, nrow(chunk) ) # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, # saving significant memory for the dependency-installer subprocesses -s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") -sprintf("S3 cache: %s files", length(s3_cache)) +s3_cache_paths <- readRDS("/mnt/cache/packages/s3_cache.rds") + +# The cache is stored as paths relative to the slot, so a pass can select the +# objects that belong to it. `build_binary_package()` compares basenames, which +# cannot distinguish `4.4/curl_1.0.tar.gz` from `curl_1.0.tar.gz`, so the choice +# has to be made here where the running R minor is known. +# +# Getting this wrong is expensive in both directions: hand it everything and a +# per-minor pass believes the flat slot's binaries are its own and builds +# nothing; hand it nothing and it rebuilds what it already has. amd64/resolute +# recompiled 8683 packages that way. +select_cache_for_pass <- function(paths, sensitive_only, r_minor) { + # A cache written before #191 holds bare basenames. Filtering those by path + # would select nothing and trigger a full rebuild, so use them as they are; + # #190's staleness check replaces it on the next pipeline anyway. + # + # The format is read from the marker `packages-to-build.R` sets, not guessed + # from the content: a slot-relative cache with no per-minor or Archive object + # holds bare names too, and treating that as legacy would hand a per-minor + # pass the flat slot's binaries and build nothing. + if (!isTRUE(attr(paths, "slot_relative"))) { + message("S3 cache is in the legacy basename format; using it unfiltered.") + return(paths) + } + in_minor <- grepl(sprintf("^%s/", r_minor), paths) + if (sensitive_only) { + basename(paths[in_minor]) + } else { + # The primary pass writes the flat slot. Archive/ counts as present there: + # those are versions built and later superseded. + basename(paths[!grepl("^[0-9]+\\.[0-9]+/", paths)]) + } +} + +s3_cache <- select_cache_for_pass(s3_cache_paths, sensitive_only, r_minor) +sprintf( + "S3 cache: %s files (%s of %s objects apply to this pass)", + length(s3_cache), + length(s3_cache), + length(s3_cache_paths) +) n <- nrow(chunk) mapply( diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 3b508fb..0654a43 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -68,9 +68,36 @@ archive_versions <- archive_versions[ ] # Now get release versions (assuming cran_release has Package and Version columns) +# +# Packages published in the last few days are held back. `check_for_binary()` +# reads the published version from the `cran` GitHub mirror +# (`GET /repos/cran//commits`), and that mirror lags CRAN: a package that +# has just appeared has no repository there yet. The call then 404s, which is +# permanent, but it is wrapped in `purrr::insistently` and retried ten times +# with a backoff capped at 60s - so one unmirrored package burns about five +# minutes and then aborts the whole shard. +# +# Holding them back costs nothing: the daily update pipeline builds new and +# updated packages anyway, and they arrive here on the next run once the mirror +# has caught up. +mirror_lag_days <- as.numeric( + Sys.getenv("CRAN_MIRROR_LAG_DAYS", unset = "3") +) +published <- as.POSIXct(cran_release$Published, tz = "UTC") +too_recent <- !is.na(published) & + published > (Sys.time() - mirror_lag_days * 86400) +if (any(too_recent)) { + message(sprintf( + "Holding back %d package(s) published in the last %g day(s); the cran GitHub mirror will not have them yet: %s", + sum(too_recent), + mirror_lag_days, + paste(utils::head(cran_release$Package[too_recent], 10L), collapse = ", ") + )) +} + release_versions <- data.table( - Package = cran_release$Package, - Version = as.character(cran_release$Version) + Package = cran_release$Package[!too_recent], + Version = as.character(cran_release$Version[!too_recent]) ) pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE)) @@ -89,7 +116,28 @@ s3_pkgs <- s3fs::s3_dir_ls( recurse = TRUE ) -file_names <- basename(s3_pkgs) +# `recurse = TRUE` walks the per-minor slots as well, and `basename()` throws +# the directory away - so `4.5/curl_1.0.tar.gz` and `curl_1.0.tar.gz` collapse +# to one name and a package present under *any* R minor counts as built for +# *all* of them. The candidate list then prunes exactly the packages a +# per-minor pass exists to build: arm64/alpine324 reported "0 remaining" for +# both 4.4 and 4.6 while its indexes were dropping 2400+ packages as missing. +# +# Per-minor objects are therefore excluded here. Presence in a specific minor +# is decided downstream, where the running R version is known: build-all.R +# filters on it, and `build_binary_package()` checks the per-minor path per +# package and skips what is already there. +# +# Archive/ is kept. Those are versions that were built and then superseded; +# dropping them would make every archived version look unbuilt. +per_minor_object <- grepl("/[0-9]+\\.[0-9]+/[^/]+$", s3_pkgs) +if (any(per_minor_object)) { + cat(sprintf( + "Excluding %d per-minor object(s) from the presence check; those are decided per pass\n", + sum(per_minor_object) + )) +} +file_names <- basename(s3_pkgs[!per_minor_object]) # An object occupying a key is not proof a binary was built: a package whose # build failed has its CRAN source published under exactly that name. Left in @@ -106,28 +154,140 @@ index_url <- sprintf( arch, codename ) -source_served <- tryCatch( - { - con_idx <- gzcon(url(index_url, open = "rb")) - on.exit(close(con_idx), add = TRUE) - idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) - sprintf( - "%s_%s.tar.gz", - idx[is.na(idx[, "Built"]), "Package"], - idx[is.na(idx[, "Built"]), "Version"] - ) - }, - error = function(e) { - cat(sprintf( - "WARNING: could not read %s (%s); keeping the full S3 cache\n", - index_url, - conditionMessage(e) - )) - character(0) - } -) +read_source_served <- function(index) { + tryCatch( + { + con_idx <- gzcon(url(index, open = "rb")) + on.exit(close(con_idx), add = TRUE) + idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) + sprintf( + "%s_%s.tar.gz", + idx[is.na(idx[, "Built"]), "Package"], + idx[is.na(idx[, "Built"]), "Version"] + ) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read %s (%s); keeping that index's cache in full\n", + index, + conditionMessage(e) + )) + character(0) + } + ) +} + +source_served <- read_source_served(index_url) binary_cache <- setdiff(file_names, source_served) + +# The existence cache and the candidate list need different views of the same +# listing, and conflating them is what made this wrong in both directions. +# +# The candidate list must ignore per-minor objects, or a package present under +# one minor prunes itself from every other minor's work (#189). The existence +# cache must NOT ignore them, or `build_binary_package()` is told nothing is +# present under any minor and recompiles the lot: amd64/resolute recompiled +# 8683 packages it had already built, reporting "already exists in S3" three +# times. +# +# So the cache keeps the path relative to the slot, and `build-all.R` selects +# the part that matches the pass it is running: the flat slot for the primary, +# `/` for a per-minor pass. +# +# `s3_dir_ls()` returns keys with the `s3://` scheme attached, so stripping the +# prefix as a fixed substring takes it out of the middle and leaves +# `s3://4.4/curl_1.0.tar.gz`. That leading scheme defeats the `^/` test +# downstream, so every per-minor object is read as a flat-slot object: the +# sensitive passes see an empty cache and recompile everything they already +# have. Anchor the pattern and swallow the scheme with it. +contrib_prefix <- sprintf( + "^(s3://)?devxy-rpkgs-binaries/%s/%s/latest/src/contrib/", + arch, + codename +) +relative_paths <- sub(contrib_prefix, "", s3_pkgs) + +# The strip is load-bearing and fails silently, so assert it. Both counts are +# derived from the same listing and use the same shape of pattern, so they must +# agree exactly; a mismatch means the prefix no longer describes the keys. +stripped_per_minor <- grepl("^[0-9]+\\.[0-9]+/[^/]+$", relative_paths) +if (sum(stripped_per_minor) != sum(per_minor_object)) { + stop(sprintf( + paste0( + "S3 prefix strip failed: %d per-minor objects in the listing, %d after ", + "stripping /%s/. Example key: %s" + ), + sum(per_minor_object), + sum(stripped_per_minor), + contrib_prefix, + if (any(per_minor_object)) { + s3_pkgs[which(per_minor_object)[1L]] + } else { + "" + } + )) +} +source_basenames <- source_served + +# A source-fallback list describes ONE index, so it may only be applied to that +# index's objects. Applied to every path it deletes real per-minor binaries from +# the cache: a package the flat slot serves as CRAN source still has a genuine +# `4.4/.tar.gz`, and dropping that name makes every per-minor pass rebuild +# it, every run, forever. +# +# The overlap is total rather than partial, which is why this pinned the skip +# rate near 0%: the sensitive candidate list is exactly "sensitive packages with +# no flat binary", which is the same set this was removing. amd64/resolute +# listed 21212 per-minor objects but cached only 13572, then recompiled osmdata, +# osqp and outbreaker2 while their 4.4 binaries sat in the bucket. +# +# So judge each minor by its own index, and the flat slot by the flat index. +is_per_minor_path <- grepl("^[0-9]+\\.[0-9]+/", relative_paths) +path_minor <- ifelse( + is_per_minor_path, + sub("^([0-9]+\\.[0-9]+)/.*$", "\\1", relative_paths), + "" +) +minors_present <- sort(unique(path_minor[is_per_minor_path])) +per_minor_source <- lapply(minors_present, function(m) { + read_source_served(sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/%s/PACKAGES.gz", + arch, + codename, + m + )) +}) +names(per_minor_source) <- minors_present +cat(sprintf( + "Source fallbacks per index: flat=%d%s\n", + length(source_basenames), + if (length(minors_present)) { + paste0( + ", ", + paste( + sprintf("%s=%d", minors_present, lengths(per_minor_source)), + collapse = ", " + ) + ) + } else { + "" + } +)) + +served_as_source <- vapply( + seq_along(relative_paths), + function(i) { + scope <- if (nzchar(path_minor[i])) { + per_minor_source[[path_minor[i]]] + } else { + source_basenames + } + basename(relative_paths[i]) %in% scope + }, + logical(1L) +) +existence_cache <- relative_paths[!served_as_source] cat(sprintf( "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", length(file_names), @@ -138,7 +298,13 @@ cat(sprintf( # Save the S3 file listing for the build step to use as s3_package_cache. # This avoids loading s3fs/reticulate in the build container, saving memory for # the dependency-installer subprocesses -saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") +# Mark the format explicitly. `build-all.R` has to tell a slot-relative cache +# from a pre-#191 basename one, and sniffing for a "/" cannot: a new-format +# cache for a slot with no per-minor or Archive objects holds bare names too, +# and would be read as legacy and used unfiltered, which makes a per-minor pass +# believe the flat slot's binaries are its own and build nothing. +attr(existence_cache, "slot_relative") <- TRUE +saveRDS(existence_cache, "/mnt/cache/packages/s3_cache.rds") # Built from the filtered listing, not the raw one: `s3_dt` is subtracted from # the build list below, so a source fallback left in here would exclude the very # package that needs building. @@ -152,11 +318,22 @@ s3_dt <- data.table( ### Get all packages with build errors +# Scoped to the R minor this snapshot is computed under. A failure is a fact +# about one interpreter: without the scope a package that failed under the +# primary minor is dropped from the candidate list for every other minor too, +# which is the same omission fixed in local/build-all.R and in bincraft's +# check_package_error(). +snapshot_r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) sql_query <- paste0( # nolint "SELECT error_occurred FROM ", "single_builds", - " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4" + " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4", + " AND substring(r_version from '^[0-9]+[.][0-9]+') = $5" ) # Function to query for a single package-version query_error <- function(pkg, ver) { @@ -164,7 +341,7 @@ query_error <- function(pkg, ver) { ~ DBI::dbGetQuery( con, sql_query, - params = list(pkg, ver, platform, arch) + params = list(pkg, ver, platform, arch, snapshot_r_minor) ), rate = purrr::rate_backoff( pause_base = 1L, diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 6c07eef..8e85cb6 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -70,8 +70,8 @@ resolve_zone_id() { zone_id=$( jq -r --arg hostname "${zone}" \ '(if type == "object" then (.Items // []) else . end)[] - | select(any(.Hostnames[]?; .Value == $hostname)) - | .Id' \ + | select(any(.Hostnames[]?; .Value == $hostname)) + | .Id' \ "${response_file}" ) rm -f "${response_file}" diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh index 3e1a7e4..190245a 100755 --- a/scripts/verify-r-minor-routing.sh +++ b/scripts/verify-r-minor-routing.sh @@ -43,10 +43,13 @@ MINORS=${MINORS:-"4.4 4.5 4.6"} EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"} # How many Path: targets to HEAD-check per slot/minor. 0 disables. SAMPLE=${SAMPLE:-5} -# Largest package-count shortfall a non-primary minor may have against the best -# minor on the same slot before coverage counts as uneven. A slot built under -# one R minor carries fewer per-minor binaries for the others; until that gap -# closes, "full coverage for ABI-sensitive packages" is not a claim we can make. +# Package-count shortfall against the best minor on the same slot, above which +# coverage is reported as uneven. Reported, not failed on: the packages a +# non-primary minor lacks are ABI-risky ones built under the primary minor, +# which a client on another minor cannot safely load anyway, so their absence +# is correct. This gates the *claim* ("full coverage for ABI-sensitive +# packages"), not whether routing is safe to enable - MAX_REGRESSIONS does +# that. PARITY_TOLERANCE=${PARITY_TOLERANCE:-25} # Source fallbacks are reported, not failed on. Since bincraft learned to keep # a matching-minor generic binary out of a fallback's shadow, a remaining @@ -101,10 +104,15 @@ fetch() { return 0 fi local status + # -L: the router answers an index request with a redirect, so the bytes a + # client ends up with are only visible by following it. + # + # no-cache: a purge is asynchronous, so a run started right after a reindex + # otherwise measures whatever the edge still holds. if [ -n "$ua" ]; then - status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + status=$(curl -sSL -A "$ua" -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) else - status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + status=$(curl -sSL -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) fi echo "$status" > "$dest.status" echo "$status" @@ -132,6 +140,23 @@ fallback_counts() { ' } +# Packages this index serves from the generic slot with a binary built under a +# different R minor, while some other per-minor slot carries a build of them - +# which proves the ABI classifier called them risky. Serving those is the +# load-time crash the per-minor slots exist to prevent. bincraft drops them at +# index time, so a non-zero count means the slot has not been reindexed since +# that guard shipped. +abi_unsafe_count() { + local minor_file=$1 minor=$2 risky_file=$3 + gunzip -c "$minor_file" 2>/dev/null | awk -v m="$minor" ' + /^Package:/ { pkg = $2; path = ""; built = "" } + /^Path:/ { path = $2 } + /^Built:/ { built = $2 " " $3 } + /^$/ { if (pkg != "" && path == "" && built != "" && built !~ ("^R " m "\\.")) print pkg; pkg = "" } + ' | sort -u > "$minor_file.mismatched" + comm -12 "$minor_file.mismatched" "$risky_file" | wc -l +} + # How many packages a client of would receive as source through # per-minor routing while the generic slot holds a binary built under that very # minor. Zero is the bar for enabling a slot. @@ -211,12 +236,25 @@ for arch in $ARCHES; do minor_count=$(wc -l < "$minor_file.names") # Union property: nothing the flat index carries may be missing here. - missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5) - missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l) - if [ "$missing_count" -ne 0 ]; then - bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))" + # bincraft deliberately drops an ABI-risky package whose only binary was + # built under another R minor: serving it is the load-time crash the + # per-minor slots exist to prevent. Those absences are correct. + # + # What must never go missing is a generic package built under *this* + # minor, which is safe to serve and has no reason to disappear. + comm -23 "$flat_file.names" "$minor_file.names" > "$minor_file.absent" + absent_count=$(wc -l < "$minor_file.absent") + gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" ' + /^Package:/ { pkg = $2; built = "" } + /^Built:/ { built = $2 " " $3 } + /^$/ { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" } + ' | sort -u > "$minor_file.flatsame" + lost=$(comm -12 "$minor_file.absent" "$minor_file.flatsame" | wc -l) + + if [ "${lost:-0}" -ne 0 ]; then + bad "$slot R $minor index dropped $lost generic package(s) built under R $minor, which were safe to serve" else - ok "$slot R $minor index: $minor_count packages, union holds" + ok "$slot R $minor index: $minor_count packages, union holds ($absent_count ABI-unsafe dropped)" fi # A per-minor entry that is a source fallback resolves fine but makes the @@ -299,12 +337,37 @@ for arch in $ARCHES; do [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap" done if [ -n "$uneven" ]; then - bad "$slot coverage uneven across minors (vs best $best):$uneven" + printf ' note: %s coverage uneven across minors (vs best %s):%s\n' \ + "$slot" "$best" "$uneven" else ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)" fi fi + # Packages carrying a Path in any per-minor index are risky by construction. + : > "$WORK/${arch}-${distro}.risky" + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz" + [ -s "$f" ] || continue + gunzip -c "$f" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; path = "" } + /^Path:/ { path = $2 } + /^$/ { if (pkg != "" && path != "") print pkg; pkg = "" } + ' >> "$WORK/${arch}-${distro}.risky" + done + sort -u -o "$WORK/${arch}-${distro}.risky" "$WORK/${arch}-${distro}.risky" + + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz" + [ -s "$f" ] || continue + unsafe=$(abi_unsafe_count "$f" "$minor" "$WORK/${arch}-${distro}.risky") + if [ "${unsafe:-0}" -gt 0 ]; then + bad "$slot R $minor serves $unsafe ABI-risky package(s) built under another R minor - reindex this slot" + else + ok "$slot R $minor serves no ABI-risky package from another minor" + fi + done + # Excluded minors: no published index, and under --live a redirect to CRAN. for minor in $EXCLUDED_MINORS; do ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz"