diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index cbbc70d..fb1229d 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -35,14 +35,6 @@ variables: - 'resolute' default: '3.24' R_VERSION: - # The slot's *primary* R minor: what `local/build-all.R` builds into the - # generic slot. The loop below already runs `--sensitive-only` for every - # other installed minor, so filling a non-primary minor's gap needs this - # left alone, not changed. - # - # 4.6.0 was briefly offered here (#183) and removed: selecting it for a - # slot whose generic binaries are 4.5-built would publish 4.6 binaries - # into the generic slot and break every 4.5 client. description: 'Primary R version under /opt/R.' options: - 4.5.3 diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml deleted file mode 100644 index b13245a..0000000 --- a/.crow/reindex.yaml +++ /dev/null @@ -1,193 +0,0 @@ -# Re-index every slot without rebuilding anything. -# -# `weekly-rebuild-reindex` exists to run after `weekly-rebuild-missing`, so it -# depends on that workflow and shares its gate: triggering it manually also -# starts hours of package rebuilds. That is the wrong tool when only the index -# needs regenerating - after a bincraft release that changes how the index is -# written, for instance, where the objects in the bucket are already correct -# and only `PACKAGES*` is stale. -# -# This workflow does the index half on its own. It installs the latest bincraft -# release, republishes the generic and per-R-minor indexes for each slot, and -# purges the edge. No package is built. -# -# Trigger with the `reindex` variable set to `all` or to a single -# `-`, e.g. -# -# crow pipeline create devxy/build-cran-binaries --var reindex=all - -variables: - # A manual pipeline creation instantiates every file in .crow/, so the - # default must match no matrix row. - reindex: - description: "Re-index target: a specific -, 'all' for every slot, or 'none'." - options: - - none - - all - - alpine-322-amd64 - - alpine-322-arm64 - - alpine-323-amd64 - - alpine-323-arm64 - - alpine-324-amd64 - - alpine-324-arm64 - - redhat-8-amd64 - - redhat-8-arm64 - - redhat-9-amd64 - - redhat-9-arm64 - - redhat-10-amd64 - - redhat-10-arm64 - - ubuntu-2204-amd64 - - ubuntu-2204-arm64 - - ubuntu-2404-amd64 - - ubuntu-2404-arm64 - - ubuntu-2604-amd64 - - ubuntu-2604-arm64 - default: none - -when: - - event: manual - evaluate: 'reindex == "all" || reindex == "${OS}-${ARCH}"' - -skip_clone: true - -labels: - group: rpkgs-${ARCH} - -matrix: - include: - - OS: alpine-322 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-322 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.22 - - OS: alpine-323 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-323 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.23 - - OS: alpine-324 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: alpine-324 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: alpine:3.24 - - OS: redhat-8 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-8 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:8 - - OS: redhat-9 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-9 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: redhat:9 - - OS: redhat-10 - ARCH: amd64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: redhat-10 - ARCH: arm64 - R_VERSION: 4.5.3 - IMG: redhat:10 - - OS: ubuntu-2204 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2204 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:jammy - - OS: ubuntu-2404 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2404 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:noble - - OS: ubuntu-2604 - ARCH: amd64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - - OS: ubuntu-2604 - ARCH: arm64 - R_VERSION: 4.4.3 - IMG: ubuntu:resolute - -steps: - - name: 'Re-index the slot' - image: reg.devxy.io/rpkgs/build-env-${IMG} - pull: true - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GIT_USER: pat-s - R_LIBS_USER: /mnt/cache/R-pkgs - R_VERSION: ${R_VERSION} - PLATFORM: ${OS} - ARCH: ${ARCH} - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - # The codename is detected from the image's /etc/os-release. - - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' - - | - for RBIN in /opt/R/[0-9]*/bin/R; do - RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) - /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true - done - - - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.24 - environment: - OTEL_R_TRACES_EXPORTER: none - OTEL_R_LOGS_EXPORTER: none - OTEL_R_METRICS_EXPORTER: none - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate - # Bunny pull zones, so both must be purged after the shared origin changes. - # The staging zone is listed too. It shares the B2 origin, so an index - # it still holds is a stale copy of the same object, and its - # cache_expiration_time is the same ~370 days: without a purge here it - # serves pre-reindex indexes indefinitely and any verification run - # against it measures the past. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' - commands: - - apk add --no-cache -q bash curl jq - # Crow carries the checkout from the re-index step into this step. - - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES - # Runs on every row rather than on one designated slot: a cron fires only - # its own slot's row, so gating on a named slot would leave every other - # slot unpurged. A manual "all" run therefore purges the zone 18 times, - # which is a cheap API call and rare. - # - # Run it even when the re-index above failed: the objects were still - # replaced, and a stale edge is exactly what keeps them hidden. - when: - - status: [success, failure] diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index c9f9a8a..2e0f2a0 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -175,12 +175,7 @@ steps: from_secret: BUNNYNET_API_KEY # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate # Bunny pull zones, so both must be purged after the shared origin changes. - # The staging zone is listed too. It shares the B2 origin, so an index - # it still holds is a stale copy of the same object, and its - # cache_expiration_time is the same ~370 days: without a purge here it - # serves pre-reindex indexes indefinitely and any verification run - # against it measures the past. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' commands: - apk add --no-cache -q bash curl jq # Crow carries the checkout from the re-index step into this step. diff --git a/cdn.tf b/cdn.tf index df30273..349486e 100644 --- a/cdn.tf +++ b/cdn.tf @@ -89,19 +89,9 @@ resource "bunnynet_compute_script" "rpkgs_router" { # slot that is not listed here would hide every package the per-minor index does # not carry, so this stays empty until a slot has been backfilled. resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { - script = bunnynet_compute_script.rpkgs_router.id - name = "UNION_SLOTS" - # Enabled. Until this was set, every client resolved against the generic - # index and never reached a per-minor binary: an R 4.6.1 client on resolute - # downloaded the 4.5-built rlang (2079570 bytes) while the correct 4.6 build - # (2075106 bytes) sat unused one directory away, and died at load with - # `undefined symbol: SETLENGTH`. - # - # Verified before enabling, against the staging zone with the same script and - # the same origin: all 16 slots report zero regressions against the generic - # slot, an excluded R minor is sent to CRAN, a client without an R minor - # still gets the generic index, and tarball requests are never rewritten. - default_value = join(",", local.rpkgs_slots) + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + default_value = "" required = false } diff --git a/local/build-all.R b/local/build-all.R index c4d7f97..37fc593 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -26,50 +26,9 @@ package_cache_files <- c( "/mnt/cache/packages/r_minor_sensitive_pkgs.rds", "/mnt/cache/packages/s3_cache.rds" ) -# Existence is not freshness. The snapshot describes S3 and CRAN state at the -# moment it was written, and the volume is per-agent, so an agent that ran an -# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324 -# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43 -# sensitive packages, while the install-deps step in the very same pipeline had -# just computed 6871 on another agent. -# -# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes -# once per agent and its shards then share the result. Off CI, or when no such -# variable is set, fall back to an age check. -snapshot_id_path <- "/mnt/cache/packages/snapshot.id" -snapshot_ttl_hours <- as.numeric( - Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2") -) -current_snapshot_id <- "" -for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) { - val <- Sys.getenv(v, unset = "") - if (nzchar(val)) { - current_snapshot_id <- paste(v, val, sep = "=") - break - } -} - -snapshot_is_stale <- function() { - if (!all(file.exists(package_cache_files))) { - return(TRUE) - } - if (nzchar(current_snapshot_id)) { - cached <- tryCatch( - readLines(snapshot_id_path, warn = FALSE)[1L], - error = function(e) NA_character_, - warning = function(w) NA_character_ - ) - return(!identical(cached, current_snapshot_id)) - } - age_hours <- as.numeric( - difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours") - ) - isTRUE(age_hours > snapshot_ttl_hours) -} - -if (snapshot_is_stale()) { +if (!all(file.exists(package_cache_files))) { message( - "Package snapshot missing or stale; recomputing via packages-to-build.R" + "Package snapshot missing from cache; recomputing via packages-to-build.R" ) dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { @@ -83,9 +42,6 @@ if (snapshot_is_stale()) { pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" ) - if (nzchar(current_snapshot_id)) { - writeLines(current_snapshot_id, snapshot_id_path) - } message("Package snapshot recomputed.") } @@ -154,25 +110,10 @@ con <- DBI::dbConnect( password = Sys.getenv("PGPASS"), sslmode = "require" ) -# Scope the skip to the R minor this pass is running under. `single_builds` -# records `r_version` per attempt, but querying without it made a non-primary -# pass skip everything the primary pass had already attempted under a different -# minor - so `--sensitive-only` under 4.6 skipped packages that had only ever -# been built for 4.5, and the per-minor slots never filled. That is why -# amd64/resolute served 4000 fewer packages to a 4.6 client than to a 4.5 one. -r_minor <- paste( - R.version$major, - strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], - sep = "." -) built <- DBI::dbGetQuery( con, - paste( - "SELECT name, tag FROM single_builds", - "WHERE platform = $1 AND arch = $2", - "AND substring(r_version from '^[0-9]+[.][0-9]+') = $3" - ), - params = list(platform, arch, r_minor) + "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2", + params = list(platform, arch) ) DBI::dbDisconnect(con) before <- nrow(chunk) @@ -180,9 +121,8 @@ chunk <- chunk[ !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] sprintf( - "Skipped %d package versions already attempted under R %s; %d remaining for this job", + "Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), - r_minor, nrow(chunk) ) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 9cc1ed6..3b508fb 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -68,36 +68,9 @@ archive_versions <- archive_versions[ ] # Now get release versions (assuming cran_release has Package and Version columns) -# -# Packages published in the last few days are held back. `check_for_binary()` -# reads the published version from the `cran` GitHub mirror -# (`GET /repos/cran//commits`), and that mirror lags CRAN: a package that -# has just appeared has no repository there yet. The call then 404s, which is -# permanent, but it is wrapped in `purrr::insistently` and retried ten times -# with a backoff capped at 60s - so one unmirrored package burns about five -# minutes and then aborts the whole shard. -# -# Holding them back costs nothing: the daily update pipeline builds new and -# updated packages anyway, and they arrive here on the next run once the mirror -# has caught up. -mirror_lag_days <- as.numeric( - Sys.getenv("CRAN_MIRROR_LAG_DAYS", unset = "3") -) -published <- as.POSIXct(cran_release$Published, tz = "UTC") -too_recent <- !is.na(published) & - published > (Sys.time() - mirror_lag_days * 86400) -if (any(too_recent)) { - message(sprintf( - "Holding back %d package(s) published in the last %g day(s); the cran GitHub mirror will not have them yet: %s", - sum(too_recent), - mirror_lag_days, - paste(utils::head(cran_release$Package[too_recent], 10L), collapse = ", ") - )) -} - release_versions <- data.table( - Package = cran_release$Package[!too_recent], - Version = as.character(cran_release$Version[!too_recent]) + Package = cran_release$Package, + Version = as.character(cran_release$Version) ) pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE)) @@ -116,28 +89,7 @@ s3_pkgs <- s3fs::s3_dir_ls( recurse = TRUE ) -# `recurse = TRUE` walks the per-minor slots as well, and `basename()` throws -# the directory away - so `4.5/curl_1.0.tar.gz` and `curl_1.0.tar.gz` collapse -# to one name and a package present under *any* R minor counts as built for -# *all* of them. The candidate list then prunes exactly the packages a -# per-minor pass exists to build: arm64/alpine324 reported "0 remaining" for -# both 4.4 and 4.6 while its indexes were dropping 2400+ packages as missing. -# -# Per-minor objects are therefore excluded here. Presence in a specific minor -# is decided downstream, where the running R version is known: build-all.R -# filters on it, and `build_binary_package()` checks the per-minor path per -# package and skips what is already there. -# -# Archive/ is kept. Those are versions that were built and then superseded; -# dropping them would make every archived version look unbuilt. -per_minor_object <- grepl("/[0-9]+\\.[0-9]+/[^/]+$", s3_pkgs) -if (any(per_minor_object)) { - cat(sprintf( - "Excluding %d per-minor object(s) from the presence check; those are decided per pass\n", - sum(per_minor_object) - )) -} -file_names <- basename(s3_pkgs[!per_minor_object]) +file_names <- basename(s3_pkgs) # An object occupying a key is not proof a binary was built: a package whose # build failed has its CRAN source published under exactly that name. Left in @@ -200,22 +152,11 @@ s3_dt <- data.table( ### Get all packages with build errors -# Scoped to the R minor this snapshot is computed under. A failure is a fact -# about one interpreter: without the scope a package that failed under the -# primary minor is dropped from the candidate list for every other minor too, -# which is the same omission fixed in local/build-all.R and in bincraft's -# check_package_error(). -snapshot_r_minor <- paste( - R.version$major, - strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], - sep = "." -) sql_query <- paste0( # nolint "SELECT error_occurred FROM ", "single_builds", - " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4", - " AND substring(r_version from '^[0-9]+[.][0-9]+') = $5" + " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4" ) # Function to query for a single package-version query_error <- function(pkg, ver) { @@ -223,7 +164,7 @@ query_error <- function(pkg, ver) { ~ DBI::dbGetQuery( con, sql_query, - params = list(pkg, ver, platform, arch, snapshot_r_minor) + params = list(pkg, ver, platform, arch) ), rate = purrr::rate_backoff( pause_base = 1L, diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 6c07eef..648dfc3 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -47,31 +47,12 @@ resolve_zone_id() { fi response_file=$(mktemp) - local status - status=$( - curl -sS -o "${response_file}" -w '%{http_code}' \ - -H "AccessKey: ${api_key}" \ - "https://api.bunny.net/pullzone?perPage=1000" - ) - - if [[ "${status}" != "200" ]]; then - echo "Listing BunnyCDN pull zones failed with HTTP ${status}:" >&2 - head -c 500 "${response_file}" >&2 - echo >&2 - rm -f "${response_file}" - exit 1 - fi - - # The endpoint answers with a bare array on some accounts and a paginated - # object on others. `.Items // .` looks like it covers both but does not: - # indexing an array with a string is an *error*, and `//` only substitutes - # for null, so the array case aborted with - # "Cannot index array with string" and the zone was never purged. + curl -sS -o "${response_file}" \ + -H "AccessKey: ${api_key}" \ + "https://api.bunny.net/pullzone" zone_id=$( jq -r --arg hostname "${zone}" \ - '(if type == "object" then (.Items // []) else . end)[] - | select(any(.Hostnames[]?; .Value == $hostname)) - | .Id' \ + '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \ "${response_file}" ) rm -f "${response_file}" @@ -81,12 +62,6 @@ resolve_zone_id() { exit 1 fi - # Two zones sharing a hostname would purge only whichever jq emitted first. - if [[ $(wc -l <<<"${zone_id}") -gt 1 ]]; then - echo "Hostname ${zone} matched multiple pull zones: ${zone_id//$'\n'/ }" >&2 - exit 1 - fi - echo "${zone_id}" } diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh index 190245a..3e1a7e4 100755 --- a/scripts/verify-r-minor-routing.sh +++ b/scripts/verify-r-minor-routing.sh @@ -43,13 +43,10 @@ MINORS=${MINORS:-"4.4 4.5 4.6"} EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"} # How many Path: targets to HEAD-check per slot/minor. 0 disables. SAMPLE=${SAMPLE:-5} -# Package-count shortfall against the best minor on the same slot, above which -# coverage is reported as uneven. Reported, not failed on: the packages a -# non-primary minor lacks are ABI-risky ones built under the primary minor, -# which a client on another minor cannot safely load anyway, so their absence -# is correct. This gates the *claim* ("full coverage for ABI-sensitive -# packages"), not whether routing is safe to enable - MAX_REGRESSIONS does -# that. +# Largest package-count shortfall a non-primary minor may have against the best +# minor on the same slot before coverage counts as uneven. A slot built under +# one R minor carries fewer per-minor binaries for the others; until that gap +# closes, "full coverage for ABI-sensitive packages" is not a claim we can make. PARITY_TOLERANCE=${PARITY_TOLERANCE:-25} # Source fallbacks are reported, not failed on. Since bincraft learned to keep # a matching-minor generic binary out of a fallback's shadow, a remaining @@ -104,15 +101,10 @@ fetch() { return 0 fi local status - # -L: the router answers an index request with a redirect, so the bytes a - # client ends up with are only visible by following it. - # - # no-cache: a purge is asynchronous, so a run started right after a reindex - # otherwise measures whatever the edge still holds. if [ -n "$ua" ]; then - status=$(curl -sSL -A "$ua" -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) else - status=$(curl -sSL -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) fi echo "$status" > "$dest.status" echo "$status" @@ -140,23 +132,6 @@ fallback_counts() { ' } -# Packages this index serves from the generic slot with a binary built under a -# different R minor, while some other per-minor slot carries a build of them - -# which proves the ABI classifier called them risky. Serving those is the -# load-time crash the per-minor slots exist to prevent. bincraft drops them at -# index time, so a non-zero count means the slot has not been reindexed since -# that guard shipped. -abi_unsafe_count() { - local minor_file=$1 minor=$2 risky_file=$3 - gunzip -c "$minor_file" 2>/dev/null | awk -v m="$minor" ' - /^Package:/ { pkg = $2; path = ""; built = "" } - /^Path:/ { path = $2 } - /^Built:/ { built = $2 " " $3 } - /^$/ { if (pkg != "" && path == "" && built != "" && built !~ ("^R " m "\\.")) print pkg; pkg = "" } - ' | sort -u > "$minor_file.mismatched" - comm -12 "$minor_file.mismatched" "$risky_file" | wc -l -} - # How many packages a client of would receive as source through # per-minor routing while the generic slot holds a binary built under that very # minor. Zero is the bar for enabling a slot. @@ -236,25 +211,12 @@ for arch in $ARCHES; do minor_count=$(wc -l < "$minor_file.names") # Union property: nothing the flat index carries may be missing here. - # bincraft deliberately drops an ABI-risky package whose only binary was - # built under another R minor: serving it is the load-time crash the - # per-minor slots exist to prevent. Those absences are correct. - # - # What must never go missing is a generic package built under *this* - # minor, which is safe to serve and has no reason to disappear. - comm -23 "$flat_file.names" "$minor_file.names" > "$minor_file.absent" - absent_count=$(wc -l < "$minor_file.absent") - gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" ' - /^Package:/ { pkg = $2; built = "" } - /^Built:/ { built = $2 " " $3 } - /^$/ { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" } - ' | sort -u > "$minor_file.flatsame" - lost=$(comm -12 "$minor_file.absent" "$minor_file.flatsame" | wc -l) - - if [ "${lost:-0}" -ne 0 ]; then - bad "$slot R $minor index dropped $lost generic package(s) built under R $minor, which were safe to serve" + missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5) + missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l) + if [ "$missing_count" -ne 0 ]; then + bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))" else - ok "$slot R $minor index: $minor_count packages, union holds ($absent_count ABI-unsafe dropped)" + ok "$slot R $minor index: $minor_count packages, union holds" fi # A per-minor entry that is a source fallback resolves fine but makes the @@ -337,37 +299,12 @@ for arch in $ARCHES; do [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap" done if [ -n "$uneven" ]; then - printf ' note: %s coverage uneven across minors (vs best %s):%s\n' \ - "$slot" "$best" "$uneven" + bad "$slot coverage uneven across minors (vs best $best):$uneven" else ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)" fi fi - # Packages carrying a Path in any per-minor index are risky by construction. - : > "$WORK/${arch}-${distro}.risky" - for minor in $MINORS; do - f="$WORK/${arch}-${distro}-${minor}.gz" - [ -s "$f" ] || continue - gunzip -c "$f" 2>/dev/null | awk ' - /^Package:/ { pkg = $2; path = "" } - /^Path:/ { path = $2 } - /^$/ { if (pkg != "" && path != "") print pkg; pkg = "" } - ' >> "$WORK/${arch}-${distro}.risky" - done - sort -u -o "$WORK/${arch}-${distro}.risky" "$WORK/${arch}-${distro}.risky" - - for minor in $MINORS; do - f="$WORK/${arch}-${distro}-${minor}.gz" - [ -s "$f" ] || continue - unsafe=$(abi_unsafe_count "$f" "$minor" "$WORK/${arch}-${distro}.risky") - if [ "${unsafe:-0}" -gt 0 ]; then - bad "$slot R $minor serves $unsafe ABI-risky package(s) built under another R minor - reindex this slot" - else - ok "$slot R $minor serves no ABI-risky package from another minor" - fi - done - # Excluded minors: no published index, and under --live a redirect to CRAN. for minor in $EXCLUDED_MINORS; do ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz"