diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index fb1229d..cbbc70d 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -35,6 +35,14 @@ variables: - 'resolute' default: '3.24' R_VERSION: + # The slot's *primary* R minor: what `local/build-all.R` builds into the + # generic slot. The loop below already runs `--sensitive-only` for every + # other installed minor, so filling a non-primary minor's gap needs this + # left alone, not changed. + # + # 4.6.0 was briefly offered here (#183) and removed: selecting it for a + # slot whose generic binaries are 4.5-built would publish 4.6 binaries + # into the generic slot and break every 4.5 client. description: 'Primary R version under /opt/R.' options: - 4.5.3 diff --git a/.crow/reindex.yaml b/.crow/reindex.yaml new file mode 100644 index 0000000..b13245a --- /dev/null +++ b/.crow/reindex.yaml @@ -0,0 +1,193 @@ +# Re-index every slot without rebuilding anything. +# +# `weekly-rebuild-reindex` exists to run after `weekly-rebuild-missing`, so it +# depends on that workflow and shares its gate: triggering it manually also +# starts hours of package rebuilds. That is the wrong tool when only the index +# needs regenerating - after a bincraft release that changes how the index is +# written, for instance, where the objects in the bucket are already correct +# and only `PACKAGES*` is stale. +# +# This workflow does the index half on its own. It installs the latest bincraft +# release, republishes the generic and per-R-minor indexes for each slot, and +# purges the edge. No package is built. +# +# Trigger with the `reindex` variable set to `all` or to a single +# `-`, e.g. +# +# crow pipeline create devxy/build-cran-binaries --var reindex=all + +variables: + # A manual pipeline creation instantiates every file in .crow/, so the + # default must match no matrix row. + reindex: + description: "Re-index target: a specific -, 'all' for every slot, or 'none'." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: manual + evaluate: 'reindex == "all" || reindex == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + PLATFORM: ${OS} + ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + # The staging zone is listed too. It shares the B2 origin, so an index + # it still holds is a stale copy of the same object, and its + # cache_expiration_time is the same ~370 days: without a purge here it + # serves pre-reindex indexes indefinitely and any verification run + # against it measures the past. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' + commands: + - apk add --no-cache -q bash curl jq + # Crow carries the checkout from the re-index step into this step. + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES + # Runs on every row rather than on one designated slot: a cron fires only + # its own slot's row, so gating on a named slot would leave every other + # slot unpurged. A manual "all" run therefore purges the zone 18 times, + # which is a cheap API call and rare. + # + # Run it even when the re-index above failed: the objects were still + # replaced, and a stale edge is exactly what keeps them hidden. + when: + - status: [success, failure] diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml index 2e0f2a0..c9f9a8a 100644 --- a/.crow/weekly-rebuild-reindex.yaml +++ b/.crow/weekly-rebuild-reindex.yaml @@ -175,7 +175,12 @@ steps: from_secret: BUNNYNET_API_KEY # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate # Bunny pull zones, so both must be purged after the shared origin changes. - BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + # The staging zone is listed too. It shares the B2 origin, so an index + # it still holds is a stale copy of the same object, and its + # cache_expiration_time is the same ~370 days: without a purge here it + # serves pre-reindex indexes indefinitely and any verification run + # against it measures the past. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io cran-rpkgs-test.b-cdn.net' commands: - apk add --no-cache -q bash curl jq # Crow carries the checkout from the re-index step into this step. diff --git a/cdn.tf b/cdn.tf index 96877b2..df30273 100644 --- a/cdn.tf +++ b/cdn.tf @@ -52,6 +52,26 @@ ### cran.rpkgs.com +locals { + rpkgs_slots = [ + for pair in setproduct( + ["amd64", "arm64"], + ["resolute", "noble", "jammy", "rhel8", "rhel9", "rhel10", "alpine323", "alpine324"] + ) : "${pair[0]}/${pair[1]}" + ] + + # The supported R minors: the current one plus the two previous, which is + # exactly what build-env-images installs as R_VERSION_LATEST / PREV1 / PREV2. + # These must stay in step. A minor listed here without a published index + # sends those clients to a 404; a published minor missing from this list + # sends them to CRAN for sources instead of serving the binaries we built. + rpkgs_supported_minors = ["4.4", "4.5", "4.6"] + + # bunny.net serves every pull zone on .b-cdn.net, so staging needs no + # DNS record and is never advertised. + rpkgs_test_hostname = "cran-rpkgs-test.b-cdn.net" +} + # The edge middleware that resolves the bare cran.rpkgs.com form to an # / slot and routes PACKAGES* to the per-R-minor slot. The source of # truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release. @@ -69,9 +89,26 @@ resource "bunnynet_compute_script" "rpkgs_router" { # slot that is not listed here would hide every package the per-minor index does # not carry, so this stays empty until a slot has been backfilled. resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + # Enabled. Until this was set, every client resolved against the generic + # index and never reached a per-minor binary: an R 4.6.1 client on resolute + # downloaded the 4.5-built rlang (2079570 bytes) while the correct 4.6 build + # (2075106 bytes) sat unused one directory away, and died at load with + # `undefined symbol: SETLENGTH`. + # + # Verified before enabling, against the staging zone with the same script and + # the same origin: all 16 slots report zero regressions against the generic + # slot, an excluded R minor is sent to CRAN, a client without an R minor + # still gets the generic index, and tarball requests are never rewritten. + default_value = join(",", local.rpkgs_slots) + required = false +} + +resource "bunnynet_compute_script_variable" "rpkgs_router_known_minors" { script = bunnynet_compute_script.rpkgs_router.id - name = "UNION_SLOTS" - default_value = "" + name = "KNOWN_MINORS" + default_value = join(",", local.rpkgs_supported_minors) required = false } @@ -147,6 +184,92 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } +### Staging zone for edge-router changes + +# Every published / slot. The staging zone enables per-minor routing +# for all of them at once; production adopts the same list only after +# `scripts/verify-r-minor-routing.sh --live` passes against staging. + +# A second copy of the same router, bound to the same B2 origin, so UNION_SLOTS +# can be exercised end to end before production is touched. +resource "bunnynet_compute_script" "rpkgs_router_test" { + type = "middleware" + name = "rpkgs-router-test" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +resource "bunnynet_compute_script_variable" "rpkgs_router_test_union_slots" { + script = bunnynet_compute_script.rpkgs_router_test.id + name = "UNION_SLOTS" + default_value = join(",", local.rpkgs_slots) + required = false +} + +# Without this the staging zone rewrites to PUBLIC_CDN_ORIGIN, so its redirects +# land on production and the test silently measures the wrong system. +resource "bunnynet_compute_script_variable" "rpkgs_router_test_extra_hosts" { + script = bunnynet_compute_script.rpkgs_router_test.id + name = "EXTRA_PUBLIC_HOSTS" + default_value = local.rpkgs_test_hostname + required = false +} + +resource "bunnynet_compute_script_variable" "rpkgs_router_test_known_minors" { + script = bunnynet_compute_script.rpkgs_router_test.id + name = "KNOWN_MINORS" + default_value = join(",", local.rpkgs_supported_minors) + required = false +} + +resource "bunnynet_pullzone" "cran_rpkgs_test" { + name = "cran-rpkgs-test" + + cache_errors = false + + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + + origin { + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router_test.id + } + + routing { + filters = [ + "scripting", + ] + } + + s3_auth_enabled = true + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" + + cache_enabled = true + request_coalescing_enabled = true + block_post_requests = true + + cache_vary_headers = ["User-Agent"] + + # Staging carries only synthetic verification traffic, so the production + # ceilings would be pure headroom. + limit_requests = 500 + limit_connections = 100 + + safehop_enabled = true + add_canonical_header = true + cache_stale = ["offline", "updating"] + block_ips = var.cdn_block_ips + + # 1 TB + limit_bandwidth = 1000000000000 + + block_root_path = true +} + + # Alliance SwissPass historically used a separate, manually configured pull # zone. Adopt it so both public repositories use the same B2 origin, middleware # release and cache behavior. diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts index 3020b2d..3e2fe8b 100644 --- a/edge/rpkgs-router.test.ts +++ b/edge/rpkgs-router.test.ts @@ -17,6 +17,8 @@ const UNION_SLOTS = 'amd64/alpine324'; const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R43_MUSL = 'R (4.3.3 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R47_MUSL = 'R (4.7.0 x86_64-pc-linux-musl x86_64 linux-musl)'; const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; const UA_R45_RESOLUTE = 'R/4.5.3 (Ubuntu 26.04) (aarch64-unknown-linux-gnu aarch64 linux-gnu)'; const UA_R45_FUTURE_UBUNTU = @@ -96,6 +98,33 @@ Deno.test('rpkgs-router', async (t) => { assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`); }); + // We publish binaries only for the supported window. An excluded minor has + // no slot we can serve safely, so it goes to CRAN for sources rather than + // to a 404 or to binaries built under another minor. + await t.step('sends an excluded R minor to CRAN for the index', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R43_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); + }); + + await t.step('sends a future R minor to CRAN too', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R47_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); + }); + + // The index and the tarballs R resolves against it have to come from the + // same place. Serving one from CRAN and the other from here would hand R a + // binary where it expects a source tarball. + await t.step('sends an excluded minor to CRAN for tarballs as well', async () => { + const res = await probe(`${SLOT}/foo_1.0.tar.gz`, UA_R43_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/foo_1.0.tar.gz'); + }); + + await t.step('leaves an excluded minor alone on a slot outside UNION_SLOTS', async () => { + const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R43_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + await t.step('routes PACKAGES and PACKAGES.rds too', async () => { for (const file of ['PACKAGES', 'PACKAGES.rds']) { const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL); @@ -146,18 +175,12 @@ Deno.test('rpkgs-router', async (t) => { await t.step('resolves Ubuntu 26.04 to the resolute slot', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_RESOLUTE); - assertEquals( - res.location, - 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz', - ); + assertEquals(res.location, 'https://cran.rpkgs.com/arm64/resolute/latest/src/contrib/PACKAGES.gz'); }); await t.step('resolves a future Ubuntu release from its codename', async () => { const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_FUTURE_UBUNTU); - assertEquals( - res.location, - 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz', - ); + assertEquals(res.location, 'https://cran.rpkgs.com/arm64/dynamic-dugong/latest/src/contrib/PACKAGES.gz'); }); await t.step('sends an unidentifiable distro to CRAN', async () => { diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts index cdeac44..05b6e18 100644 --- a/edge/rpkgs-router.ts +++ b/edge/rpkgs-router.ts @@ -27,7 +27,17 @@ import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; const CRAN_ORIGIN = 'https://cran.r-project.org'; -const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']); +const PUBLIC_CDN_HOSTS = new Set([ + 'cran.rpkgs.com', + 'cran.allianceswisspass.devxy.io', + // Staging hostnames, so the identical script can run on a test pull zone and + // redirect within itself. Without this a test zone rewrites to + // PUBLIC_CDN_ORIGIN, quietly exercising production instead of itself. + ...(Deno.env.get('EXTRA_PUBLIC_HOSTS') ?? '') + .split(',') + .map((host) => host.trim()) + .filter((host) => host.length > 0), +]); /** Slots ("/", comma separated) whose per-minor index is a union. */ const UNION_SLOTS = new Set( @@ -37,6 +47,21 @@ const UNION_SLOTS = new Set( .filter((slot) => slot.length > 0), ); +/** + * R minors for which a per-minor index is actually published. + * + * contribPath() has no way to probe the origin, so a minor that is not + * published here must fall back to the flat index. Routing an unlisted minor + * would send that client to a 404 and it would see no packages at all - a + * silent, total failure rather than a degraded one. + */ +const KNOWN_MINORS = new Set( + (Deno.env.get('KNOWN_MINORS') ?? '4.4,4.5,4.6') + .split(',') + .map((minor) => minor.trim()) + .filter((minor) => minor.length > 0), +); + /** `///latest/src/contrib[/]` */ const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/; @@ -96,6 +121,18 @@ function publicCdnOrigin(url: URL): string { return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; } +/** + * True when the client reports an R minor that we deliberately do not serve. + * + * A client that reports no minor at all is not "unsupported": non-R fetchers + * (mirror scripts, image builds) must keep getting the flat slot. Only a + * known-and-excluded minor falls through to CRAN. + */ +function isExcludedMinor(userAgent: string): boolean { + const rMinor = extractRMinor(userAgent); + return rMinor !== null && !KNOWN_MINORS.has(rMinor); +} + function extractRMinor(userAgent: string): string | null { for (const regex of R_MINOR_REGEXES) { const match = userAgent.match(regex); @@ -177,8 +214,8 @@ function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: * The contrib path a request should be served from, relative to the slot. * * Returns the per-minor path for an index file when the slot is known to carry - * a union index and the client's R minor is known; otherwise the flat path, - * which is what every client sees today. + * a union index and the client's R minor is one we publish; otherwise the flat + * path, which is what every client sees today. */ function contribPath(slot: string, rest: string, userAgent: string): string { const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`; @@ -188,7 +225,7 @@ function contribPath(slot: string, rest: string, userAgent: string): string { } const rMinor = extractRMinor(userAgent); - return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; + return rMinor && KNOWN_MINORS.has(rMinor) ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; } BunnySDK.net.http @@ -224,6 +261,16 @@ BunnySDK.net.http return Promise.resolve(ctx.request); } + // An R minor outside the supported window has no binaries we can safely + // serve, so the whole interaction goes to CRAN: the index and the + // tarballs R will resolve against it. Serving the index from CRAN but + // tarballs from here would hand R a binary where it expects a source + // tarball, which fails in a far more confusing way than not being + // served at all. + if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}/src/contrib/${rest}`)); + } + const target = contribPath(slot, rest, userAgent); if (target === path) { return Promise.resolve(ctx.request); @@ -238,6 +285,10 @@ BunnySDK.net.http return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); } + if (UNION_SLOTS.has(slot) && isExcludedMinor(userAgent)) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + const rest = srcContrib ? srcContrib[1] : ''; return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); } diff --git a/local/build-all.R b/local/build-all.R index 37fc593..c4d7f97 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -26,9 +26,50 @@ package_cache_files <- c( "/mnt/cache/packages/r_minor_sensitive_pkgs.rds", "/mnt/cache/packages/s3_cache.rds" ) -if (!all(file.exists(package_cache_files))) { +# Existence is not freshness. The snapshot describes S3 and CRAN state at the +# moment it was written, and the volume is per-agent, so an agent that ran an +# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324 +# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43 +# sensitive packages, while the install-deps step in the very same pipeline had +# just computed 6871 on another agent. +# +# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes +# once per agent and its shards then share the result. Off CI, or when no such +# variable is set, fall back to an age check. +snapshot_id_path <- "/mnt/cache/packages/snapshot.id" +snapshot_ttl_hours <- as.numeric( + Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2") +) +current_snapshot_id <- "" +for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) { + val <- Sys.getenv(v, unset = "") + if (nzchar(val)) { + current_snapshot_id <- paste(v, val, sep = "=") + break + } +} + +snapshot_is_stale <- function() { + if (!all(file.exists(package_cache_files))) { + return(TRUE) + } + if (nzchar(current_snapshot_id)) { + cached <- tryCatch( + readLines(snapshot_id_path, warn = FALSE)[1L], + error = function(e) NA_character_, + warning = function(w) NA_character_ + ) + return(!identical(cached, current_snapshot_id)) + } + age_hours <- as.numeric( + difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours") + ) + isTRUE(age_hours > snapshot_ttl_hours) +} + +if (snapshot_is_stale()) { message( - "Package snapshot missing from cache; recomputing via packages-to-build.R" + "Package snapshot missing or stale; recomputing via packages-to-build.R" ) dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { @@ -42,6 +83,9 @@ if (!all(file.exists(package_cache_files))) { pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" ) + if (nzchar(current_snapshot_id)) { + writeLines(current_snapshot_id, snapshot_id_path) + } message("Package snapshot recomputed.") } @@ -110,10 +154,25 @@ con <- DBI::dbConnect( password = Sys.getenv("PGPASS"), sslmode = "require" ) +# Scope the skip to the R minor this pass is running under. `single_builds` +# records `r_version` per attempt, but querying without it made a non-primary +# pass skip everything the primary pass had already attempted under a different +# minor - so `--sensitive-only` under 4.6 skipped packages that had only ever +# been built for 4.5, and the per-minor slots never filled. That is why +# amd64/resolute served 4000 fewer packages to a 4.6 client than to a 4.5 one. +r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) built <- DBI::dbGetQuery( con, - "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2", - params = list(platform, arch) + paste( + "SELECT name, tag FROM single_builds", + "WHERE platform = $1 AND arch = $2", + "AND substring(r_version from '^[0-9]+[.][0-9]+') = $3" + ), + params = list(platform, arch, r_minor) ) DBI::dbDisconnect(con) before <- nrow(chunk) @@ -121,8 +180,9 @@ chunk <- chunk[ !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] sprintf( - "Skipped %d already-attempted package versions; %d remaining for this job", + "Skipped %d package versions already attempted under R %s; %d remaining for this job", before - nrow(chunk), + r_minor, nrow(chunk) ) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 3b508fb..9cc1ed6 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -68,9 +68,36 @@ archive_versions <- archive_versions[ ] # Now get release versions (assuming cran_release has Package and Version columns) +# +# Packages published in the last few days are held back. `check_for_binary()` +# reads the published version from the `cran` GitHub mirror +# (`GET /repos/cran//commits`), and that mirror lags CRAN: a package that +# has just appeared has no repository there yet. The call then 404s, which is +# permanent, but it is wrapped in `purrr::insistently` and retried ten times +# with a backoff capped at 60s - so one unmirrored package burns about five +# minutes and then aborts the whole shard. +# +# Holding them back costs nothing: the daily update pipeline builds new and +# updated packages anyway, and they arrive here on the next run once the mirror +# has caught up. +mirror_lag_days <- as.numeric( + Sys.getenv("CRAN_MIRROR_LAG_DAYS", unset = "3") +) +published <- as.POSIXct(cran_release$Published, tz = "UTC") +too_recent <- !is.na(published) & + published > (Sys.time() - mirror_lag_days * 86400) +if (any(too_recent)) { + message(sprintf( + "Holding back %d package(s) published in the last %g day(s); the cran GitHub mirror will not have them yet: %s", + sum(too_recent), + mirror_lag_days, + paste(utils::head(cran_release$Package[too_recent], 10L), collapse = ", ") + )) +} + release_versions <- data.table( - Package = cran_release$Package, - Version = as.character(cran_release$Version) + Package = cran_release$Package[!too_recent], + Version = as.character(cran_release$Version[!too_recent]) ) pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE)) @@ -89,7 +116,28 @@ s3_pkgs <- s3fs::s3_dir_ls( recurse = TRUE ) -file_names <- basename(s3_pkgs) +# `recurse = TRUE` walks the per-minor slots as well, and `basename()` throws +# the directory away - so `4.5/curl_1.0.tar.gz` and `curl_1.0.tar.gz` collapse +# to one name and a package present under *any* R minor counts as built for +# *all* of them. The candidate list then prunes exactly the packages a +# per-minor pass exists to build: arm64/alpine324 reported "0 remaining" for +# both 4.4 and 4.6 while its indexes were dropping 2400+ packages as missing. +# +# Per-minor objects are therefore excluded here. Presence in a specific minor +# is decided downstream, where the running R version is known: build-all.R +# filters on it, and `build_binary_package()` checks the per-minor path per +# package and skips what is already there. +# +# Archive/ is kept. Those are versions that were built and then superseded; +# dropping them would make every archived version look unbuilt. +per_minor_object <- grepl("/[0-9]+\\.[0-9]+/[^/]+$", s3_pkgs) +if (any(per_minor_object)) { + cat(sprintf( + "Excluding %d per-minor object(s) from the presence check; those are decided per pass\n", + sum(per_minor_object) + )) +} +file_names <- basename(s3_pkgs[!per_minor_object]) # An object occupying a key is not proof a binary was built: a package whose # build failed has its CRAN source published under exactly that name. Left in @@ -152,11 +200,22 @@ s3_dt <- data.table( ### Get all packages with build errors +# Scoped to the R minor this snapshot is computed under. A failure is a fact +# about one interpreter: without the scope a package that failed under the +# primary minor is dropped from the candidate list for every other minor too, +# which is the same omission fixed in local/build-all.R and in bincraft's +# check_package_error(). +snapshot_r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) sql_query <- paste0( # nolint "SELECT error_occurred FROM ", "single_builds", - " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4" + " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4", + " AND substring(r_version from '^[0-9]+[.][0-9]+') = $5" ) # Function to query for a single package-version query_error <- function(pkg, ver) { @@ -164,7 +223,7 @@ query_error <- function(pkg, ver) { ~ DBI::dbGetQuery( con, sql_query, - params = list(pkg, ver, platform, arch) + params = list(pkg, ver, platform, arch, snapshot_r_minor) ), rate = purrr::rate_backoff( pause_base = 1L, diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh index 648dfc3..6c07eef 100755 --- a/scripts/purge_cdn_zone.sh +++ b/scripts/purge_cdn_zone.sh @@ -47,12 +47,31 @@ resolve_zone_id() { fi response_file=$(mktemp) - curl -sS -o "${response_file}" \ - -H "AccessKey: ${api_key}" \ - "https://api.bunny.net/pullzone" + local status + status=$( + curl -sS -o "${response_file}" -w '%{http_code}' \ + -H "AccessKey: ${api_key}" \ + "https://api.bunny.net/pullzone?perPage=1000" + ) + + if [[ "${status}" != "200" ]]; then + echo "Listing BunnyCDN pull zones failed with HTTP ${status}:" >&2 + head -c 500 "${response_file}" >&2 + echo >&2 + rm -f "${response_file}" + exit 1 + fi + + # The endpoint answers with a bare array on some accounts and a paginated + # object on others. `.Items // .` looks like it covers both but does not: + # indexing an array with a string is an *error*, and `//` only substitutes + # for null, so the array case aborted with + # "Cannot index array with string" and the zone was never purged. zone_id=$( jq -r --arg hostname "${zone}" \ - '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \ + '(if type == "object" then (.Items // []) else . end)[] + | select(any(.Hostnames[]?; .Value == $hostname)) + | .Id' \ "${response_file}" ) rm -f "${response_file}" @@ -62,6 +81,12 @@ resolve_zone_id() { exit 1 fi + # Two zones sharing a hostname would purge only whichever jq emitted first. + if [[ $(wc -l <<<"${zone_id}") -gt 1 ]]; then + echo "Hostname ${zone} matched multiple pull zones: ${zone_id//$'\n'/ }" >&2 + exit 1 + fi + echo "${zone_id}" } diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh new file mode 100755 index 0000000..190245a --- /dev/null +++ b/scripts/verify-r-minor-routing.sh @@ -0,0 +1,434 @@ +#!/usr/bin/env bash +# +# Verify per-R-minor index routing for cran.rpkgs.com across every published +# / slot. +# +# The edge router (edge/rpkgs-router.ts) rewrites PACKAGES* requests to +# `contrib//` when the slot is listed in UNION_SLOTS and the client's +# User-Agent carries an R minor. Two properties have to hold before a slot may +# be added to UNION_SLOTS: +# +# 1. the per-minor index is a UNION of the per-minor and flat slots, so +# routing to it hides nothing the flat index carries; and +# 2. every R minor a client might report resolves to an index that exists, +# because contribPath() does not check existence and has no fallback. +# +# Modes: +# (default) Resolve routing decisions without depending on UNION_SLOTS being +# set. Safe to run before enabling: it reads the per-minor indexes +# directly and reproduces the router's target path. +# --live Additionally drive the real CDN with R User-Agents and assert the +# bytes served match the expected index. Only meaningful once the +# slot is in UNION_SLOTS. +# +# Usage: +# scripts/verify-r-minor-routing.sh +# scripts/verify-r-minor-routing.sh --live +# MINORS="4.4 4.5" SAMPLE=10 scripts/verify-r-minor-routing.sh +# +# Exits non-zero if any check fails. + +set -uo pipefail + +BASE=${BASE:-https://cran.rpkgs.com} +ARCHES=${ARCHES:-"amd64 arm64"} +DISTROS=${DISTROS:-"resolute noble jammy rhel8 rhel9 rhel10 alpine323 alpine324"} +# The supported window: the current R minor plus the two previous, matching +# build-env-images' R_VERSION_LATEST/PREV1/PREV2 and cdn.tf's +# local.rpkgs_supported_minors. Each of these must have a published index. +MINORS=${MINORS:-"4.4 4.5 4.6"} +# Minors we deliberately do not serve. These must have NO published index and, +# once routing is live, must be sent to CRAN for sources rather than 404ing or +# being handed binaries built under another minor. +EXCLUDED_MINORS=${EXCLUDED_MINORS:-"4.3"} +# How many Path: targets to HEAD-check per slot/minor. 0 disables. +SAMPLE=${SAMPLE:-5} +# Package-count shortfall against the best minor on the same slot, above which +# coverage is reported as uneven. Reported, not failed on: the packages a +# non-primary minor lacks are ABI-risky ones built under the primary minor, +# which a client on another minor cannot safely load anyway, so their absence +# is correct. This gates the *claim* ("full coverage for ABI-sensitive +# packages"), not whether routing is safe to enable - MAX_REGRESSIONS does +# that. +PARITY_TOLERANCE=${PARITY_TOLERANCE:-25} +# Source fallbacks are reported, not failed on. Since bincraft learned to keep +# a matching-minor generic binary out of a fallback's shadow, a remaining +# fallback means the generic slot's binary was built under a *different* minor, +# which is unsafe for this client anyway: serving source there is correct, just +# slow. The gate below is what actually matters. +# +# A REGRESSION is a package this client would receive as source through +# per-minor routing but as a binary built under its own minor from the generic +# slot. That is strictly worse than not routing at all, and must be zero before +# a slot is added to UNION_SLOTS. +MAX_REGRESSIONS=${MAX_REGRESSIONS:-0} +LIVE=0 + +for arg in "$@"; do + case "$arg" in + --live) LIVE=1 ;; + -h | --help) + sed -n '2,32p' "$0" + exit 0 + ;; + *) + echo "unknown argument: $arg" >&2 + exit 2 + ;; + esac +done + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +PASS=0 +FAIL=0 +FAILURES="" + +ok() { + PASS=$((PASS + 1)) + printf ' ok %s\n' "$1" +} + +bad() { + FAIL=$((FAIL + 1)) + FAILURES="${FAILURES}\n - $1" + printf ' FAIL %s\n' "$1" +} + +# Fetch a URL into a file, echoing the HTTP status. Cached per URL. +fetch() { + local url=$1 dest=$2 ua=${3:-} + if [ -s "$dest" ]; then + cat "$dest.status" + return 0 + fi + local status + # -L: the router answers an index request with a redirect, so the bytes a + # client ends up with are only visible by following it. + # + # no-cache: a purge is asynchronous, so a run started right after a reindex + # otherwise measures whatever the edge still holds. + if [ -n "$ua" ]; then + status=$(curl -sSL -A "$ua" -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + else + status=$(curl -sSL -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null) + fi + echo "$status" > "$dest.status" + echo "$status" +} + +head_status() { + curl -sS -o /dev/null -w '%{http_code}' -I --max-time 60 "$1" 2>/dev/null +} + +# Package names from a gzipped PACKAGES index, sorted. +pkg_names() { + gunzip -c "$1" 2>/dev/null | awk '/^Package:/ {print $2}' | sort -u +} + +# " " for a per-minor index: how many entries carry a +# Path: field, and how many of those lack a Built: field (i.e. are sources). +fallback_counts() { + gunzip -c "$1" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; path = ""; built = "" } + /^Path:/ { path = $2 } + /^Built:/ { built = $2 } + /^$/ { if (pkg != "" && path != "") { n++; if (built == "") s++ } pkg = "" } + END { if (pkg != "" && path != "") { n++; if (built == "") s++ } + printf "%d %d\n", n, s } + ' +} + +# Packages this index serves from the generic slot with a binary built under a +# different R minor, while some other per-minor slot carries a build of them - +# which proves the ABI classifier called them risky. Serving those is the +# load-time crash the per-minor slots exist to prevent. bincraft drops them at +# index time, so a non-zero count means the slot has not been reindexed since +# that guard shipped. +abi_unsafe_count() { + local minor_file=$1 minor=$2 risky_file=$3 + gunzip -c "$minor_file" 2>/dev/null | awk -v m="$minor" ' + /^Package:/ { pkg = $2; path = ""; built = "" } + /^Path:/ { path = $2 } + /^Built:/ { built = $2 " " $3 } + /^$/ { if (pkg != "" && path == "" && built != "" && built !~ ("^R " m "\\.")) print pkg; pkg = "" } + ' | sort -u > "$minor_file.mismatched" + comm -12 "$minor_file.mismatched" "$risky_file" | wc -l +} + +# How many packages a client of would receive as source through +# per-minor routing while the generic slot holds a binary built under that very +# minor. Zero is the bar for enabling a slot. +regression_count() { + local minor_file=$1 flat_file=$2 minor=$3 + gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" ' + /^Package:/ { pkg = $2; built = "" } + /^Built:/ { built = $2 " " $3 } + /^$/ { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" } + ' | sort -u > "$minor_file.flatbin" + gunzip -c "$minor_file" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; path = ""; built = "" } + /^Path:/ { path = $2 } + /^Built:/ { built = $2 } + /^$/ { if (pkg != "" && path != "" && built == "") print pkg; pkg = "" } + ' | sort -u > "$minor_file.src" + comm -12 "$minor_file.src" "$minor_file.flatbin" | wc -l +} + +# " " pairs for entries that carry a Path: field. +path_entries() { + gunzip -c "$1" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; ver = ""; path = "" } + /^Version:/ { ver = $2 } + /^Path:/ { path = $2 } + /^$/ { if (pkg != "" && path != "") print pkg, ver, path; pkg = "" } + END { if (pkg != "" && path != "") print pkg, ver, path } + ' +} + +# An R User-Agent of the shape R actually sends. +r_user_agent() { + printf 'R/%s.0 (Ubuntu 24.04; codename=noble) (x86_64-pc-linux-gnu x86_64 linux-gnu)' "$1" +} + +echo "verify-r-minor-routing: $BASE" +echo " slots: $(echo "$ARCHES" | wc -w) arch x $(echo "$DISTROS" | wc -w) os" +echo " minors: $MINORS (excluded: $EXCLUDED_MINORS)" +echo " live: $LIVE" +echo + +for arch in $ARCHES; do + for distro in $DISTROS; do + slot="$arch/$distro" + echo "$slot" + + flat_url="$BASE/$slot/latest/src/contrib/PACKAGES.gz" + flat_file="$WORK/${arch}-${distro}-flat.gz" + flat_status=$(fetch "$flat_url" "$flat_file") + + if [ "$flat_status" != "200" ]; then + bad "$slot flat index unreachable (HTTP $flat_status)" + continue + fi + + pkg_names "$flat_file" > "$flat_file.names" + flat_count=$(wc -l < "$flat_file.names") + if [ "$flat_count" -lt 1000 ]; then + bad "$slot flat index has only $flat_count packages" + continue + fi + ok "$slot flat index: $flat_count packages" + + for minor in $MINORS; do + minor_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" + minor_file="$WORK/${arch}-${distro}-${minor}.gz" + minor_status=$(fetch "$minor_url" "$minor_file") + + # A minor the router would route to must exist, or clients on that R + # version get a 404 and see no packages at all. + if [ "$minor_status" != "200" ]; then + bad "$slot R $minor index missing (HTTP $minor_status) - routing would 404 for R $minor clients" + continue + fi + + pkg_names "$minor_file" > "$minor_file.names" + minor_count=$(wc -l < "$minor_file.names") + + # Union property: nothing the flat index carries may be missing here. + # bincraft deliberately drops an ABI-risky package whose only binary was + # built under another R minor: serving it is the load-time crash the + # per-minor slots exist to prevent. Those absences are correct. + # + # What must never go missing is a generic package built under *this* + # minor, which is safe to serve and has no reason to disappear. + comm -23 "$flat_file.names" "$minor_file.names" > "$minor_file.absent" + absent_count=$(wc -l < "$minor_file.absent") + gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" ' + /^Package:/ { pkg = $2; built = "" } + /^Built:/ { built = $2 " " $3 } + /^$/ { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" } + ' | sort -u > "$minor_file.flatsame" + lost=$(comm -12 "$minor_file.absent" "$minor_file.flatsame" | wc -l) + + if [ "${lost:-0}" -ne 0 ]; then + bad "$slot R $minor index dropped $lost generic package(s) built under R $minor, which were safe to serve" + else + ok "$slot R $minor index: $minor_count packages, union holds ($absent_count ABI-unsafe dropped)" + fi + + # A per-minor entry that is a source fallback resolves fine but makes the + # client compile. Routing to a slot that is mostly fallbacks does not + # deliver the binaries we advertise. + read -r steered fallbacks <<< "$(fallback_counts "$minor_file")" + if [ "${steered:-0}" -gt 0 ]; then + pct=$((fallbacks * 100 / steered)) + printf ' note: %s/%s per-minor entries are source fallbacks (%s%%)\n' \ + "$fallbacks" "$steered" "$pct" + fi + + # The gate: nothing may arrive as source here that the generic slot would + # have served as a binary built under this same minor. + regressions=$(regression_count "$minor_file" "$flat_file" "$minor") + if [ "${regressions:-0}" -gt "$MAX_REGRESSIONS" ]; then + bad "$slot R $minor: $regressions packages would be served as source but exist as an R $minor binary in the generic slot" + else + ok "$slot R $minor: no regression against the generic slot" + fi + + # Path: entries steer to per-minor binaries; they must resolve. + if [ "$SAMPLE" -gt 0 ]; then + path_entries "$minor_file" > "$minor_file.paths" + total_paths=$(wc -l < "$minor_file.paths") + broken=0 + checked=0 + while read -r pkg ver path; do + [ -z "${pkg:-}" ] && continue + tarball="$BASE/$slot/latest/src/contrib/$path/${pkg}_${ver}.tar.gz" + status=$(head_status "$tarball") + checked=$((checked + 1)) + if [ "$status" != "200" ]; then + broken=$((broken + 1)) + [ "$broken" -le 2 ] && printf ' broken target: %s (HTTP %s)\n' "$tarball" "$status" + fi + done < <(shuf -n "$SAMPLE" "$minor_file.paths" 2>/dev/null || head -n "$SAMPLE" "$minor_file.paths") + + if [ "$broken" -ne 0 ]; then + bad "$slot R $minor: $broken/$checked sampled Path: targets do not resolve (of $total_paths total)" + elif [ "$checked" -gt 0 ]; then + ok "$slot R $minor: $checked/$checked sampled Path: targets resolve (of $total_paths total)" + fi + fi + + # Live routing: what a real R client on this minor actually receives. + if [ "$LIVE" -eq 1 ]; then + ua=$(r_user_agent "$minor") + live_file="$WORK/${arch}-${distro}-${minor}-live.gz" + live_status=$(fetch "$flat_url" "$live_file" "$ua") + if [ "$live_status" != "200" ]; then + bad "$slot R $minor live request failed (HTTP $live_status)" + elif cmp -s "$live_file" "$minor_file"; then + ok "$slot R $minor live request served the per-minor index" + elif cmp -s "$live_file" "$flat_file"; then + bad "$slot R $minor live request served the FLAT index - slot not in UNION_SLOTS?" + else + bad "$slot R $minor live request served neither the per-minor nor the flat index" + fi + fi + done + + # Coverage parity across minors. The union property only guarantees no + # client loses packages relative to the flat index; it says nothing about a + # 4.4 client seeing fewer packages than a 4.5 client on the same slot. + best=0 + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz.names" + [ -s "$f" ] || continue + c=$(wc -l < "$f") + [ "$c" -gt "$best" ] && best=$c + done + if [ "$best" -gt 0 ]; then + uneven="" + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz.names" + [ -s "$f" ] || continue + c=$(wc -l < "$f") + gap=$((best - c)) + [ "$gap" -gt "$PARITY_TOLERANCE" ] && uneven="$uneven R$minor:-$gap" + done + if [ -n "$uneven" ]; then + printf ' note: %s coverage uneven across minors (vs best %s):%s\n' \ + "$slot" "$best" "$uneven" + else + ok "$slot coverage parity across minors (best $best, all within $PARITY_TOLERANCE)" + fi + fi + + # Packages carrying a Path in any per-minor index are risky by construction. + : > "$WORK/${arch}-${distro}.risky" + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz" + [ -s "$f" ] || continue + gunzip -c "$f" 2>/dev/null | awk ' + /^Package:/ { pkg = $2; path = "" } + /^Path:/ { path = $2 } + /^$/ { if (pkg != "" && path != "") print pkg; pkg = "" } + ' >> "$WORK/${arch}-${distro}.risky" + done + sort -u -o "$WORK/${arch}-${distro}.risky" "$WORK/${arch}-${distro}.risky" + + for minor in $MINORS; do + f="$WORK/${arch}-${distro}-${minor}.gz" + [ -s "$f" ] || continue + unsafe=$(abi_unsafe_count "$f" "$minor" "$WORK/${arch}-${distro}.risky") + if [ "${unsafe:-0}" -gt 0 ]; then + bad "$slot R $minor serves $unsafe ABI-risky package(s) built under another R minor - reindex this slot" + else + ok "$slot R $minor serves no ABI-risky package from another minor" + fi + done + + # Excluded minors: no published index, and under --live a redirect to CRAN. + for minor in $EXCLUDED_MINORS; do + ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz" + ex_status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 60 "$ex_url" 2>/dev/null) + if [ "$ex_status" = "200" ]; then + bad "$slot R $minor is excluded but an index is published - the two lists disagree" + else + ok "$slot R $minor correctly has no published index" + fi + + if [ "$LIVE" -eq 1 ]; then + loc=$(curl -sS -o /dev/null -w '%{redirect_url}' -A "$(r_user_agent "$minor")" \ + --max-time 60 "$flat_url" 2>/dev/null) + case "$loc" in + https://cran.r-project.org/*) + ok "$slot R $minor is sent to CRAN ($loc)" + ;; + "") + bad "$slot R $minor was served directly instead of being sent to CRAN" + ;; + *) + bad "$slot R $minor redirected somewhere unexpected: $loc" + ;; + esac + fi + done + + # A client whose User-Agent carries no R version must keep getting the flat + # index, never a per-minor one. + if [ "$LIVE" -eq 1 ]; then + plain_file="$WORK/${arch}-${distro}-plain.gz" + plain_status=$(fetch "$flat_url" "$plain_file" "curl/8.0.0") + if [ "$plain_status" != "200" ]; then + bad "$slot non-R User-Agent request failed (HTTP $plain_status)" + elif cmp -s "$plain_file" "$flat_file"; then + ok "$slot non-R User-Agent still served the flat index" + else + bad "$slot non-R User-Agent was routed away from the flat index" + fi + + # Tarball requests must never be rewritten into a per-minor directory: + # flat-slot packages do not live there. + sample_pkg=$(gunzip -c "$flat_file" | awk '/^Package:/ {p=$2} /^Version:/ {print p, $2; exit}') + if [ -n "$sample_pkg" ]; then + # shellcheck disable=SC2086 # deliberate split into $1 (package) and $2 (version) + set -- $sample_pkg + tb="$BASE/$slot/latest/src/contrib/${1}_${2}.tar.gz" + tb_status=$(curl -sS -o /dev/null -w '%{http_code}' -A "$(r_user_agent 4.5)" --max-time 60 "$tb" 2>/dev/null) + if [ "$tb_status" = "200" ]; then + ok "$slot tarball request under an R User-Agent still resolves" + else + bad "$slot tarball ${1}_${2}.tar.gz broke under an R User-Agent (HTTP $tb_status)" + fi + fi + fi + done +done + +echo +echo "passed: $PASS failed: $FAIL" +if [ "$FAIL" -ne 0 ]; then + printf 'failures:%b\n' "$FAILURES" + exit 1 +fi