test(verify): follow redirects, and allow the guard's deliberate drops (#185)

Two checks that no longer matched the system.

`fetch()` did not pass `-L`, so every `--live` check against a routed index read a 302 body rather than the index a client receives — 48 spurious failures against production. It also did not bypass the edge cache. Both were fixed on the branch behind #180, but that PR merged at `+9/-5`, capturing only the parity commit, so neither reached `main`.

The union check asserted flat ⊆ every per-minor index. Since rpkgs/bincraft#116, that is deliberately false: `amd64/resolute` drops 2228 packages from its 4.6 index because their only binary was built under another R minor. Those absences **are** the fix working.

It now asserts the thing that must hold — no generic package built under *this* minor may go missing — and reports the deliberate drops as context.

Verified against production: `amd64/resolute` goes from 5 failures to 14 passed / 0 failed.

Reviewed-on: #185

scripts/verify-r-minor-routing.sh

@@ -104,10 +104,15 @@ fetch() {
     return 0
   fi
   local status
+  # -L: the router answers an index request with a redirect, so the bytes a
+  # client ends up with are only visible by following it.
+  #
+  # no-cache: a purge is asynchronous, so a run started right after a reindex
+  # otherwise measures whatever the edge still holds.
   if [ -n "$ua" ]; then
-    status=$(curl -sS -A "$ua" -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
+    status=$(curl -sSL -A "$ua" -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
   else
-    status=$(curl -sS -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
+    status=$(curl -sSL -H 'Cache-Control: no-cache' -o "$dest" -w '%{http_code}' --max-time 120 "$url" 2>/dev/null)
   fi
   echo "$status" > "$dest.status"
   echo "$status"
@@ -135,6 +140,23 @@ fallback_counts() {
   '
 }
 
+# Packages this index serves from the generic slot with a binary built under a
+# different R minor, while some other per-minor slot carries a build of them -
+# which proves the ABI classifier called them risky. Serving those is the
+# load-time crash the per-minor slots exist to prevent. bincraft drops them at
+# index time, so a non-zero count means the slot has not been reindexed since
+# that guard shipped.
+abi_unsafe_count() {
+  local minor_file=$1 minor=$2 risky_file=$3
+  gunzip -c "$minor_file" 2>/dev/null | awk -v m="$minor" '
+    /^Package:/ { pkg = $2; path = ""; built = "" }
+    /^Path:/    { path = $2 }
+    /^Built:/   { built = $2 " " $3 }
+    /^$/        { if (pkg != "" && path == "" && built != "" && built !~ ("^R " m "\\.")) print pkg; pkg = "" }
+  ' | sort -u > "$minor_file.mismatched"
+  comm -12 "$minor_file.mismatched" "$risky_file" | wc -l
+}
+
 # How many packages a client of <minor> would receive as source through
 # per-minor routing while the generic slot holds a binary built under that very
 # minor. Zero is the bar for enabling a slot.
@@ -214,12 +236,25 @@ for arch in $ARCHES; do
       minor_count=$(wc -l < "$minor_file.names")
 
       # Union property: nothing the flat index carries may be missing here.
-      missing=$(comm -23 "$flat_file.names" "$minor_file.names" | head -5)
-      missing_count=$(comm -23 "$flat_file.names" "$minor_file.names" | wc -l)
-      if [ "$missing_count" -ne 0 ]; then
-        bad "$slot R $minor index is not a union: $missing_count flat packages absent (e.g. $(echo "$missing" | tr '\n' ' '))"
+      # bincraft deliberately drops an ABI-risky package whose only binary was
+      # built under another R minor: serving it is the load-time crash the
+      # per-minor slots exist to prevent. Those absences are correct.
+      #
+      # What must never go missing is a generic package built under *this*
+      # minor, which is safe to serve and has no reason to disappear.
+      comm -23 "$flat_file.names" "$minor_file.names" > "$minor_file.absent"
+      absent_count=$(wc -l < "$minor_file.absent")
+      gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" '
+        /^Package:/ { pkg = $2; built = "" }
+        /^Built:/   { built = $2 " " $3 }
+        /^$/        { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" }
+      ' | sort -u > "$minor_file.flatsame"
+      lost=$(comm -12 "$minor_file.absent" "$minor_file.flatsame" | wc -l)
+
+      if [ "${lost:-0}" -ne 0 ]; then
+        bad "$slot R $minor index dropped $lost generic package(s) built under R $minor, which were safe to serve"
       else
-        ok "$slot R $minor index: $minor_count packages, union holds"
+        ok "$slot R $minor index: $minor_count packages, union holds ($absent_count ABI-unsafe dropped)"
       fi
 
       # A per-minor entry that is a source fallback resolves fine but makes the
@@ -309,6 +344,30 @@ for arch in $ARCHES; do
       fi
     fi
 
+    # Packages carrying a Path in any per-minor index are risky by construction.
+    : > "$WORK/${arch}-${distro}.risky"
+    for minor in $MINORS; do
+      f="$WORK/${arch}-${distro}-${minor}.gz"
+      [ -s "$f" ] || continue
+      gunzip -c "$f" 2>/dev/null | awk '
+        /^Package:/ { pkg = $2; path = "" }
+        /^Path:/    { path = $2 }
+        /^$/        { if (pkg != "" && path != "") print pkg; pkg = "" }
+      ' >> "$WORK/${arch}-${distro}.risky"
+    done
+    sort -u -o "$WORK/${arch}-${distro}.risky" "$WORK/${arch}-${distro}.risky"
+
+    for minor in $MINORS; do
+      f="$WORK/${arch}-${distro}-${minor}.gz"
+      [ -s "$f" ] || continue
+      unsafe=$(abi_unsafe_count "$f" "$minor" "$WORK/${arch}-${distro}.risky")
+      if [ "${unsafe:-0}" -gt 0 ]; then
+        bad "$slot R $minor serves $unsafe ABI-risky package(s) built under another R minor - reindex this slot"
+      else
+        ok "$slot R $minor serves no ABI-risky package from another minor"
+      fi
+    done
+
     # Excluded minors: no published index, and under --live a redirect to CRAN.
     for minor in $EXCLUDED_MINORS; do
       ex_url="$BASE/$slot/latest/src/contrib/$minor/PACKAGES.gz"