From cfa552f54e2abfdd06b5ce625fb76b74e05fee80 Mon Sep 17 00:00:00 2001
From: pat-s <patrick.schratz@gmail.com>
Date: Mon, 31 Aug 2026 09:35:28 +0000
Subject: [PATCH] test(verify): gate on regressions against the generic slot,
 not fallback rate

A source-fallback percentage stopped being a readiness signal once
bincraft learned to keep a matching-minor generic binary out of a
fallback's shadow. What survives now is the case where the generic
binary was built under a different minor: unsafe for this client anyway,
so serving source is correct, just slow. Failing on that share would
block slots that are in fact ready - amd64/noble sits at 53% for 4.5 and
4.6 while regressing nobody.

Gate on the thing that decides it instead: packages this client would
receive as source through per-minor routing while the generic slot holds
a binary built under its own minor. That is strictly worse than not
routing, and must be zero. Fallback share is still printed, as context
rather than a verdict.

Measured zero across every reindexed slot and minor.
---
 scripts/verify-r-minor-routing.sh | 51 +++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 10 deletions(-)

diff --git a/scripts/verify-r-minor-routing.sh b/scripts/verify-r-minor-routing.sh
index cad6c38..3e1a7e4 100755
--- a/scripts/verify-r-minor-routing.sh
+++ b/scripts/verify-r-minor-routing.sh
@@ -48,11 +48,17 @@ SAMPLE=${SAMPLE:-5}
 # one R minor carries fewer per-minor binaries for the others; until that gap
 # closes, "full coverage for ABI-sensitive packages" is not a claim we can make.
 PARITY_TOLERANCE=${PARITY_TOLERANCE:-25}
-# Largest share (percent) of per-minor entries that may be source fallbacks
-# rather than binaries. A source fallback resolves with HTTP 200 and installs
-# correctly, but it compiles on the client: routing to it is not the binary
-# service we advertise. Entries without a `Built:` field are source fallbacks.
-SOURCE_FALLBACK_TOLERANCE=${SOURCE_FALLBACK_TOLERANCE:-10}
+# Source fallbacks are reported, not failed on. Since bincraft learned to keep
+# a matching-minor generic binary out of a fallback's shadow, a remaining
+# fallback means the generic slot's binary was built under a *different* minor,
+# which is unsafe for this client anyway: serving source there is correct, just
+# slow. The gate below is what actually matters.
+#
+# A REGRESSION is a package this client would receive as source through
+# per-minor routing but as a binary built under its own minor from the generic
+# slot. That is strictly worse than not routing at all, and must be zero before
+# a slot is added to UNION_SLOTS.
+MAX_REGRESSIONS=${MAX_REGRESSIONS:-0}
 LIVE=0
 
 for arg in "$@"; do
@@ -126,6 +132,25 @@ fallback_counts() {
   '
 }
 
+# How many packages a client of <minor> would receive as source through
+# per-minor routing while the generic slot holds a binary built under that very
+# minor. Zero is the bar for enabling a slot.
+regression_count() {
+  local minor_file=$1 flat_file=$2 minor=$3
+  gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" '
+    /^Package:/ { pkg = $2; built = "" }
+    /^Built:/   { built = $2 " " $3 }
+    /^$/        { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" }
+  ' | sort -u > "$minor_file.flatbin"
+  gunzip -c "$minor_file" 2>/dev/null | awk '
+    /^Package:/ { pkg = $2; path = ""; built = "" }
+    /^Path:/    { path = $2 }
+    /^Built:/   { built = $2 }
+    /^$/        { if (pkg != "" && path != "" && built == "") print pkg; pkg = "" }
+  ' | sort -u > "$minor_file.src"
+  comm -12 "$minor_file.src" "$minor_file.flatbin" | wc -l
+}
+
 # "<Package> <Path>" pairs for entries that carry a Path: field.
 path_entries() {
   gunzip -c "$1" 2>/dev/null | awk '
@@ -200,11 +225,17 @@ for arch in $ARCHES; do
       read -r steered fallbacks <<< "$(fallback_counts "$minor_file")"
       if [ "${steered:-0}" -gt 0 ]; then
         pct=$((fallbacks * 100 / steered))
-        if [ "$pct" -gt "$SOURCE_FALLBACK_TOLERANCE" ]; then
-          bad "$slot R $minor: $fallbacks/$steered per-minor entries are source fallbacks (${pct}%), not binaries"
-        else
-          ok "$slot R $minor: $((steered - fallbacks))/$steered per-minor entries are binaries"
-        fi
+        printf '        note: %s/%s per-minor entries are source fallbacks (%s%%)\n' \
+          "$fallbacks" "$steered" "$pct"
+      fi
+
+      # The gate: nothing may arrive as source here that the generic slot would
+      # have served as a binary built under this same minor.
+      regressions=$(regression_count "$minor_file" "$flat_file" "$minor")
+      if [ "${regressions:-0}" -gt "$MAX_REGRESSIONS" ]; then
+        bad "$slot R $minor: $regressions packages would be served as source but exist as an R $minor binary in the generic slot"
+      else
+        ok "$slot R $minor: no regression against the generic slot"
       fi
 
       # Path: entries steer to per-minor binaries; they must resolve.