test(verify): gate on regressions against the generic slot, not fallback rate #179

Merged
pat-s merged 1 commit from test/regression-gate into main 2026-08-31 09:55:45 +00:00

View file

@ -48,6 +48,17 @@ SAMPLE=${SAMPLE:-5}
# one R minor carries fewer per-minor binaries for the others; until that gap
# closes, "full coverage for ABI-sensitive packages" is not a claim we can make.
PARITY_TOLERANCE=${PARITY_TOLERANCE:-25}
# Source fallbacks are reported, not failed on. Since bincraft learned to keep
# a matching-minor generic binary out of a fallback's shadow, a remaining
# fallback means the generic slot's binary was built under a *different* minor,
# which is unsafe for this client anyway: serving source there is correct, just
# slow. The gate below is what actually matters.
#
# A REGRESSION is a package this client would receive as source through
# per-minor routing but as a binary built under its own minor from the generic
# slot. That is strictly worse than not routing at all, and must be zero before
# a slot is added to UNION_SLOTS.
MAX_REGRESSIONS=${MAX_REGRESSIONS:-0}
LIVE=0
for arg in "$@"; do
@ -108,6 +119,38 @@ pkg_names() {
gunzip -c "$1" 2>/dev/null | awk '/^Package:/ {print $2}' | sort -u
}
# "<steered> <source-fallbacks>" for a per-minor index: how many entries carry a
# Path: field, and how many of those lack a Built: field (i.e. are sources).
fallback_counts() {
gunzip -c "$1" 2>/dev/null | awk '
/^Package:/ { pkg = $2; path = ""; built = "" }
/^Path:/ { path = $2 }
/^Built:/ { built = $2 }
/^$/ { if (pkg != "" && path != "") { n++; if (built == "") s++ } pkg = "" }
END { if (pkg != "" && path != "") { n++; if (built == "") s++ }
printf "%d %d\n", n, s }
'
}
# How many packages a client of <minor> would receive as source through
# per-minor routing while the generic slot holds a binary built under that very
# minor. Zero is the bar for enabling a slot.
regression_count() {
local minor_file=$1 flat_file=$2 minor=$3
gunzip -c "$flat_file" 2>/dev/null | awk -v m="$minor" '
/^Package:/ { pkg = $2; built = "" }
/^Built:/ { built = $2 " " $3 }
/^$/ { if (pkg != "" && built ~ ("^R " m "\\.")) print pkg; pkg = "" }
' | sort -u > "$minor_file.flatbin"
gunzip -c "$minor_file" 2>/dev/null | awk '
/^Package:/ { pkg = $2; path = ""; built = "" }
/^Path:/ { path = $2 }
/^Built:/ { built = $2 }
/^$/ { if (pkg != "" && path != "" && built == "") print pkg; pkg = "" }
' | sort -u > "$minor_file.src"
comm -12 "$minor_file.src" "$minor_file.flatbin" | wc -l
}
# "<Package> <Path>" pairs for entries that carry a Path: field.
path_entries() {
gunzip -c "$1" 2>/dev/null | awk '
@ -176,6 +219,25 @@ for arch in $ARCHES; do
ok "$slot R $minor index: $minor_count packages, union holds"
fi
# A per-minor entry that is a source fallback resolves fine but makes the
# client compile. Routing to a slot that is mostly fallbacks does not
# deliver the binaries we advertise.
read -r steered fallbacks <<< "$(fallback_counts "$minor_file")"
if [ "${steered:-0}" -gt 0 ]; then
pct=$((fallbacks * 100 / steered))
printf ' note: %s/%s per-minor entries are source fallbacks (%s%%)\n' \
"$fallbacks" "$steered" "$pct"
fi
# The gate: nothing may arrive as source here that the generic slot would
# have served as a binary built under this same minor.
regressions=$(regression_count "$minor_file" "$flat_file" "$minor")
if [ "${regressions:-0}" -gt "$MAX_REGRESSIONS" ]; then
bad "$slot R $minor: $regressions packages would be served as source but exist as an R $minor binary in the generic slot"
else
ok "$slot R $minor: no regression against the generic slot"
fi
# Path: entries steer to per-minor binaries; they must resolve.
if [ "$SAMPLE" -gt 0 ]; then
path_entries "$minor_file" > "$minor_file.paths"