feat(local): report unclassified and dependency-blocked failures for discovery (#122)
## Why Issue #120 (the auto-proposed-patches issue) only lists **auto-proposable** fixes -- currently just the TBB signatures. So a reasonable read of it was "TBB is our only build failure", when in fact three whole categories are simply not shown there: - **Unclassified failures** -- anything that doesn't match a seeded signature is routed to human triage and never appears (we've only seeded TBB and libuv signatures). - **Dependency-blocked failures** -- the ~800 RcppParallel dependents (post #121) are still failing; they only show as a log line. - Human-only signatures (libuv). These blind spots are exactly where the *next* signatures should come from, so they deserve the same visibility as the proposals. ## What Extend the feedback-loop tracker to surface the classifier's blind spots: - **`unclassified_summary()`** -- groups every unknown-signature failure by normalised fingerprint, ranked by build count, capped with an explicit `dropped_groups` count (no silent truncation), each with example packages + platforms. These are the candidates for new `build_signatures()` rules. - **`blocked_summary()`** -- lists each dependency (e.g. RcppParallel) and how many dependents wait on it. - `proposal-tracking.R` prints both sections, and a new **`--open-issue`** mode posts/updates a *"Unclassified build failures (needs signatures) (#115)"* Forgejo issue. - The weekly crow pipeline now runs the tracker with `--open-issue`, so it maintains a second tracking issue alongside the proposals one. Read-only on the DB; the only writes are the two issues. ## Verification - New tests cover `unclassified_summary` (ranking + both caps) and `blocked_summary`. - Tracker smoke with a stubbed DB (proposable + blocked + unclassified mix) prints the hit rate, `Blocked on a dependency: RcppParallel: 2 dependent(s)`, and `Unclassified failures ... [2 builds | 2 pkgs] ld: undefined reference ...`. - Full suite: 95 tests pass; all pre-commit hooks pass (air, prettier, markdownlint, yamllint, validate-patches). Reviewed-on: #122
This commit is contained in:
Patrick Schratz
parent
9bc4973159
commit
21a2fe9e6c
1 changed files with 278 additions and 15 deletions
|
|
@ -1,12 +1,16 @@
|
|||
#!/usr/bin/env Rscript
|
||||
|
||||
# Feedback loop for the failure classifier (issue #115, step 4): report the
|
||||
# signature hit rate, proposed-vs-merged status, and registry entries that look
|
||||
# retirable, so the rule set can improve and stale entries can be pruned.
|
||||
# Read-only: queries `single_builds` and reads the registry + proposals ledger.
|
||||
# signature hit rate, proposed-vs-merged status, registry entries that look
|
||||
# retirable, and -- so the classifier's blind spots get the same visibility as
|
||||
# its proposals -- the failures it could NOT classify (candidates for new
|
||||
# signatures) plus the groups blocked on a dependency build.
|
||||
# Read-only on the DB/registry; the only optional write is the Forgejo issue.
|
||||
#
|
||||
# Usage:
|
||||
# PGPASS=... Rscript local/proposal-tracking.R [--json PATH]
|
||||
# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] [--open-issue]
|
||||
# --open-issue post/update a Forgejo issue listing the unclassified and
|
||||
# dependency-blocked failures (needs FORGEJO_TOKEN)
|
||||
|
||||
options(error = function() {
|
||||
cat("ERROR:", geterrmessage(), "\n", file = stdout())
|
||||
|
|
@ -33,6 +37,7 @@ json_out <- local({
|
|||
i <- match("--json", args)
|
||||
if (!is.na(i) && i < length(args)) args[[i + 1L]] else NA_character_
|
||||
})
|
||||
do_issue <- "--open-issue" %in% args
|
||||
|
||||
if (nchar(Sys.getenv("PGPASS")) == 0L) {
|
||||
stop("PGPASS env var is not set; a DB password is required.")
|
||||
|
|
@ -119,12 +124,52 @@ if (length(retire) > 0L) {
|
|||
cat(" (none)\n")
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Blind spots: failures the classifier could not auto-propose.
|
||||
# ---------------------------------------------------------------------------
|
||||
blocked <- blocked_summary(report)
|
||||
unmatched <- unclassified_summary(report)
|
||||
|
||||
cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n")
|
||||
if (length(blocked) > 0L) {
|
||||
for (b in blocked) {
|
||||
cat(sprintf(" %s: %d dependent(s) waiting\n", b$blocked_on, b$n_packages))
|
||||
}
|
||||
} else {
|
||||
cat(" (none)\n")
|
||||
}
|
||||
|
||||
cat(sprintf(
|
||||
"\nUnclassified failures (candidates for new signatures): %d group(s), %d builds.\n",
|
||||
unmatched$total_groups,
|
||||
unmatched$total_builds
|
||||
))
|
||||
cat(strrep("-", 60L), "\n", sep = "")
|
||||
for (g in unmatched$groups) {
|
||||
cat(sprintf(
|
||||
" [%d builds | %d pkgs] %s\n e.g. %s%s\n",
|
||||
g$build_count,
|
||||
g$n_packages,
|
||||
g$fingerprint,
|
||||
toString(g$packages),
|
||||
if (isTRUE(g$packages_truncated)) ", ..." else ""
|
||||
))
|
||||
}
|
||||
if (unmatched$dropped_groups > 0L) {
|
||||
cat(sprintf(
|
||||
" (+%d more unclassified group(s) not shown)\n",
|
||||
unmatched$dropped_groups
|
||||
))
|
||||
}
|
||||
|
||||
if (!is.na(json_out)) {
|
||||
jsonlite::write_json(
|
||||
list(
|
||||
signature_hit_rate = hit,
|
||||
proposed_vs_merged = pvm,
|
||||
retirement_candidates = retire
|
||||
retirement_candidates = retire,
|
||||
blocked = blocked,
|
||||
unclassified = unmatched
|
||||
),
|
||||
json_out,
|
||||
auto_unbox = TRUE,
|
||||
|
|
@ -133,3 +178,127 @@ if (!is.na(json_out)) {
|
|||
)
|
||||
cat(sprintf("\nWrote metrics to %s\n", json_out))
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optionally publish the blind-spots to a Forgejo tracking issue.
|
||||
# ---------------------------------------------------------------------------
|
||||
if (do_issue) {
|
||||
forgejo_token <- Sys.getenv("FORGEJO_TOKEN")
|
||||
if (nchar(forgejo_token) == 0L) {
|
||||
stop("--open-issue requires FORGEJO_TOKEN.")
|
||||
}
|
||||
suppressPackageStartupMessages(library(httr2, quietly = TRUE))
|
||||
forgejo_base <- "https://git.devxy.io/api/v1"
|
||||
repo <- "devxy/build-cran-binaries"
|
||||
issue_title <- "Unclassified build failures (needs signatures) (#115)"
|
||||
now <- format(Sys.time(), "%Y-%m-%d %H:%M:%S")
|
||||
|
||||
body_lines <- c(
|
||||
sprintf("_Generated %s from `single_builds` failures._", now),
|
||||
"",
|
||||
"Failures the classifier could **not** auto-propose a fix for.",
|
||||
"Each unclassified group is a candidate for a new signature in `local/failing-builds-classify.R`; the blocked groups clear once the named dependency builds.",
|
||||
""
|
||||
)
|
||||
|
||||
body_lines <- c(body_lines, "## Blocked on a dependency", "")
|
||||
if (length(blocked) > 0L) {
|
||||
for (b in blocked) {
|
||||
body_lines <- c(
|
||||
body_lines,
|
||||
sprintf(
|
||||
"- **%s**: %d dependent(s) waiting (e.g. %s%s)",
|
||||
b$blocked_on,
|
||||
b$n_packages,
|
||||
toString(b$packages),
|
||||
if (isTRUE(b$packages_truncated)) ", ..." else ""
|
||||
)
|
||||
)
|
||||
}
|
||||
} else {
|
||||
body_lines <- c(body_lines, "_None._")
|
||||
}
|
||||
|
||||
body_lines <- c(
|
||||
body_lines,
|
||||
"",
|
||||
sprintf(
|
||||
"## Unclassified failures (%d groups, %d builds)",
|
||||
unmatched$total_groups,
|
||||
unmatched$total_builds
|
||||
),
|
||||
""
|
||||
)
|
||||
if (length(unmatched$groups) > 0L) {
|
||||
for (g in unmatched$groups) {
|
||||
body_lines <- c(
|
||||
body_lines,
|
||||
sprintf(
|
||||
"### %d builds / %d pkg(s)",
|
||||
g$build_count,
|
||||
g$n_packages
|
||||
),
|
||||
"",
|
||||
sprintf("Fingerprint: `%s`", g$fingerprint),
|
||||
sprintf(
|
||||
"Packages: %s%s",
|
||||
toString(g$packages),
|
||||
if (isTRUE(g$packages_truncated)) ", ..." else ""
|
||||
),
|
||||
sprintf("Platforms: %s", toString(g$platforms)),
|
||||
""
|
||||
)
|
||||
}
|
||||
if (unmatched$dropped_groups > 0L) {
|
||||
body_lines <- c(
|
||||
body_lines,
|
||||
sprintf("_(+%d more group(s) not shown.)_", unmatched$dropped_groups)
|
||||
)
|
||||
}
|
||||
} else {
|
||||
body_lines <- c(body_lines, "_None -- every failure is classified._")
|
||||
}
|
||||
new_body <- paste(body_lines, collapse = "\n")
|
||||
|
||||
search_url <- sprintf(
|
||||
"%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50",
|
||||
forgejo_base,
|
||||
repo,
|
||||
utils::URLencode(issue_title, reserved = TRUE)
|
||||
)
|
||||
existing <- httr2::request(search_url) |>
|
||||
httr2::req_headers(Authorization = paste("token", forgejo_token)) |>
|
||||
httr2::req_perform() |>
|
||||
httr2::resp_body_json(simplifyVector = FALSE)
|
||||
match_idx <- which(vapply(
|
||||
existing,
|
||||
function(x) identical(x$title, issue_title),
|
||||
logical(1L)
|
||||
))
|
||||
if (length(match_idx) > 0L) {
|
||||
num <- existing[[match_idx[1]]]$number
|
||||
httr2::request(sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, num)) |>
|
||||
httr2::req_headers(
|
||||
Authorization = paste("token", forgejo_token),
|
||||
`Content-Type` = "application/json"
|
||||
) |>
|
||||
httr2::req_body_json(list(body = new_body)) |>
|
||||
httr2::req_method("PATCH") |>
|
||||
httr2::req_perform()
|
||||
cat(sprintf("\nUpdated tracking issue #%d.\n", num))
|
||||
} else {
|
||||
created <- httr2::request(sprintf(
|
||||
"%s/repos/%s/issues",
|
||||
forgejo_base,
|
||||
repo
|
||||
)) |>
|
||||
httr2::req_headers(
|
||||
Authorization = paste("token", forgejo_token),
|
||||
`Content-Type` = "application/json"
|
||||
) |>
|
||||
httr2::req_body_json(list(title = issue_title, body = new_body)) |>
|
||||
httr2::req_perform() |>
|
||||
httr2::resp_body_json()
|
||||
cat(sprintf("\nOpened tracking issue #%d.\n", created$number))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue