feat(local): report unclassified and dependency-blocked failures for discovery
The proposer issue only lists auto-proposable fixes (currently the TBB signatures), so every failure without a signature -- and every group blocked on a dependency build -- was invisible unless someone read the run log. That hid the classifier's blind spots and, with them, which new signatures are worth adding next. Extend the tracker to surface those blind spots and give them the same visibility as the proposals: - add pure `unclassified_summary()` (unknown-signature groups ranked by build count, capped with an explicit dropped count) and `blocked_summary()` helpers - print both sections in proposal-tracking.R and add a `--open-issue` mode that posts/updates a "Unclassified build failures (needs signatures)" Forgejo issue - run the tracker with --open-issue in the weekly crow pipeline (second issue) - cover the new helpers with tests and document the flow in the patches README
This commit is contained in:
parent
9bc4973159
commit
6ed17f3fb1
1 changed files with 278 additions and 15 deletions
|
|
@ -1,12 +1,16 @@
|
|||
#!/usr/bin/env Rscript
|
||||
|
||||
# Feedback loop for the failure classifier (issue #115, step 4): report the
|
||||
# signature hit rate, proposed-vs-merged status, and registry entries that look
|
||||
# retirable, so the rule set can improve and stale entries can be pruned.
|
||||
# Read-only: queries `single_builds` and reads the registry + proposals ledger.
|
||||
# signature hit rate, proposed-vs-merged status, registry entries that look
|
||||
# retirable, and -- so the classifier's blind spots get the same visibility as
|
||||
# its proposals -- the failures it could NOT classify (candidates for new
|
||||
# signatures) plus the groups blocked on a dependency build.
|
||||
# Read-only on the DB/registry; the only optional write is the Forgejo issue.
|
||||
#
|
||||
# Usage:
|
||||
# PGPASS=... Rscript local/proposal-tracking.R [--json PATH]
|
||||
# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] [--open-issue]
|
||||
# --open-issue post/update a Forgejo issue listing the unclassified and
|
||||
# dependency-blocked failures (needs FORGEJO_TOKEN)
|
||||
|
||||
options(error = function() {
|
||||
cat("ERROR:", geterrmessage(), "\n", file = stdout())
|
||||
|
|
@ -33,6 +37,7 @@ json_out <- local({
|
|||
i <- match("--json", args)
|
||||
if (!is.na(i) && i < length(args)) args[[i + 1L]] else NA_character_
|
||||
})
|
||||
do_issue <- "--open-issue" %in% args
|
||||
|
||||
if (nchar(Sys.getenv("PGPASS")) == 0L) {
|
||||
stop("PGPASS env var is not set; a DB password is required.")
|
||||
|
|
@ -119,12 +124,52 @@ if (length(retire) > 0L) {
|
|||
cat(" (none)\n")
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Blind spots: failures the classifier could not auto-propose.
|
||||
# ---------------------------------------------------------------------------
|
||||
blocked <- blocked_summary(report)
|
||||
unmatched <- unclassified_summary(report)
|
||||
|
||||
cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n")
|
||||
if (length(blocked) > 0L) {
|
||||
for (b in blocked) {
|
||||
cat(sprintf(" %s: %d dependent(s) waiting\n", b$blocked_on, b$n_packages))
|
||||
}
|
||||
} else {
|
||||
cat(" (none)\n")
|
||||
}
|
||||
|
||||
cat(sprintf(
|
||||
"\nUnclassified failures (candidates for new signatures): %d group(s), %d builds.\n",
|
||||
unmatched$total_groups,
|
||||
unmatched$total_builds
|
||||
))
|
||||
cat(strrep("-", 60L), "\n", sep = "")
|
||||
for (g in unmatched$groups) {
|
||||
cat(sprintf(
|
||||
" [%d builds | %d pkgs] %s\n e.g. %s%s\n",
|
||||
g$build_count,
|
||||
g$n_packages,
|
||||
g$fingerprint,
|
||||
toString(g$packages),
|
||||
if (isTRUE(g$packages_truncated)) ", ..." else ""
|
||||
))
|
||||
}
|
||||
if (unmatched$dropped_groups > 0L) {
|
||||
cat(sprintf(
|
||||
" (+%d more unclassified group(s) not shown)\n",
|
||||
unmatched$dropped_groups
|
||||
))
|
||||
}
|
||||
|
||||
if (!is.na(json_out)) {
|
||||
jsonlite::write_json(
|
||||
list(
|
||||
signature_hit_rate = hit,
|
||||
proposed_vs_merged = pvm,
|
||||
retirement_candidates = retire
|
||||
retirement_candidates = retire,
|
||||
blocked = blocked,
|
||||
unclassified = unmatched
|
||||
),
|
||||
json_out,
|
||||
auto_unbox = TRUE,
|
||||
|
|
@ -133,3 +178,127 @@ if (!is.na(json_out)) {
|
|||
)
|
||||
cat(sprintf("\nWrote metrics to %s\n", json_out))
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optionally publish the blind-spots to a Forgejo tracking issue.
|
||||
# ---------------------------------------------------------------------------
|
||||
if (do_issue) {
|
||||
forgejo_token <- Sys.getenv("FORGEJO_TOKEN")
|
||||
if (nchar(forgejo_token) == 0L) {
|
||||
stop("--open-issue requires FORGEJO_TOKEN.")
|
||||
}
|
||||
suppressPackageStartupMessages(library(httr2, quietly = TRUE))
|
||||
forgejo_base <- "https://git.devxy.io/api/v1"
|
||||
repo <- "devxy/build-cran-binaries"
|
||||
issue_title <- "Unclassified build failures (needs signatures) (#115)"
|
||||
now <- format(Sys.time(), "%Y-%m-%d %H:%M:%S")
|
||||
|
||||
body_lines <- c(
|
||||
sprintf("_Generated %s from `single_builds` failures._", now),
|
||||
"",
|
||||
"Failures the classifier could **not** auto-propose a fix for.",
|
||||
"Each unclassified group is a candidate for a new signature in `local/failing-builds-classify.R`; the blocked groups clear once the named dependency builds.",
|
||||
""
|
||||
)
|
||||
|
||||
body_lines <- c(body_lines, "## Blocked on a dependency", "")
|
||||
if (length(blocked) > 0L) {
|
||||
for (b in blocked) {
|
||||
body_lines <- c(
|
||||
body_lines,
|
||||
sprintf(
|
||||
"- **%s**: %d dependent(s) waiting (e.g. %s%s)",
|
||||
b$blocked_on,
|
||||
b$n_packages,
|
||||
toString(b$packages),
|
||||
if (isTRUE(b$packages_truncated)) ", ..." else ""
|
||||
)
|
||||
)
|
||||
}
|
||||
} else {
|
||||
body_lines <- c(body_lines, "_None._")
|
||||
}
|
||||
|
||||
body_lines <- c(
|
||||
body_lines,
|
||||
"",
|
||||
sprintf(
|
||||
"## Unclassified failures (%d groups, %d builds)",
|
||||
unmatched$total_groups,
|
||||
unmatched$total_builds
|
||||
),
|
||||
""
|
||||
)
|
||||
if (length(unmatched$groups) > 0L) {
|
||||
for (g in unmatched$groups) {
|
||||
body_lines <- c(
|
||||
body_lines,
|
||||
sprintf(
|
||||
"### %d builds / %d pkg(s)",
|
||||
g$build_count,
|
||||
g$n_packages
|
||||
),
|
||||
"",
|
||||
sprintf("Fingerprint: `%s`", g$fingerprint),
|
||||
sprintf(
|
||||
"Packages: %s%s",
|
||||
toString(g$packages),
|
||||
if (isTRUE(g$packages_truncated)) ", ..." else ""
|
||||
),
|
||||
sprintf("Platforms: %s", toString(g$platforms)),
|
||||
""
|
||||
)
|
||||
}
|
||||
if (unmatched$dropped_groups > 0L) {
|
||||
body_lines <- c(
|
||||
body_lines,
|
||||
sprintf("_(+%d more group(s) not shown.)_", unmatched$dropped_groups)
|
||||
)
|
||||
}
|
||||
} else {
|
||||
body_lines <- c(body_lines, "_None -- every failure is classified._")
|
||||
}
|
||||
new_body <- paste(body_lines, collapse = "\n")
|
||||
|
||||
search_url <- sprintf(
|
||||
"%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50",
|
||||
forgejo_base,
|
||||
repo,
|
||||
utils::URLencode(issue_title, reserved = TRUE)
|
||||
)
|
||||
existing <- httr2::request(search_url) |>
|
||||
httr2::req_headers(Authorization = paste("token", forgejo_token)) |>
|
||||
httr2::req_perform() |>
|
||||
httr2::resp_body_json(simplifyVector = FALSE)
|
||||
match_idx <- which(vapply(
|
||||
existing,
|
||||
function(x) identical(x$title, issue_title),
|
||||
logical(1L)
|
||||
))
|
||||
if (length(match_idx) > 0L) {
|
||||
num <- existing[[match_idx[1]]]$number
|
||||
httr2::request(sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, num)) |>
|
||||
httr2::req_headers(
|
||||
Authorization = paste("token", forgejo_token),
|
||||
`Content-Type` = "application/json"
|
||||
) |>
|
||||
httr2::req_body_json(list(body = new_body)) |>
|
||||
httr2::req_method("PATCH") |>
|
||||
httr2::req_perform()
|
||||
cat(sprintf("\nUpdated tracking issue #%d.\n", num))
|
||||
} else {
|
||||
created <- httr2::request(sprintf(
|
||||
"%s/repos/%s/issues",
|
||||
forgejo_base,
|
||||
repo
|
||||
)) |>
|
||||
httr2::req_headers(
|
||||
Authorization = paste("token", forgejo_token),
|
||||
`Content-Type` = "application/json"
|
||||
) |>
|
||||
httr2::req_body_json(list(title = issue_title, body = new_body)) |>
|
||||
httr2::req_perform() |>
|
||||
httr2::resp_body_json()
|
||||
cat(sprintf("\nOpened tracking issue #%d.\n", created$number))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue