diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index f956f3f..0284e93 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -1,12 +1,13 @@ # Weekly failure-triage proposals (issue #115, steps 3 + 4). # Classifies the recorded `single_builds` failures and: -# 1. posts/updates a Forgejo tracking issue with the auto-proposable registry -# entries (human reviews, trial-builds, and opens the PR -- nothing merges), and -# 2. prints the feedback-loop metrics (signature hit rate, proposed-vs-merged, -# retirement candidates) to the run log. +# 1. posts/updates a Forgejo issue with the auto-proposable registry entries +# (human reviews, trial-builds, and opens the PR -- nothing merges), +# 2. posts/updates a second Forgejo issue with the classifier's blind spots +# (unclassified failures = candidates for new signatures, and groups +# blocked on a dependency), and prints the feedback-loop metrics to the log. # Global across platforms (the classifier groups over all of single_builds), so -# a single job -- no matrix. Clones read-only; the only write is the Forgejo -# issue via FORGEJO_TOKEN. +# a single job -- no matrix. Clones read-only; the only writes are the two +# Forgejo issues via FORGEJO_TOKEN. when: - event: manual - event: cron @@ -36,7 +37,7 @@ steps: - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R + - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: kubernetes: resources: diff --git a/local/patches/README.md b/local/patches/README.md index e875217..992395f 100644 --- a/local/patches/README.md +++ b/local/patches/README.md @@ -86,15 +86,21 @@ Rscript local/trial-build-patch.R ### Feedback loop (step 4) `local/proposal-tracking.R` reports the signature hit rate, proposed-vs-merged status (a proposal counts as merged once its package appears in the registry), and retirement candidates (registry entries whose package no longer appears in any current failure, so the upstream cause was likely fixed). -It is read-only. +It also surfaces the classifier's **blind spots**: the unclassified failures (candidates for a new signature) and the groups blocked on a dependency build, so the unknown buckets get the same visibility as the proposals. +It is read-only on the DB, with an optional Forgejo issue as the only write. ```bash +# Print the metrics + blind spots. PGPASS=... Rscript local/proposal-tracking.R --json metrics.json +# Post/update a "needs signatures" tracking issue with the unclassified failures. +PGPASS=... FORGEJO_TOKEN=... Rscript local/proposal-tracking.R --open-issue ``` +The unclassified groups are the natural place to discover which new signatures are worth adding to `build_signatures()`. The pure metric/ledger helpers live in `local/proposal-tracking-lib.R` and are covered by `local/tests/test-proposal-tracking-lib.R`. ### Scheduled run -`.crow/weekly-patch-proposals.yaml` runs both steps weekly (register the `weekly-patch-proposals` cron in the crow UI): it posts/updates a Forgejo tracking issue with the auto-proposable entries and logs the feedback-loop metrics. -It clones read-only; the only write is the tracking issue. +`.crow/weekly-patch-proposals.yaml` runs both steps weekly (register the `weekly-patch-proposals` cron in the crow UI). +It posts/updates two Forgejo issues -- one with the auto-proposable entries, one with the unclassified/blocked failures -- and logs the feedback-loop metrics. +It clones read-only; the only writes are the two tracking issues. diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R index 2edd9c8..5efc87c 100644 --- a/local/proposal-tracking-lib.R +++ b/local/proposal-tracking-lib.R @@ -131,4 +131,48 @@ retirement_candidates <- function(registry_entries, failing_pkgs) { vapply(keep, function(e) as.character(e$package), character(1L)) } +# Discovery view: the failure groups the classifier could NOT auto-propose, so +# they get the same visibility as proposals. `report` is a build_triage_report +# result. Returns the unclassified groups (unknown signature -> candidates for a +# new signature) ranked by build count, capped to `max_groups`, each with up to +# `max_pkgs` example packages. `dropped_groups`/`packages_truncated` record any +# cap so nothing is silently hidden. +unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) { + un <- Filter(function(g) !isTRUE(g$matched), report) + un <- un[order(-vapply(un, function(g) g$build_count, integer(1L)))] + shown <- utils::head(un, max_groups) + groups <- lapply(shown, function(g) { + pkgs <- g$packages + list( + fingerprint = g$fingerprint, + build_count = g$build_count, + n_packages = length(pkgs), + packages = utils::head(pkgs, max_pkgs), + packages_truncated = length(pkgs) > max_pkgs, + platforms = g$platforms + ) + }) + list( + groups = groups, + total_groups = length(un), + dropped_groups = max(0L, length(un) - length(shown)), + total_builds = sum(vapply(un, function(g) g$build_count, integer(1L))) + ) +} + +# Groups blocked on a dependency (a package-specific fix pinned via `applies_to` +# whose dependents merely carry its error): report the dependency + how many +# dependents wait on it, so fixing it once is recognised as clearing the batch. +blocked_summary <- function(report, max_pkgs = 15L) { + bl <- Filter(function(g) !is.null(g$blocked_on), report) + lapply(bl, function(g) { + list( + blocked_on = g$blocked_on, + n_packages = length(g$packages), + packages = utils::head(g$packages, max_pkgs), + packages_truncated = length(g$packages) > max_pkgs + ) + }) +} + `%||%` <- function(a, b) if (is.null(a)) b else a diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R index 60bbd10..4d82a11 100644 --- a/local/proposal-tracking.R +++ b/local/proposal-tracking.R @@ -1,12 +1,16 @@ #!/usr/bin/env Rscript # Feedback loop for the failure classifier (issue #115, step 4): report the -# signature hit rate, proposed-vs-merged status, and registry entries that look -# retirable, so the rule set can improve and stale entries can be pruned. -# Read-only: queries `single_builds` and reads the registry + proposals ledger. +# signature hit rate, proposed-vs-merged status, registry entries that look +# retirable, and -- so the classifier's blind spots get the same visibility as +# its proposals -- the failures it could NOT classify (candidates for new +# signatures) plus the groups blocked on a dependency build. +# Read-only on the DB/registry; the only optional write is the Forgejo issue. # # Usage: -# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] +# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] [--open-issue] +# --open-issue post/update a Forgejo issue listing the unclassified and +# dependency-blocked failures (needs FORGEJO_TOKEN) options(error = function() { cat("ERROR:", geterrmessage(), "\n", file = stdout()) @@ -33,6 +37,7 @@ json_out <- local({ i <- match("--json", args) if (!is.na(i) && i < length(args)) args[[i + 1L]] else NA_character_ }) +do_issue <- "--open-issue" %in% args if (nchar(Sys.getenv("PGPASS")) == 0L) { stop("PGPASS env var is not set; a DB password is required.") @@ -119,12 +124,52 @@ if (length(retire) > 0L) { cat(" (none)\n") } +# --------------------------------------------------------------------------- +# Blind spots: failures the classifier could not auto-propose. +# --------------------------------------------------------------------------- +blocked <- blocked_summary(report) +unmatched <- unclassified_summary(report) + +cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") +if (length(blocked) > 0L) { + for (b in blocked) { + cat(sprintf(" %s: %d dependent(s) waiting\n", b$blocked_on, b$n_packages)) + } +} else { + cat(" (none)\n") +} + +cat(sprintf( + "\nUnclassified failures (candidates for new signatures): %d group(s), %d builds.\n", + unmatched$total_groups, + unmatched$total_builds +)) +cat(strrep("-", 60L), "\n", sep = "") +for (g in unmatched$groups) { + cat(sprintf( + " [%d builds | %d pkgs] %s\n e.g. %s%s\n", + g$build_count, + g$n_packages, + g$fingerprint, + toString(g$packages), + if (isTRUE(g$packages_truncated)) ", ..." else "" + )) +} +if (unmatched$dropped_groups > 0L) { + cat(sprintf( + " (+%d more unclassified group(s) not shown)\n", + unmatched$dropped_groups + )) +} + if (!is.na(json_out)) { jsonlite::write_json( list( signature_hit_rate = hit, proposed_vs_merged = pvm, - retirement_candidates = retire + retirement_candidates = retire, + blocked = blocked, + unclassified = unmatched ), json_out, auto_unbox = TRUE, @@ -133,3 +178,127 @@ if (!is.na(json_out)) { ) cat(sprintf("\nWrote metrics to %s\n", json_out)) } + +# --------------------------------------------------------------------------- +# Optionally publish the blind-spots to a Forgejo tracking issue. +# --------------------------------------------------------------------------- +if (do_issue) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-issue requires FORGEJO_TOKEN.") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + issue_title <- "Unclassified build failures (needs signatures) (#115)" + now <- format(Sys.time(), "%Y-%m-%d %H:%M:%S") + + body_lines <- c( + sprintf("_Generated %s from `single_builds` failures._", now), + "", + "Failures the classifier could **not** auto-propose a fix for.", + "Each unclassified group is a candidate for a new signature in `local/failing-builds-classify.R`; the blocked groups clear once the named dependency builds.", + "" + ) + + body_lines <- c(body_lines, "## Blocked on a dependency", "") + if (length(blocked) > 0L) { + for (b in blocked) { + body_lines <- c( + body_lines, + sprintf( + "- **%s**: %d dependent(s) waiting (e.g. %s%s)", + b$blocked_on, + b$n_packages, + toString(b$packages), + if (isTRUE(b$packages_truncated)) ", ..." else "" + ) + ) + } + } else { + body_lines <- c(body_lines, "_None._") + } + + body_lines <- c( + body_lines, + "", + sprintf( + "## Unclassified failures (%d groups, %d builds)", + unmatched$total_groups, + unmatched$total_builds + ), + "" + ) + if (length(unmatched$groups) > 0L) { + for (g in unmatched$groups) { + body_lines <- c( + body_lines, + sprintf( + "### %d builds / %d pkg(s)", + g$build_count, + g$n_packages + ), + "", + sprintf("Fingerprint: `%s`", g$fingerprint), + sprintf( + "Packages: %s%s", + toString(g$packages), + if (isTRUE(g$packages_truncated)) ", ..." else "" + ), + sprintf("Platforms: %s", toString(g$platforms)), + "" + ) + } + if (unmatched$dropped_groups > 0L) { + body_lines <- c( + body_lines, + sprintf("_(+%d more group(s) not shown.)_", unmatched$dropped_groups) + ) + } + } else { + body_lines <- c(body_lines, "_None -- every failure is classified._") + } + new_body <- paste(body_lines, collapse = "\n") + + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(issue_title, reserved = TRUE) + ) + existing <- httr2::request(search_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + existing, + function(x) identical(x$title, issue_title), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- existing[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("\nUpdated tracking issue #%d.\n", num)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/issues", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = issue_title, body = new_body)) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf("\nOpened tracking issue #%d.\n", created$number)) + } +} diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R index a2fbe05..02a29f1 100644 --- a/local/tests/test-proposal-tracking-lib.R +++ b/local/tests/test-proposal-tracking-lib.R @@ -119,6 +119,49 @@ test_that("proposed_vs_merged marks a package merged once it is registered", { expect_identical(stan$status, "merged") }) +test_that("unclassified_summary ranks unknown groups and caps output", { + failures <- data.frame( + name = c("a", "b", "c", "d", "solo"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = c( + # 4 builds share one unknown fingerprint; 1 build a different unknown. + rep("mystery linker meltdown at stage 3", 4L), + "a totally different unknown boom" + ), + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + s <- unclassified_summary(report, max_groups = 30L, max_pkgs = 2L) + expect_identical(s$total_groups, 2L) + expect_identical(s$total_builds, 5L) + # Largest group first, and its example packages are capped at max_pkgs. + expect_identical(s$groups[[1L]]$build_count, 4L) + expect_length(s$groups[[1L]]$packages, 2L) + expect_true(s$groups[[1L]]$packages_truncated) + + # max_groups cap is reported, not silently dropped. + capped <- unclassified_summary(report, max_groups = 1L) + expect_length(capped$groups, 1L) + expect_identical(capped$dropped_groups, 1L) +}) + +test_that("blocked_summary lists each dependency and its dependent count", { + failures <- data.frame( + name = c("ACEsimFit", "AovBay", "AdaptGauss"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + b <- blocked_summary(report, max_pkgs = 2L) + expect_length(b, 1L) + expect_identical(b[[1L]]$blocked_on, "RcppParallel") + expect_identical(b[[1L]]$n_packages, 3L) + expect_true(b[[1L]]$packages_truncated) +}) + test_that("retirement_candidates flags entries whose package no longer fails", { entries <- list( list(package = "RcppParallel"),