feat(local): auto-propose registry patches and track the feedback loop (#117)
Implements steps 3 + 4 of #115, building on the classifier merged in #116. Now that bincraft **v4.4.3** applies registry `patch`/`makevars`/`configure_args` to the *target* package build (previously deps-only), a trial patched build is a meaningful acceptance gate, so the "propose" half is viable. ## Step 3 — propose, do not apply - **`local/propose-patches.R`** — for each classified, safe fix affecting a package with no current registry entry, emits a pre-filled `registry.json` entry and validates the candidate set against a *temporary* merged registry (the real one is never touched unless asked). - default: print candidates + validation, **take no action** - `--write`: append entries to `registry.json` + the proposals ledger (you commit + open the PR) - `--open-issue`: post/update a Forgejo tracking issue (reuses the weekly-audit `httr2` + `FORGEJO_TOKEN` pattern) - **`local/trial-build-patch.R`** — isolated bincraft build of one package with the registry applied (no upload/archive/metadata; `patchhash` keeps it out of the real cache). Exit 0/1, so it gates a CI step or manual pre-merge check. The human gate stays: nothing merges. Acceptance = `validate-patches.R` passes (checked automatically) **and** the trial build succeeds. Novel source diffs and unknown signatures are never proposed (they carry `auto = FALSE`). ## Step 4 — feedback loop - **`local/proposal-tracking.R`** (read-only) — signature hit rate (builds/pkgs/addressed/open per signature), proposed-vs-merged (a proposal counts merged once its package is in the registry), and retirement candidates (registry entries whose package no longer fails, i.e. likely fixed upstream). - **`local/proposal-tracking-lib.R`** — the pure metric/ledger helpers. ## Supporting changes - Refactored the classify helpers to expose a pure `build_triage_report()` + a list-returning entry builder; `failing-builds-report.R` now renders from the shared function (no behaviour change). - `validate-patches.R` gains optional `PATCH_DIR`/`REGISTRY_FILE` overrides (backward-compatible) so a candidate registry can be validated in isolation. - Documented the propose/trial-build/tracking workflow in `local/patches/README.md`. ## Verification - 71 unit tests pass (incl. new `test-proposal-tracking-lib.R`) under the Dockerized R 4.5.3 build env. - All pre-commit hooks pass (`air-format`, `validate-patches`, prettier, etc.). - Smoke-tested all three entrypoints end-to-end with a stubbed DB: dry-run, `--write` (produces a registry that passes the canonical validator + a valid ledger, then reverted), and the tracker. Closes #115 Reviewed-on: #117
This commit is contained in:
Patrick Schratz
parent
f11ba7172f
commit
1c297c01bf
1 changed files with 990 additions and 119 deletions
135
local/proposal-tracking.R
Normal file
135
local/proposal-tracking.R
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
#!/usr/bin/env Rscript
|
||||
|
||||
# Feedback loop for the failure classifier (issue #115, step 4): report the
|
||||
# signature hit rate, proposed-vs-merged status, and registry entries that look
|
||||
# retirable, so the rule set can improve and stale entries can be pruned.
|
||||
# Read-only: queries `single_builds` and reads the registry + proposals ledger.
|
||||
#
|
||||
# Usage:
|
||||
# PGPASS=... Rscript local/proposal-tracking.R [--json PATH]
|
||||
|
||||
options(error = function() {
|
||||
cat("ERROR:", geterrmessage(), "\n", file = stdout())
|
||||
q(status = 1)
|
||||
})
|
||||
|
||||
suppressPackageStartupMessages({
|
||||
library(DBI, quietly = TRUE)
|
||||
library(RPostgres, quietly = TRUE)
|
||||
library(jsonlite, quietly = TRUE)
|
||||
})
|
||||
|
||||
script_path <- local({
|
||||
a <- commandArgs(trailingOnly = FALSE)
|
||||
f <- sub("^--file=", "", a[grepl("^--file=", a)])
|
||||
if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_
|
||||
})
|
||||
script_dir <- if (is.na(script_path)) "local" else dirname(script_path)
|
||||
source(file.path(script_dir, "failing-builds-classify.R"))
|
||||
source(file.path(script_dir, "proposal-tracking-lib.R"))
|
||||
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
json_out <- local({
|
||||
i <- match("--json", args)
|
||||
if (!is.na(i) && i < length(args)) args[[i + 1L]] else NA_character_
|
||||
})
|
||||
|
||||
if (nchar(Sys.getenv("PGPASS")) == 0L) {
|
||||
stop("PGPASS env var is not set; a DB password is required.")
|
||||
}
|
||||
|
||||
registry_file <- file.path(script_dir, "patches", "registry.json")
|
||||
ledger_file <- file.path(script_dir, "patches", "proposals-log.json")
|
||||
|
||||
con <- DBI::dbConnect(
|
||||
RPostgres::Postgres(),
|
||||
dbname = "build_metadata",
|
||||
host = "r-binaries.devxy.io",
|
||||
port = 15432,
|
||||
user = "rpkgs",
|
||||
password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
on.exit(DBI::dbDisconnect(con), add = TRUE)
|
||||
|
||||
failures <- DBI::dbGetQuery(
|
||||
con,
|
||||
paste0(
|
||||
"SELECT name, tag, platform, arch, r_version, timestamp, error_text ",
|
||||
"FROM single_builds WHERE error_occurred = TRUE AND removed = FALSE"
|
||||
)
|
||||
)
|
||||
|
||||
existing_entries <- if (file.exists(registry_file)) {
|
||||
jsonlite::fromJSON(registry_file, simplifyVector = FALSE)
|
||||
} else {
|
||||
list()
|
||||
}
|
||||
registered_pkgs <- unique(vapply(
|
||||
existing_entries,
|
||||
function(e) as.character(e$package %||% ""),
|
||||
character(1L)
|
||||
))
|
||||
ledger <- if (file.exists(ledger_file)) {
|
||||
jsonlite::fromJSON(ledger_file, simplifyVector = FALSE)
|
||||
} else {
|
||||
list()
|
||||
}
|
||||
|
||||
report <- build_triage_report(failures, registered_pkgs)
|
||||
failing_pkgs <- unique(unlist(lapply(report, function(g) g$packages)))
|
||||
|
||||
hit <- signature_hit_rate(report, registered_pkgs)
|
||||
pvm <- proposed_vs_merged(ledger, registered_pkgs)
|
||||
retire <- retirement_candidates(existing_entries, failing_pkgs)
|
||||
|
||||
cat(sprintf(
|
||||
"Feedback loop over %d failing builds, %d registry entries, %d ledger records.\n",
|
||||
nrow(failures),
|
||||
length(existing_entries),
|
||||
length(ledger)
|
||||
))
|
||||
|
||||
cat("\nSignature hit rate:\n")
|
||||
cat(strrep("-", 60L), "\n", sep = "")
|
||||
for (h in hit) {
|
||||
cat(sprintf(
|
||||
" %-26s %3d builds | %2d pkgs | %2d addressed | %2d open%s\n",
|
||||
h$signature,
|
||||
h$builds,
|
||||
h$packages,
|
||||
h$addressed,
|
||||
h$open,
|
||||
if (isTRUE(h$auto_proposable)) " [auto]" else ""
|
||||
))
|
||||
}
|
||||
|
||||
cat(sprintf("\nProposed vs merged: %d / %d merged.\n", pvm$merged, pvm$total))
|
||||
for (r in pvm$records) {
|
||||
cat(sprintf(" %-24s %-26s %s\n", r$package, r$signature, r$status))
|
||||
}
|
||||
|
||||
cat(sprintf(
|
||||
"\nRetirement candidates (%d): registry entries with no current failure.\n",
|
||||
length(retire)
|
||||
))
|
||||
if (length(retire) > 0L) {
|
||||
cat(paste0(" - ", retire, "\n"), sep = "")
|
||||
} else {
|
||||
cat(" (none)\n")
|
||||
}
|
||||
|
||||
if (!is.na(json_out)) {
|
||||
jsonlite::write_json(
|
||||
list(
|
||||
signature_hit_rate = hit,
|
||||
proposed_vs_merged = pvm,
|
||||
retirement_candidates = retire
|
||||
),
|
||||
json_out,
|
||||
auto_unbox = TRUE,
|
||||
pretty = TRUE,
|
||||
null = "null"
|
||||
)
|
||||
cat(sprintf("\nWrote metrics to %s\n", json_out))
|
||||
}
|
||||
Loading…
Reference in a new issue