build-cran-binaries/local/dedupe-audit-issue.R
pat-s 71f3f0c601
Some checks failed
ci/crow/manual/weekly-audit-missing/13 Pipeline was successful
ci/crow/manual/weekly-audit-missing/15 Pipeline was successful
ci/crow/manual/weekly-audit-missing/4 Pipeline was successful
ci/crow/manual/weekly-audit-missing/5 Pipeline was successful
ci/crow/manual/weekly-audit-missing/1 Pipeline was successful
ci/crow/manual/weekly-audit-missing/11 Pipeline was successful
ci/crow/manual/weekly-audit-missing/2 Pipeline was successful
ci/crow/manual/weekly-audit-missing/3 Pipeline was successful
ci/crow/manual/weekly-audit-missing/7 Pipeline was successful
ci/crow/manual/weekly-audit-missing/9 Pipeline was successful
ci/crow/manual/weekly-audit-missing/17 Pipeline was successful
ci/crow/manual/weekly-audit-missing/6 Pipeline was successful
ci/crow/manual/weekly-audit-missing/8 Pipeline was successful
ci/crow/manual/weekly-audit-missing/10 Pipeline was successful
ci/crow/manual/weekly-audit-missing/12 Pipeline was successful
ci/crow/manual/weekly-audit-missing/14 Pipeline was successful
ci/crow/manual/weekly-audit-missing/16 Pipeline was successful
ci/crow/manual/weekly-audit-missing/18 Pipeline was successful
ci/crow/manual/weekly-rebuild-missing/12 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/14 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/16 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/18 Pipeline is pending
ci/crow/manual/weekly-rebuild-missing/1 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/4 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/11 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/15 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/10 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/6 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/8 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/2 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/7 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/5 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/9 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/3 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/17 Pipeline was canceled
ci/crow/manual/weekly-rebuild-missing/13 Pipeline was canceled
ci/crow/cron/process-updates/15 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline failed
ci/crow/cron/process-updates/18 Pipeline failed
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline failed
ci/crow/cron/process-updates/17 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/1 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/2 Pipeline failed
ci/crow/cron/process-updates/6 Pipeline failed
ci/crow/cron/process-updates/4 Pipeline was canceled
ci/crow/cron/process-updates/13 Pipeline was canceled
ci/crow/cron/process-updates/8 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
fix(audit): replace arch subsection in place instead of appending duplicates (#138)
## Problem

A fresh `weekly-audit-missing` run followed by `weekly-rebuild-missing` reported that almost every package "already exists in the remote bucket", even though the audit had just flagged them as missing.

Root cause: the audit's Forgejo-issue update matched the existing `### <arch>` subsection by its **bare** header (`### arm64`) while writing headers with a ` (N missing, M to rebuild)` suffix.
The equality test never matched, so every run **appended** a new block instead of replacing the old one.
Issue #63 had accumulated 38 arch subsections under `## alpine-323` where 2 are expected (75 total across the alpine platforms; body ~77k lines).

`fetch-rebuild-packages-from-issue.R` reads the **first** matching block, which was the oldest snapshot.
So the rebuild kept re-checking a months-old list (180 packages, mostly already built), while the genuinely-missing packages, ~1921 for alpine-323/arm64 in the freshest block, were never fed to the rebuild and the backlog grew silently.

## Changes

- **`local/weekly-missing-binaries-audit.R`**: match arch subsections by prefix (`^### <arch>( |$)`) and remove **all** blocks for that arch before writing one fresh block. Accumulation now self-heals on every run.
- **`local/dedupe-audit-issue.R`** (new): one-off cleanup that collapses each `## platform` section to the freshest block per arch across the three OS-family issues. Supports `DRY_RUN=1`.
- **`local/fetch-rebuild-packages-from-issue.R`**: prefer the audit's freshly-written RDS (overwritten each run, immune to issue-body drift), falling back to issue parsing when absent.

## Validation

Simulated the dedupe logic against the live #63 body: **75 → 8** arch subsections, body 77k → 28k lines, and the kept alpine-323/arm64 block correctly resolves to `GARCH.X (3.0)` (the stale first block held `2.0`).

## Follow-ups (not in this PR)

- `alpine-321` is audited but has no row in the rebuild matrix (7,230 missing, never rebuilt).
- An `alpine-324` section exists in the issue but is in neither matrix.

Reviewed-on: #138
2026-07-21 08:51:23 +00:00

150 lines
4.5 KiB
R

# One-off maintenance: collapse the duplicate arch subsections that accumulated
# in the "Missing package binaries for latest version (<family>)" issues.
#
# A bug in weekly-missing-binaries-audit.R matched the existing "### <arch>"
# subsection by its bare header while writing headers with a
# " (N missing, M to rebuild)" suffix, so every audit run appended a fresh block
# instead of replacing it. This script rewrites each "## <platform>" section to
# keep only the *last* (freshest) block per arch. The audit fix prevents further
# accumulation; this cleans up what is already there.
#
# Env: FORGEJO_TOKEN (required). DRY_RUN=1 to preview counts without patching.
library(httr2, quietly = TRUE)
forgejo_base <- "https://git.devxy.io/api/v1"
repo <- "devxy/build-cran-binaries"
token <- Sys.getenv("FORGEJO_TOKEN")
dry_run <- nchar(Sys.getenv("DRY_RUN")) > 0
if (nchar(token) == 0) {
stop("FORGEJO_TOKEN env var is not set")
}
issue_titles <- c(
"Missing package binaries for latest version (Alpine)",
"Missing package binaries for latest version (Ubuntu)",
"Missing package binaries for latest version (Red Hat)"
)
# Collapse one "## <platform>" block: keep only the last block per arch,
# emitted in order of first appearance. `pl[1]` is the "## <platform>" header.
dedupe_platform <- function(pl) {
sub_hdr <- which(grepl("^### ", pl))
if (length(sub_hdr) == 0) {
return(pl)
}
preamble <- pl[seq_len(sub_hdr[1] - 1)]
sub_end <- c(sub_hdr[-1] - 1, length(pl))
blocks <- lapply(seq_along(sub_hdr), function(k) {
pl[seq(sub_hdr[k], sub_end[k])]
})
arches <- vapply(
blocks,
function(b) sub("^### (\\S+).*", "\\1", b[1]),
character(1)
)
# Index of the last block for each arch, kept in first-appearance order.
last_idx <- vapply(
unique(arches),
function(a) max(which(arches == a)),
integer(1)
)
keep <- sort(last_idx)
out <- preamble
for (i in keep) {
out <- c(out, blocks[[i]])
}
out
}
process_issue <- function(title) {
search_url <- sprintf(
"%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50",
forgejo_base,
repo,
utils::URLencode(title, reserved = TRUE)
)
resp <- request(search_url) |>
req_headers(Authorization = paste("token", token)) |>
req_perform()
issues <- resp_body_json(resp, simplifyVector = FALSE)
match_idx <- which(vapply(issues, function(x) x$title, character(1)) == title)
if (length(match_idx) == 0) {
cat(sprintf("[skip] No issue found: %s\n", title))
return(invisible())
}
issue_number <- issues[[match_idx[1]]]$number
body <- issues[[match_idx[1]]]$body
if (is.null(body) || nchar(body) == 0) {
cat(sprintf("[skip] Empty body: #%d %s\n", issue_number, title))
return(invisible())
}
lines <- strsplit(body, "\n", fixed = TRUE)[[1]]
before <- sum(grepl("^### ", lines))
# Split off the "## Excluded packages" footer so it is preserved verbatim.
excl_idx <- which(lines == "## Excluded packages")
footer <- character(0)
if (length(excl_idx) > 0) {
pre_dash <- which(lines == "---" & seq_along(lines) < excl_idx[1])
cut <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_idx[1]
footer <- lines[seq(cut, length(lines))]
lines <- lines[seq_len(cut - 1)]
}
# Platform headers ("## <platform>"); everything before the first is preamble.
plat_idx <- which(grepl("^## ", lines))
if (length(plat_idx) == 0) {
cat(sprintf("[skip] No platform sections: #%d %s\n", issue_number, title))
return(invisible())
}
top <- lines[seq_len(plat_idx[1] - 1)]
plat_end <- c(plat_idx[-1] - 1, length(lines))
new_lines <- top
for (j in seq_along(plat_idx)) {
pl <- lines[seq(plat_idx[j], plat_end[j])]
new_lines <- c(new_lines, dedupe_platform(pl))
}
if (length(footer) > 0) {
new_lines <- c(new_lines, footer)
}
after <- sum(grepl("^### ", new_lines))
cat(sprintf(
"#%d %s: %d -> %d arch subsections%s\n",
issue_number,
title,
before,
after,
if (dry_run) " (dry run, not patched)" else ""
))
if (dry_run) {
return(invisible())
}
patch_url <- sprintf(
"%s/repos/%s/issues/%d",
forgejo_base,
repo,
issue_number
)
request(patch_url) |>
req_headers(
Authorization = paste("token", token),
`Content-Type` = "application/json"
) |>
req_body_json(list(body = paste(new_lines, collapse = "\n"))) |>
req_method("PATCH") |>
req_perform()
cat(sprintf(" patched #%d\n", issue_number))
}
for (t in issue_titles) {
process_issue(t)
}
cat("Done.\n")