diff --git a/local/dedupe-audit-issue.R b/local/dedupe-audit-issue.R new file mode 100644 index 0000000..c78015a --- /dev/null +++ b/local/dedupe-audit-issue.R @@ -0,0 +1,150 @@ +# One-off maintenance: collapse the duplicate arch subsections that accumulated +# in the "Missing package binaries for latest version ()" issues. +# +# A bug in weekly-missing-binaries-audit.R matched the existing "### " +# subsection by its bare header while writing headers with a +# " (N missing, M to rebuild)" suffix, so every audit run appended a fresh block +# instead of replacing it. This script rewrites each "## " section to +# keep only the *last* (freshest) block per arch. The audit fix prevents further +# accumulation; this cleans up what is already there. +# +# Env: FORGEJO_TOKEN (required). DRY_RUN=1 to preview counts without patching. + +library(httr2, quietly = TRUE) + +forgejo_base <- "https://git.devxy.io/api/v1" +repo <- "devxy/build-cran-binaries" +token <- Sys.getenv("FORGEJO_TOKEN") +dry_run <- nchar(Sys.getenv("DRY_RUN")) > 0 + +if (nchar(token) == 0) { + stop("FORGEJO_TOKEN env var is not set") +} + +issue_titles <- c( + "Missing package binaries for latest version (Alpine)", + "Missing package binaries for latest version (Ubuntu)", + "Missing package binaries for latest version (Red Hat)" +) + +# Collapse one "## " block: keep only the last block per arch, +# emitted in order of first appearance. `pl[1]` is the "## " header. +dedupe_platform <- function(pl) { + sub_hdr <- which(grepl("^### ", pl)) + if (length(sub_hdr) == 0) { + return(pl) + } + preamble <- pl[seq_len(sub_hdr[1] - 1)] + sub_end <- c(sub_hdr[-1] - 1, length(pl)) + blocks <- lapply(seq_along(sub_hdr), function(k) { + pl[seq(sub_hdr[k], sub_end[k])] + }) + arches <- vapply( + blocks, + function(b) sub("^### (\\S+).*", "\\1", b[1]), + character(1) + ) + # Index of the last block for each arch, kept in first-appearance order. + last_idx <- vapply( + unique(arches), + function(a) max(which(arches == a)), + integer(1) + ) + keep <- sort(last_idx) + out <- preamble + for (i in keep) { + out <- c(out, blocks[[i]]) + } + out +} + +process_issue <- function(title) { + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(title, reserved = TRUE) + ) + resp <- request(search_url) |> + req_headers(Authorization = paste("token", token)) |> + req_perform() + issues <- resp_body_json(resp, simplifyVector = FALSE) + match_idx <- which(vapply(issues, function(x) x$title, character(1)) == title) + if (length(match_idx) == 0) { + cat(sprintf("[skip] No issue found: %s\n", title)) + return(invisible()) + } + + issue_number <- issues[[match_idx[1]]]$number + body <- issues[[match_idx[1]]]$body + if (is.null(body) || nchar(body) == 0) { + cat(sprintf("[skip] Empty body: #%d %s\n", issue_number, title)) + return(invisible()) + } + + lines <- strsplit(body, "\n", fixed = TRUE)[[1]] + before <- sum(grepl("^### ", lines)) + + # Split off the "## Excluded packages" footer so it is preserved verbatim. + excl_idx <- which(lines == "## Excluded packages") + footer <- character(0) + if (length(excl_idx) > 0) { + pre_dash <- which(lines == "---" & seq_along(lines) < excl_idx[1]) + cut <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_idx[1] + footer <- lines[seq(cut, length(lines))] + lines <- lines[seq_len(cut - 1)] + } + + # Platform headers ("## "); everything before the first is preamble. + plat_idx <- which(grepl("^## ", lines)) + if (length(plat_idx) == 0) { + cat(sprintf("[skip] No platform sections: #%d %s\n", issue_number, title)) + return(invisible()) + } + top <- lines[seq_len(plat_idx[1] - 1)] + plat_end <- c(plat_idx[-1] - 1, length(lines)) + + new_lines <- top + for (j in seq_along(plat_idx)) { + pl <- lines[seq(plat_idx[j], plat_end[j])] + new_lines <- c(new_lines, dedupe_platform(pl)) + } + if (length(footer) > 0) { + new_lines <- c(new_lines, footer) + } + + after <- sum(grepl("^### ", new_lines)) + cat(sprintf( + "#%d %s: %d -> %d arch subsections%s\n", + issue_number, + title, + before, + after, + if (dry_run) " (dry run, not patched)" else "" + )) + + if (dry_run) { + return(invisible()) + } + + patch_url <- sprintf( + "%s/repos/%s/issues/%d", + forgejo_base, + repo, + issue_number + ) + request(patch_url) |> + req_headers( + Authorization = paste("token", token), + `Content-Type` = "application/json" + ) |> + req_body_json(list(body = paste(new_lines, collapse = "\n"))) |> + req_method("PATCH") |> + req_perform() + cat(sprintf(" patched #%d\n", issue_number)) +} + +for (t in issue_titles) { + process_issue(t) +} +cat("Done.\n") diff --git a/local/fetch-rebuild-packages-from-issue.R b/local/fetch-rebuild-packages-from-issue.R index 26e544d..9d8043c 100644 --- a/local/fetch-rebuild-packages-from-issue.R +++ b/local/fetch-rebuild-packages-from-issue.R @@ -4,7 +4,6 @@ forgejo_base <- "https://git.devxy.io/api/v1" repo <- "devxy/build-cran-binaries" platform <- Sys.getenv("PLATFORM") arch <- Sys.getenv("ARCH") -token <- Sys.getenv("FORGEJO_TOKEN") output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") if (nchar(platform) == 0) { @@ -13,6 +12,36 @@ if (nchar(platform) == 0) { if (nchar(arch) == 0) { stop("ARCH env var is not set") } + +# Prefer the audit's freshly-written RDS. The audit overwrites it each run +# (saveRDS), so unlike the Forgejo issue body it is never subject to the +# duplicate-subsection accumulation bug. Fall back to parsing the issue when the +# RDS is absent (e.g. a fresh runner with no shared cache). +rds_file <- file.path( + Sys.getenv("REBUILD_PKG_RDS_DIR", "/mnt/cache/packages"), + sprintf("weekly_rebuild_%s_%s.rds", platform, arch) +) +if (file.exists(rds_file)) { + pkgs <- tryCatch(as.character(readRDS(rds_file)), error = function(e) NULL) + if (!is.null(pkgs) && length(pkgs) > 0) { + cat(sprintf( + "Using audit RDS %s: %d rebuildable packages for %s/%s\n", + rds_file, + length(pkgs), + platform, + arch + )) + writeLines(pkgs, output_file) + cat(sprintf("Wrote package list to %s\n", output_file)) + q("no") + } + cat(sprintf( + "RDS %s present but empty/unreadable -- falling back to issue\n", + rds_file + )) +} + +token <- Sys.getenv("FORGEJO_TOKEN") if (nchar(token) == 0) { stop("FORGEJO_TOKEN env var is not set") } diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index 6b32037..7b0b65d 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -328,7 +328,6 @@ if (nchar(forgejo_token) == 0) { } plat_header <- sprintf("## %s", platform) - arch_header <- sprintf("### %s", arch) plat_idx <- which(lines == plat_header) @@ -359,31 +358,40 @@ if (nchar(forgejo_token) == 0) { } plat_lines <- lines[seq(pi, plat_end)] - arch_local_idx <- which(plat_lines == arch_header) - if (length(arch_local_idx) == 0) { - # Append arch subsection at end of platform block - lines <- c( - lines[seq_len(plat_end)], - "", - arch_lines, - lines[seq(plat_end + 1, length(lines))] - ) + # Arch subsection headers within the platform block (### arm64 / ### amd64). + # Match by prefix: headers carry a " (N missing, M to rebuild)" suffix, so + # exact-equality matching never found the existing block and silently + # appended a duplicate on every run. Remove *all* blocks for this arch + # (collapsing any previously accumulated duplicates), then write one fresh + # block, so the issue holds a single current subsection per arch. + sub_hdr <- which(grepl("^### ", plat_lines)) + arch_re <- sprintf("^### %s( |$)", arch) + + if (length(sub_hdr) == 0) { + # No arch subsections yet -- append after the platform header/preamble. + new_plat_lines <- c(plat_lines, "", arch_lines) } else { - ai <- pi + arch_local_idx[1] - 1 # absolute line index - - # End of arch subsection - next_arch <- which( - grepl("^### |^## |^---", lines) & seq_along(lines) > ai - ) - arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end - - lines <- c( - lines[seq_len(ai - 1)], - arch_lines, - lines[seq(arch_end + 1, length(lines))] - ) + preamble <- plat_lines[seq_len(sub_hdr[1] - 1)] + # Each subsection runs from its ### header to the line before the next + # ### header (#### known-failures stays inside its own block). + sub_end <- c(sub_hdr[-1] - 1, length(plat_lines)) + kept <- character(0) + for (k in seq_along(sub_hdr)) { + block <- plat_lines[seq(sub_hdr[k], sub_end[k])] + if (!grepl(arch_re, block[1])) { + kept <- c(kept, block) + } + } + new_plat_lines <- c(preamble, kept, "", arch_lines) } + + tail_lines <- if (plat_end < length(lines)) { + lines[seq(plat_end + 1, length(lines))] + } else { + character(0) + } + lines <- c(lines[seq_len(pi - 1)], new_plat_lines, tail_lines) } # Rebuild excluded footer