feat(local): detect dependency-cascade failures generally, not just RcppParallel
The first trial-build gate run correctly went red: the 10 auto-proposed
tbb-stddef packages (BFpack, BayesERtools, GMLTM, ...) all fail while building
their shared `rstan` dependency, not in their own code -- the `tbb/tbb_stddef.h`
error is in rstan's Module.cpp. So a per-package makevars entry is useless for
them; they are blocked on rstan (which already has an entry). This is the same
cascade the RcppParallel `applies_to` guard catches, but the tbb-stddef
signature is generic and had no such pin.
Add `failing_dependency(error_text, package)`: when the log shows a DIFFERENT
package failed to compile ("Failed to build source package X", "compilation
failed for package 'X'", ...), that package is the real cause. build_triage_report
now blocks any package whose every failing build is such a cascade -- reported
as blocked_on the dependency, never proposed a bogus entry. A package that fails
in its OWN compilation is still proposed. This unifies the applies_to and
data-driven cases into one `blocked_packages` / `blocked_on` model.
- report/proposer/blocked_summary now count the actually-blocked packages
- blocked note shows even when a group also has genuine proposals
- tests cover the rstan-cascade vs own-compile split (with the real log strings)
This commit is contained in:
parent
ff9f5f5177
commit
c98aaa548f
5 changed files with 135 additions and 23 deletions
|
|
@ -155,6 +155,44 @@ fingerprint_error <- function(error_text, package = NULL, max_chars = 200L) {
|
||||||
fp
|
fp
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Dependency-cascade detection
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# If a build's error_text shows the failure was actually in a DIFFERENT package
|
||||||
|
# (a dependency that would not compile), return that dependency's name;
|
||||||
|
# otherwise NA. Used to avoid proposing a per-package fix for a package that
|
||||||
|
# only fails because a shared dependency does not build (e.g. the ~73 Stan
|
||||||
|
# packages that fail while building `rstan`). Generalises the RcppParallel
|
||||||
|
# `applies_to` guard to any dependency named in the log.
|
||||||
|
failing_dependency <- function(error_text, package) {
|
||||||
|
if (length(error_text) == 0L || is.na(error_text) || !nzchar(error_text)) {
|
||||||
|
return(NA_character_)
|
||||||
|
}
|
||||||
|
x <- as.character(error_text)
|
||||||
|
# optional opening quote before the package name: apostrophe, double-quote,
|
||||||
|
# backtick, or curly quotes -- written as \u escapes so the pattern stays
|
||||||
|
# valid UTF-8 regardless of source encoding.
|
||||||
|
q <- "[\u0027\u0022\u0060\u2018\u2019]?"
|
||||||
|
name <- "([A-Za-z][A-Za-z0-9._]+)"
|
||||||
|
# Markers R/pak emit naming the package that actually failed to compile.
|
||||||
|
pats <- c(
|
||||||
|
paste0("compilation failed for package ", q, name),
|
||||||
|
paste0("Failed to build source package ", q, name),
|
||||||
|
paste0("Error in building package ", q, name),
|
||||||
|
paste0("dependenc(?:y|ies) ", q, name, q, "?[^\\n]*not available")
|
||||||
|
)
|
||||||
|
deps <- character(0L)
|
||||||
|
for (p in pats) {
|
||||||
|
hits <- regmatches(x, gregexpr(p, x, perl = TRUE))[[1L]]
|
||||||
|
if (length(hits) > 0L) {
|
||||||
|
deps <- c(deps, sub(p, "\\1", hits, perl = TRUE))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
deps <- sub("[._]+$", "", deps) # drop a trailing sentence period (e.g. "rstan.")
|
||||||
|
deps <- setdiff(unique(deps), package) # a package failing on its OWN code is not a cascade
|
||||||
|
if (length(deps) == 0L) NA_character_ else deps[[1L]]
|
||||||
|
}
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Classification
|
# Classification
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
@ -291,18 +329,38 @@ build_triage_report <- function(
|
||||||
unregistered <- setdiff(pkgs, registered_pkgs)
|
unregistered <- setdiff(pkgs, registered_pkgs)
|
||||||
auto_proposable <- isTRUE(sig$auto) && sig$matched
|
auto_proposable <- isTRUE(sig$auto) && sig$matched
|
||||||
|
|
||||||
# A signature whose fix is package-specific (`applies_to`) may only be
|
# Decide, per package, whether it is genuinely fixable or merely blocked on
|
||||||
# proposed for that package. Everything else matching it is a downstream
|
# a dependency (so a per-package entry would be useless). Two blocking modes:
|
||||||
# failure blocked on that package (e.g. RcppParallel dependents carrying
|
# 1. `applies_to`: a package-specific patch (e.g. RcppParallel's) is only
|
||||||
# RcppParallel's own error text) -- never propose those a bogus entry.
|
# valid for its own package; other matches are downstream of it.
|
||||||
|
# 2. data-driven cascade: the package's error_text shows a *different*
|
||||||
|
# package failed to compile (e.g. the ~73 Stan packages blocked on rstan).
|
||||||
|
# `blocked_map` maps a blocked package -> the dependency it waits on.
|
||||||
|
g$blocked_dep <- vapply(
|
||||||
|
seq_len(nrow(g)),
|
||||||
|
function(i) failing_dependency(g$error_text[[i]], g$name[[i]]),
|
||||||
|
character(1L)
|
||||||
|
)
|
||||||
proposable <- unregistered
|
proposable <- unregistered
|
||||||
blocked_on <- NULL
|
blocked_map <- list()
|
||||||
if (!is.null(sig$applies_to)) {
|
if (!is.null(sig$applies_to)) {
|
||||||
proposable <- intersect(unregistered, sig$applies_to)
|
for (p in setdiff(pkgs, sig$applies_to)) {
|
||||||
downstream <- setdiff(pkgs, sig$applies_to)
|
blocked_map[[p]] <- sig$applies_to
|
||||||
if (length(downstream) > 0L) {
|
|
||||||
blocked_on <- sig$applies_to
|
|
||||||
}
|
}
|
||||||
|
proposable <- intersect(proposable, sig$applies_to)
|
||||||
|
}
|
||||||
|
for (p in proposable) {
|
||||||
|
deps <- g$blocked_dep[g$name == p]
|
||||||
|
if (!any(is.na(deps))) {
|
||||||
|
# every failing build of p is a cascade -> blocked, not fixable here
|
||||||
|
blocked_map[[p]] <- unique(deps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
proposable <- setdiff(proposable, names(blocked_map))
|
||||||
|
blocked_packages <- names(blocked_map)
|
||||||
|
blocked_on <- unique(unlist(blocked_map, use.names = FALSE))
|
||||||
|
if (length(blocked_on) == 0L) {
|
||||||
|
blocked_on <- NULL
|
||||||
}
|
}
|
||||||
|
|
||||||
proposed <- list()
|
proposed <- list()
|
||||||
|
|
@ -326,6 +384,7 @@ build_triage_report <- function(
|
||||||
suggested_fix = sig$fix,
|
suggested_fix = sig$fix,
|
||||||
applies_to = sig$applies_to,
|
applies_to = sig$applies_to,
|
||||||
blocked_on = blocked_on,
|
blocked_on = blocked_on,
|
||||||
|
blocked_packages = blocked_packages,
|
||||||
fingerprint = names(fp_tab)[[1L]],
|
fingerprint = names(fp_tab)[[1L]],
|
||||||
fingerprint_variants = length(fp_tab),
|
fingerprint_variants = length(fp_tab),
|
||||||
build_count = nrow(g),
|
build_count = nrow(g),
|
||||||
|
|
|
||||||
|
|
@ -194,15 +194,17 @@ for (r in report) {
|
||||||
)
|
)
|
||||||
cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "")
|
cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "")
|
||||||
}
|
}
|
||||||
} else if (!is.null(r$blocked_on)) {
|
} else if (r$auto_proposable && length(r$blocked_packages) == 0L) {
|
||||||
|
cat(" (all affected packages already have a registry entry)\n")
|
||||||
|
}
|
||||||
|
# Blocked packages are shown even when the group also has proposals.
|
||||||
|
if (length(r$blocked_packages) > 0L) {
|
||||||
cat(sprintf(
|
cat(sprintf(
|
||||||
" (blocked on %s -- these %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n",
|
" (blocked on %s -- %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n",
|
||||||
toString(r$blocked_on),
|
toString(r$blocked_on),
|
||||||
length(r$packages),
|
length(r$blocked_packages),
|
||||||
toString(r$blocked_on)
|
toString(r$blocked_on)
|
||||||
))
|
))
|
||||||
} else if (r$auto_proposable) {
|
|
||||||
cat(" (all affected packages already have a registry entry)\n")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -160,17 +160,22 @@ unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
# Groups blocked on a dependency (a package-specific fix pinned via `applies_to`
|
# Groups with packages blocked on a dependency (a package-specific fix pinned
|
||||||
# whose dependents merely carry its error): report the dependency + how many
|
# via `applies_to`, or a data-driven cascade where the package fails building a
|
||||||
# dependents wait on it, so fixing it once is recognised as clearing the batch.
|
# dependency): report the dependency + how many dependents wait on it, so fixing
|
||||||
|
# it once is recognised as clearing the batch.
|
||||||
blocked_summary <- function(report, max_pkgs = 15L) {
|
blocked_summary <- function(report, max_pkgs = 15L) {
|
||||||
bl <- Filter(function(g) !is.null(g$blocked_on), report)
|
bl <- Filter(
|
||||||
|
function(g) length(g$blocked_packages %||% character(0L)) > 0L,
|
||||||
|
report
|
||||||
|
)
|
||||||
lapply(bl, function(g) {
|
lapply(bl, function(g) {
|
||||||
|
pkgs <- g$blocked_packages
|
||||||
list(
|
list(
|
||||||
blocked_on = g$blocked_on,
|
blocked_on = g$blocked_on,
|
||||||
n_packages = length(g$packages),
|
n_packages = length(pkgs),
|
||||||
packages = utils::head(g$packages, max_pkgs),
|
packages = utils::head(pkgs, max_pkgs),
|
||||||
packages_truncated = length(g$packages) > max_pkgs
|
packages_truncated = length(pkgs) > max_pkgs
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -150,14 +150,17 @@ for (r in report) {
|
||||||
|
|
||||||
# Groups blocked on a dependency (e.g. RcppParallel dependents) are reported,
|
# Groups blocked on a dependency (e.g. RcppParallel dependents) are reported,
|
||||||
# not proposed: fixing the named dependency clears them all at once.
|
# not proposed: fixing the named dependency clears them all at once.
|
||||||
blocked <- Filter(function(r) !is.null(r$blocked_on), report)
|
blocked <- Filter(
|
||||||
|
function(r) length(r$blocked_packages) > 0L,
|
||||||
|
report
|
||||||
|
)
|
||||||
if (length(blocked) > 0L) {
|
if (length(blocked) > 0L) {
|
||||||
cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n")
|
cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n")
|
||||||
for (r in blocked) {
|
for (r in blocked) {
|
||||||
cat(sprintf(
|
cat(sprintf(
|
||||||
" %s: %d package(s) fail because %s does not build\n",
|
" %s: %d package(s) fail because %s does not build\n",
|
||||||
toString(r$blocked_on),
|
toString(r$blocked_on),
|
||||||
length(r$packages),
|
length(r$blocked_packages),
|
||||||
toString(r$blocked_on)
|
toString(r$blocked_on)
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -105,6 +105,49 @@ test_that("RcppParallel dependents are blocked, not proposed a per-package patch
|
||||||
expect_identical(grp$blocked_on, "RcppParallel")
|
expect_identical(grp$blocked_on, "RcppParallel")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test_that("failing_dependency names the dependency that actually failed", {
|
||||||
|
# A leaf package (BFpack) that fails building its rstan dependency.
|
||||||
|
txt <- paste(
|
||||||
|
"Error in installing dependencies for package BFpack with tag 1.6.1",
|
||||||
|
"Failed to build source package rstan.",
|
||||||
|
"ERROR: compilation failed for package ‘rstan’",
|
||||||
|
sep = "\n"
|
||||||
|
)
|
||||||
|
expect_identical(failing_dependency(txt, "BFpack"), "rstan")
|
||||||
|
# A package failing in its OWN compilation is not a cascade.
|
||||||
|
own <- "ERROR: compilation failed for package ‘BFpack’"
|
||||||
|
expect_true(is.na(failing_dependency(own, "BFpack")))
|
||||||
|
expect_true(is.na(failing_dependency(NA_character_, "x")))
|
||||||
|
expect_true(is.na(failing_dependency("some unrelated error", "x")))
|
||||||
|
})
|
||||||
|
|
||||||
|
test_that("Stan packages blocked on rstan are not proposed a per-package entry", {
|
||||||
|
# BFpack/GMLTM fail building rstan; the tbb error is in rstan's compile.
|
||||||
|
cascade <- paste(
|
||||||
|
"Failed to build source package rstan.",
|
||||||
|
"fatal error: tbb/tbb_stddef.h: No such file or directory",
|
||||||
|
sep = "\n"
|
||||||
|
)
|
||||||
|
failures <- data.frame(
|
||||||
|
name = c("BFpack", "GMLTM", "someOwnPkg"),
|
||||||
|
platform = "ubuntu-2604",
|
||||||
|
arch = "amd64",
|
||||||
|
error_text = c(
|
||||||
|
cascade,
|
||||||
|
cascade,
|
||||||
|
# someOwnPkg fails in its OWN compile on the same header -> fixable.
|
||||||
|
"someOwnPkg.cpp: fatal error: tbb/tbb_stddef.h: No such file or directory"
|
||||||
|
),
|
||||||
|
stringsAsFactors = FALSE
|
||||||
|
)
|
||||||
|
report <- build_triage_report(failures, registered_pkgs = character(0L))
|
||||||
|
grp <- Filter(function(g) g$signature == "tbb-stddef-removed", report)[[1L]]
|
||||||
|
# Only the own-compile package is proposed; the rstan cascades are blocked.
|
||||||
|
expect_identical(names(grp$proposed_entries), "someOwnPkg")
|
||||||
|
expect_setequal(grp$blocked_packages, c("BFpack", "GMLTM"))
|
||||||
|
expect_identical(grp$blocked_on, "rstan")
|
||||||
|
})
|
||||||
|
|
||||||
test_that("RcppParallel itself is still proposed when it is the failing package", {
|
test_that("RcppParallel itself is still proposed when it is the failing package", {
|
||||||
failures <- data.frame(
|
failures <- data.frame(
|
||||||
name = "RcppParallel",
|
name = "RcppParallel",
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue