feat(rebuild): shard the weekly rebuild and make each shard resumable
Split every OS/arch row of weekly-rebuild-missing into three shards and move the build loop out of the inline R one-liner into local/rebuild-missing.R. A shard re-derives its outstanding set from the bucket on every start: an object whose ETag equals CRAN's published MD5sum is still a source fallback and needs building. That is bincraft's check_s3_root_package() evaluated in bulk, so a restart resumes rather than replaying thousands of per-package HEADs, and it stays correct when a sibling shard or a process-updates run finishes work in the meantime. No progress file, no volume, no database cursor. Give each shard a 20h wall-clock budget so it exits cleanly instead of having to be killed. A kill matches neither success nor failure, which is how pipeline 10910 skipped its CDN purge and left ~4600 rebuilt binaries behind stale edge copies. Move re-indexing and the purge into weekly-rebuild-reindex.yaml, which depends on the rebuild and runs on success or failure. Three shards writing one slot's PACKAGES concurrently would race: update_PACKAGES lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work.
This commit is contained in:
parent
02e3966523
commit
c072b780b2
1 changed files with 838 additions and 36 deletions
87
local/rebuild-missing-helpers.R
Normal file
87
local/rebuild-missing-helpers.R
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
# Pure helpers for local/rebuild-missing.R, kept separate so local/tests can
|
||||
# source them without executing a rebuild.
|
||||
|
||||
# Interleaved slice of the rebuild list.
|
||||
#
|
||||
# The list is alphabetical and build cost clusters by name (Rcpp*, Bioc*,
|
||||
# rstan*), so contiguous thirds would be badly unbalanced. Interleaving also
|
||||
# makes each shard's progress counter representative of the slot as a whole.
|
||||
shard_slice <- function(pkgs, split_into, split_index) {
|
||||
split_into <- as.integer(split_into)
|
||||
split_index <- as.integer(split_index)
|
||||
if (is.na(split_into) || is.na(split_index)) {
|
||||
stop("shard_slice(): split_into and split_index must be integers")
|
||||
}
|
||||
if (split_into < 1L || split_index < 1L || split_index > split_into) {
|
||||
stop(sprintf(
|
||||
"shard_slice(): need 1 <= split_index <= split_into, got %s of %s",
|
||||
split_index,
|
||||
split_into
|
||||
))
|
||||
}
|
||||
# seq() errors on a descending range, which is what an empty list or a shard
|
||||
# index past the end would produce.
|
||||
if (length(pkgs) < split_index) {
|
||||
return(pkgs[0L])
|
||||
}
|
||||
pkgs[seq.int(split_index, length(pkgs), by = split_into)]
|
||||
}
|
||||
|
||||
# Packages that still need building, decided from the bucket rather than from
|
||||
# remembered progress.
|
||||
#
|
||||
# This is bincraft's `check_s3_root_package()` evaluated in bulk: an object
|
||||
# whose ETag equals CRAN's published MD5sum is byte-identical to CRAN's source,
|
||||
# so the build that was supposed to replace it has not happened yet.
|
||||
#
|
||||
# `etag_by_file` named by `<pkg>_<ver>.tar.gz`, values are unquoted ETags
|
||||
# `cran_version` named by package
|
||||
# `cran_md5` named by `<pkg>_<ver>`
|
||||
#
|
||||
# Unknown always means "already a binary", never "rebuild it", so an unreadable
|
||||
# CRAN index or a multipart ETag can never mass-schedule work.
|
||||
outstanding_packages <- function(pkgs, etag_by_file, cran_version, cran_md5) {
|
||||
if (length(pkgs) == 0L) {
|
||||
return(pkgs)
|
||||
}
|
||||
|
||||
# An empty table indexes to zero length rather than to NA, which would
|
||||
# recycle the whole result away and silently report "nothing to build".
|
||||
lookup <- function(table, key) {
|
||||
if (length(table) == 0L) {
|
||||
return(rep(NA_character_, length(key)))
|
||||
}
|
||||
unname(as.character(table[key]))
|
||||
}
|
||||
|
||||
version <- lookup(cran_version, pkgs)
|
||||
file <- sprintf("%s_%s.tar.gz", pkgs, version)
|
||||
etag <- lookup(etag_by_file, file)
|
||||
md5 <- lookup(cran_md5, paste(pkgs, version, sep = "_"))
|
||||
|
||||
# No CRAN version means the package cannot be resolved to a tarball at all;
|
||||
# leave it in and let bincraft report why.
|
||||
unresolved <- is.na(version)
|
||||
# No object at the key: never built, so it is outstanding by definition.
|
||||
absent <- !unresolved & is.na(etag)
|
||||
# A multipart upload carries a compound ETag rather than an MD5.
|
||||
unknown <- !is.na(etag) & grepl("-", etag, fixed = TRUE)
|
||||
|
||||
is_source <- !unresolved &
|
||||
!is.na(etag) &
|
||||
!unknown &
|
||||
!is.na(md5) &
|
||||
etag == md5
|
||||
|
||||
pkgs[unresolved | absent | is_source]
|
||||
}
|
||||
|
||||
parse_rebuild_args <- function(args) {
|
||||
pos <- args[!startsWith(args, "--")]
|
||||
budget <- as.numeric(pos[3L])
|
||||
list(
|
||||
split_into = as.integer(pos[1L]),
|
||||
split_index = as.integer(pos[2L]),
|
||||
budget_hours = if (is.na(budget)) 20 else budget
|
||||
)
|
||||
}
|
||||
Loading…
Reference in a new issue