pat-s
c072b780b2Split every OS/arch row of weekly-rebuild-missing into three shards and move the build loop out of the inline R one-liner into local/rebuild-missing.R. A shard re-derives its outstanding set from the bucket on every start: an object whose ETag equals CRAN's published MD5sum is still a source fallback and needs building. That is bincraft's check_s3_root_package() evaluated in bulk, so a restart resumes rather than replaying thousands of per-package HEADs, and it stays correct when a sibling shard or a process-updates run finishes work in the meantime. No progress file, no volume, no database cursor. Give each shard a 20h wall-clock budget so it exits cleanly instead of having to be killed. A kill matches neither success nor failure, which is how pipeline 10910 skipped its CDN purge and left ~4600 rebuilt binaries behind stale edge copies. Move re-indexing and the purge into weekly-rebuild-reindex.yaml, which depends on the rebuild and runs on success or failure. Three shards writing one slot's PACKAGES concurrently would race: update_PACKAGES lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work.
87 lines
3.1 KiB
R
87 lines
3.1 KiB
R
# Pure helpers for local/rebuild-missing.R, kept separate so local/tests can
|
|
# source them without executing a rebuild.
|
|
|
|
# Interleaved slice of the rebuild list.
|
|
#
|
|
# The list is alphabetical and build cost clusters by name (Rcpp*, Bioc*,
|
|
# rstan*), so contiguous thirds would be badly unbalanced. Interleaving also
|
|
# makes each shard's progress counter representative of the slot as a whole.
|
|
shard_slice <- function(pkgs, split_into, split_index) {
|
|
split_into <- as.integer(split_into)
|
|
split_index <- as.integer(split_index)
|
|
if (is.na(split_into) || is.na(split_index)) {
|
|
stop("shard_slice(): split_into and split_index must be integers")
|
|
}
|
|
if (split_into < 1L || split_index < 1L || split_index > split_into) {
|
|
stop(sprintf(
|
|
"shard_slice(): need 1 <= split_index <= split_into, got %s of %s",
|
|
split_index,
|
|
split_into
|
|
))
|
|
}
|
|
# seq() errors on a descending range, which is what an empty list or a shard
|
|
# index past the end would produce.
|
|
if (length(pkgs) < split_index) {
|
|
return(pkgs[0L])
|
|
}
|
|
pkgs[seq.int(split_index, length(pkgs), by = split_into)]
|
|
}
|
|
|
|
# Packages that still need building, decided from the bucket rather than from
|
|
# remembered progress.
|
|
#
|
|
# This is bincraft's `check_s3_root_package()` evaluated in bulk: an object
|
|
# whose ETag equals CRAN's published MD5sum is byte-identical to CRAN's source,
|
|
# so the build that was supposed to replace it has not happened yet.
|
|
#
|
|
# `etag_by_file` named by `<pkg>_<ver>.tar.gz`, values are unquoted ETags
|
|
# `cran_version` named by package
|
|
# `cran_md5` named by `<pkg>_<ver>`
|
|
#
|
|
# Unknown always means "already a binary", never "rebuild it", so an unreadable
|
|
# CRAN index or a multipart ETag can never mass-schedule work.
|
|
outstanding_packages <- function(pkgs, etag_by_file, cran_version, cran_md5) {
|
|
if (length(pkgs) == 0L) {
|
|
return(pkgs)
|
|
}
|
|
|
|
# An empty table indexes to zero length rather than to NA, which would
|
|
# recycle the whole result away and silently report "nothing to build".
|
|
lookup <- function(table, key) {
|
|
if (length(table) == 0L) {
|
|
return(rep(NA_character_, length(key)))
|
|
}
|
|
unname(as.character(table[key]))
|
|
}
|
|
|
|
version <- lookup(cran_version, pkgs)
|
|
file <- sprintf("%s_%s.tar.gz", pkgs, version)
|
|
etag <- lookup(etag_by_file, file)
|
|
md5 <- lookup(cran_md5, paste(pkgs, version, sep = "_"))
|
|
|
|
# No CRAN version means the package cannot be resolved to a tarball at all;
|
|
# leave it in and let bincraft report why.
|
|
unresolved <- is.na(version)
|
|
# No object at the key: never built, so it is outstanding by definition.
|
|
absent <- !unresolved & is.na(etag)
|
|
# A multipart upload carries a compound ETag rather than an MD5.
|
|
unknown <- !is.na(etag) & grepl("-", etag, fixed = TRUE)
|
|
|
|
is_source <- !unresolved &
|
|
!is.na(etag) &
|
|
!unknown &
|
|
!is.na(md5) &
|
|
etag == md5
|
|
|
|
pkgs[unresolved | absent | is_source]
|
|
}
|
|
|
|
parse_rebuild_args <- function(args) {
|
|
pos <- args[!startsWith(args, "--")]
|
|
budget <- as.numeric(pos[3L])
|
|
list(
|
|
split_into = as.integer(pos[1L]),
|
|
split_index = as.integer(pos[2L]),
|
|
budget_hours = if (is.na(budget)) 20 else budget
|
|
)
|
|
}
|