pat-s
c072b780b2Split every OS/arch row of weekly-rebuild-missing into three shards and move the build loop out of the inline R one-liner into local/rebuild-missing.R. A shard re-derives its outstanding set from the bucket on every start: an object whose ETag equals CRAN's published MD5sum is still a source fallback and needs building. That is bincraft's check_s3_root_package() evaluated in bulk, so a restart resumes rather than replaying thousands of per-package HEADs, and it stays correct when a sibling shard or a process-updates run finishes work in the meantime. No progress file, no volume, no database cursor. Give each shard a 20h wall-clock budget so it exits cleanly instead of having to be killed. A kill matches neither success nor failure, which is how pipeline 10910 skipped its CDN purge and left ~4600 rebuilt binaries behind stale edge copies. Move re-indexing and the purge into weekly-rebuild-reindex.yaml, which depends on the rebuild and runs on success or failure. Three shards writing one slot's PACKAGES concurrently would race: update_PACKAGES lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work.
94 lines
2.9 KiB
R
94 lines
2.9 KiB
R
source(file.path("..", "rebuild-missing-helpers.R"))
|
|
|
|
test_that("shard_slice partitions the list without gaps or overlap", {
|
|
pkgs <- letters[1:10]
|
|
parts <- lapply(1:3, function(i) shard_slice(pkgs, 3, i))
|
|
|
|
expect_identical(parts[[1]], c("a", "d", "g", "j"))
|
|
expect_identical(parts[[2]], c("b", "e", "h"))
|
|
expect_identical(parts[[3]], c("c", "f", "i"))
|
|
|
|
expect_identical(sort(unlist(parts)), sort(pkgs))
|
|
expect_identical(anyDuplicated(unlist(parts)), 0L)
|
|
})
|
|
|
|
test_that("shard_slice is deterministic and survives short lists", {
|
|
expect_identical(
|
|
shard_slice(letters[1:10], 3, 2),
|
|
shard_slice(letters[1:10], 3, 2)
|
|
)
|
|
expect_identical(shard_slice(character(0), 3, 1), character(0))
|
|
# more shards than packages: the tail shards get nothing rather than erroring
|
|
expect_identical(shard_slice(c("a"), 3, 1), "a")
|
|
expect_identical(shard_slice(c("a"), 3, 2), character(0))
|
|
})
|
|
|
|
test_that("shard_slice rejects an out-of-range index", {
|
|
expect_error(shard_slice(letters, 3, 4), "split_index")
|
|
expect_error(shard_slice(letters, 3, 0), "split_index")
|
|
})
|
|
|
|
test_that("outstanding_packages keeps source fallbacks and drops real binaries", {
|
|
cran_version <- c(httr = "1.4.8", R6 = "2.6.1", curl = "7.1.0")
|
|
cran_md5 <- c(
|
|
httr_1.4.8 = "8756015b94a9cff6f410ca4de8557f12",
|
|
R6_2.6.1 = "f01b1787f12797c29194d63c9afd5d70",
|
|
curl_7.1.0 = "8af2ccbf5d85dc18866f45f1f26f348d"
|
|
)
|
|
etag <- c(
|
|
# byte-identical to CRAN: the build never happened
|
|
"httr_1.4.8.tar.gz" = "8756015b94a9cff6f410ca4de8557f12",
|
|
# a real binary was published
|
|
"R6_2.6.1.tar.gz" = "9d6087ee9adda3f0a3b8067cfc652c05"
|
|
# curl has no object at all
|
|
)
|
|
|
|
out <- outstanding_packages(
|
|
c("httr", "R6", "curl"),
|
|
etag,
|
|
cran_version,
|
|
cran_md5
|
|
)
|
|
expect_identical(out, c("httr", "curl"))
|
|
})
|
|
|
|
test_that("outstanding_packages treats unknowns as already built", {
|
|
cran_version <- c(a = "1.0", b = "1.0")
|
|
cran_md5 <- c(a_1.0 = "aaaa")
|
|
|
|
# a multipart ETag carries no MD5, and `b` is missing from CRAN's index:
|
|
# neither may schedule a rebuild
|
|
etag <- c("a_1.0.tar.gz" = "abc-3", "b_1.0.tar.gz" = "bbbb")
|
|
|
|
expect_identical(
|
|
outstanding_packages(c("a", "b"), etag, cran_version, cran_md5),
|
|
character(0)
|
|
)
|
|
})
|
|
|
|
test_that("outstanding_packages keeps a package CRAN has no version for", {
|
|
out <- outstanding_packages(
|
|
"ghost",
|
|
c(),
|
|
c(other = "1.0"),
|
|
c(other_1.0 = "aaaa")
|
|
)
|
|
expect_identical(out, "ghost")
|
|
})
|
|
|
|
test_that("outstanding_packages handles an empty list", {
|
|
expect_identical(
|
|
outstanding_packages(character(0), c(), c(), c()),
|
|
character(0)
|
|
)
|
|
})
|
|
|
|
test_that("parse_rebuild_args defaults the budget", {
|
|
a <- parse_rebuild_args(c("3", "2"))
|
|
expect_identical(a$split_into, 3L)
|
|
expect_identical(a$split_index, 2L)
|
|
expect_identical(a$budget_hours, 20)
|
|
|
|
b <- parse_rebuild_args(c("3", "2", "1.5"))
|
|
expect_identical(b$budget_hours, 1.5)
|
|
})
|