perf: seed pure-R binaries into a new slot, interleave shards, and stop builds before the timeout
- Add local/seed-pure-r.R, opt-in via SEED_FROM on install-deps: copy pure-R binaries server-side from a slot of the same OS family and arch, after verifying each is a binary, has no compiled code, was built by the primary R minor, and passes bincraft's cross-minor ABI check; write the single_builds and abi_flat_safety rows an upload would. - Interleave build-all shards with shard_slice() instead of contiguous alphabetical blocks, which ranged from 0.1h to 28h per shard. - Share one BUILD_DEADLINE (BUILD_BUDGET_HOURS, default 44h) across every build pass of a build-all-versions step so long runs stop cleanly and still archive and index what they built. - Derive the single_builds platform with codename_platform() instead of a hard-coded list.
This commit is contained in:
parent
63149e246a
commit
cdfdcd035d
2 changed files with 726 additions and 51 deletions
|
|
@ -36,6 +36,13 @@ variables:
|
|||
- 4.5.3
|
||||
- 4.4.3
|
||||
default: 4.5.3
|
||||
SEED_FROM:
|
||||
# Only for bootstrapping a new slot. Copies the pure-R binaries of these
|
||||
# slots (same OS family and arch, same primary R minor; see
|
||||
# local/seed-pure-r.R) before the build list is computed, so build-all only
|
||||
# compiles what actually needs compiling. Space-separated, first wins.
|
||||
description: 'Existing slots to copy pure-R binaries from, e.g. "alpine324 alpine323". Empty: no seeding.'
|
||||
default: ''
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
|
|
@ -88,6 +95,7 @@ steps:
|
|||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite purrr
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||
- if [ -n "${SEED_FROM}" ]; then /opt/R/$R_VERSION/bin/Rscript local/seed-pure-r.R ${SEED_FROM}; fi
|
||||
- /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))"
|
||||
backend_options:
|
||||
docker:
|
||||
|
|
|
|||
|
|
@ -136,11 +136,18 @@ steps:
|
|||
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
CCACHE_DIR: /mnt/cache/ccache
|
||||
NCPUS: 2
|
||||
# Whole hours the build passes below may use, measured from the step's
|
||||
# start. The repo timeout is 50h; the remainder is left for archiving the
|
||||
# missed packages, so a long run ends cleanly instead of being killed.
|
||||
BUILD_BUDGET_HOURS: 44
|
||||
volumes:
|
||||
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- . local/compiler-cache.sh
|
||||
# One deadline shared by every build pass in this step, read by
|
||||
# local/build-all.R and local/build-newer-r.R before each package.
|
||||
- export BUILD_DEADLINE=$(( $(date +%s) + BUILD_BUDGET_HOURS * 3600 ))
|
||||
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
# The primary pass must not rely on build-all-versions-install-deps having
|
||||
# run on *this* agent: depends_on only orders the steps, but the cache
|
||||
|
|
|
|||
Loading…
Reference in a new issue