perf: use ccache and parallel make, and skip already-built newer-R candidates in bulk #240

Merged
pat-s merged 2 commits from perf/ccache-and-newer-r-prefilter into main 2026-10-23 13:18:00 +00:00
4 changed files with 39 additions and 0 deletions
Showing only changes of commit 7f6a15abb2 - Show all commits

perf(ci): route compiles through ccache and use parallel make

Prepend ccache's masquerade dir to PATH in the build pipelines and set the ccache options that make hits possible for R's per-package temp dirs, plus MAKEFLAGS matched to each step's CPU limit.
Patrick Schratz 2026-10-23 13:16:33 +00:00
No known key found for this signature in database
GPG key ID: 62050D5BC68AB6DC

View file

@ -140,6 +140,7 @@ steps:
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- . local/compiler-cache.sh
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
# The primary pass must not rely on build-all-versions-install-deps having
# run on *this* agent: depends_on only orders the steps, but the cache

View file

@ -179,6 +179,8 @@ steps:
R_LIBS_USER: /mnt/cache/R-pkgs
R_VERSION: ${R_VERSION}
CCACHE_DIR: /mnt/cache/ccache
# parallel make within one package build, matched to the step's CPU limit
MAKE_JOBS: 3
PLATFORM: ${OS}
OS_ID: ${OS_ID}
ARCH: ${ARCH}
@ -186,6 +188,7 @@ steps:
INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- . local/compiler-cache.sh
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R

View file

@ -373,6 +373,8 @@ steps:
R_LIBS_USER: /mnt/cache/R-pkgs
R_VERSION: ${R_VERSION}
CCACHE_DIR: /mnt/cache/ccache
# parallel make within one package build, matched to the step's CPU limit
MAKE_JOBS: 3
PLATFORM: ${OS}
ARCH: ${ARCH}
NCPUS: 2
@ -384,6 +386,7 @@ steps:
REBUILD_BUDGET_HOURS: 20
commands:
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
- . local/compiler-cache.sh
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R

32
local/compiler-cache.sh Normal file
View file

@ -0,0 +1,32 @@
# Sourced (`. local/compiler-cache.sh`) by the build pipelines before any
# package is compiled. POSIX sh: the Alpine images run steps under busybox.
#
# Routes gcc/g++ through ccache. The images name their compilers by bare
# `gcc`/`g++` in both R's Makeconf and /root/.R/Makevars, so prepending ccache's
# masquerade dir catches every compile, CMake-driven vendored builds included.
# Wrapping CC in a Makevars instead would need R_MAKEVARS_USER, which replaces
# the images' /root/.R/Makevars (CXX_STD, the musl defines) rather than adding
# to it. The dir differs per distro: Alpine, RHEL, Debian/Ubuntu.
for dir in /usr/lib/ccache/bin /usr/lib64/ccache /usr/lib/ccache; do
if [ -x "$dir/gcc" ]; then
PATH="$dir:$PATH"
break
fi
done
export PATH
# R compiles every package in a fresh /tmp/RtmpXXXXXX/<pkg> dir. R's default
# flags include -g, which makes ccache hash the working directory, and the line
# markers carry the absolute source path, so without these two settings no
# compile ever hits the cache across runs.
export CCACHE_BASEDIR="${CCACHE_BASEDIR:-/tmp}"
export CCACHE_NOHASHDIR=1
# The images are rebuilt regularly, which changes the compiler's mtime but not
# its content. ccache's default mtime check would discard the cache each time.
export CCACHE_COMPILERCHECK=content
# Parallel make inside a single package build. Match it to the step's CPU limit
# via MAKE_JOBS; an explicit MAKEFLAGS wins.
export MAKEFLAGS="${MAKEFLAGS:--j${MAKE_JOBS:-2}}"
echo "compiler-cache: $(command -v gcc), CCACHE_DIR=${CCACHE_DIR:-unset}, MAKEFLAGS=$MAKEFLAGS"