perf(ci): route compiles through ccache and use parallel make
Prepend ccache's masquerade dir to PATH in the build pipelines and set the ccache options that make hits possible for R's per-package temp dirs, plus MAKEFLAGS matched to each step's CPU limit.
This commit is contained in:
parent
e7821a75d5
commit
7f6a15abb2
4 changed files with 39 additions and 0 deletions
|
|
@ -140,6 +140,7 @@ steps:
|
||||||
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
||||||
commands:
|
commands:
|
||||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
|
- . local/compiler-cache.sh
|
||||||
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||||
# The primary pass must not rely on build-all-versions-install-deps having
|
# The primary pass must not rely on build-all-versions-install-deps having
|
||||||
# run on *this* agent: depends_on only orders the steps, but the cache
|
# run on *this* agent: depends_on only orders the steps, but the cache
|
||||||
|
|
|
||||||
|
|
@ -179,6 +179,8 @@ steps:
|
||||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||||
R_VERSION: ${R_VERSION}
|
R_VERSION: ${R_VERSION}
|
||||||
CCACHE_DIR: /mnt/cache/ccache
|
CCACHE_DIR: /mnt/cache/ccache
|
||||||
|
# parallel make within one package build, matched to the step's CPU limit
|
||||||
|
MAKE_JOBS: 3
|
||||||
PLATFORM: ${OS}
|
PLATFORM: ${OS}
|
||||||
OS_ID: ${OS_ID}
|
OS_ID: ${OS_ID}
|
||||||
ARCH: ${ARCH}
|
ARCH: ${ARCH}
|
||||||
|
|
@ -186,6 +188,7 @@ steps:
|
||||||
INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
|
INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
|
||||||
commands:
|
commands:
|
||||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
|
- . local/compiler-cache.sh
|
||||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
|
||||||
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||||
|
|
|
||||||
|
|
@ -373,6 +373,8 @@ steps:
|
||||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||||
R_VERSION: ${R_VERSION}
|
R_VERSION: ${R_VERSION}
|
||||||
CCACHE_DIR: /mnt/cache/ccache
|
CCACHE_DIR: /mnt/cache/ccache
|
||||||
|
# parallel make within one package build, matched to the step's CPU limit
|
||||||
|
MAKE_JOBS: 3
|
||||||
PLATFORM: ${OS}
|
PLATFORM: ${OS}
|
||||||
ARCH: ${ARCH}
|
ARCH: ${ARCH}
|
||||||
NCPUS: 2
|
NCPUS: 2
|
||||||
|
|
@ -384,6 +386,7 @@ steps:
|
||||||
REBUILD_BUDGET_HOURS: 20
|
REBUILD_BUDGET_HOURS: 20
|
||||||
commands:
|
commands:
|
||||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
|
- . local/compiler-cache.sh
|
||||||
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||||
|
|
|
||||||
32
local/compiler-cache.sh
Normal file
32
local/compiler-cache.sh
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
# Sourced (`. local/compiler-cache.sh`) by the build pipelines before any
|
||||||
|
# package is compiled. POSIX sh: the Alpine images run steps under busybox.
|
||||||
|
#
|
||||||
|
# Routes gcc/g++ through ccache. The images name their compilers by bare
|
||||||
|
# `gcc`/`g++` in both R's Makeconf and /root/.R/Makevars, so prepending ccache's
|
||||||
|
# masquerade dir catches every compile, CMake-driven vendored builds included.
|
||||||
|
# Wrapping CC in a Makevars instead would need R_MAKEVARS_USER, which replaces
|
||||||
|
# the images' /root/.R/Makevars (CXX_STD, the musl defines) rather than adding
|
||||||
|
# to it. The dir differs per distro: Alpine, RHEL, Debian/Ubuntu.
|
||||||
|
for dir in /usr/lib/ccache/bin /usr/lib64/ccache /usr/lib/ccache; do
|
||||||
|
if [ -x "$dir/gcc" ]; then
|
||||||
|
PATH="$dir:$PATH"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
export PATH
|
||||||
|
|
||||||
|
# R compiles every package in a fresh /tmp/RtmpXXXXXX/<pkg> dir. R's default
|
||||||
|
# flags include -g, which makes ccache hash the working directory, and the line
|
||||||
|
# markers carry the absolute source path, so without these two settings no
|
||||||
|
# compile ever hits the cache across runs.
|
||||||
|
export CCACHE_BASEDIR="${CCACHE_BASEDIR:-/tmp}"
|
||||||
|
export CCACHE_NOHASHDIR=1
|
||||||
|
# The images are rebuilt regularly, which changes the compiler's mtime but not
|
||||||
|
# its content. ccache's default mtime check would discard the cache each time.
|
||||||
|
export CCACHE_COMPILERCHECK=content
|
||||||
|
|
||||||
|
# Parallel make inside a single package build. Match it to the step's CPU limit
|
||||||
|
# via MAKE_JOBS; an explicit MAKEFLAGS wins.
|
||||||
|
export MAKEFLAGS="${MAKEFLAGS:--j${MAKE_JOBS:-2}}"
|
||||||
|
|
||||||
|
echo "compiler-cache: $(command -v gcc), CCACHE_DIR=${CCACHE_DIR:-unset}, MAKEFLAGS=$MAKEFLAGS"
|
||||||
Loading…
Reference in a new issue