feat(build): raise BuildKit cache-mount budget for remote builders (#112)
Some checks failed
ci/crow/cron/weekly-audit-missing/3 Pipeline was successful
ci/crow/cron/weekly-audit-missing/4 Pipeline was successful
ci/crow/cron/weekly-audit-missing/5 Pipeline was successful
ci/crow/cron/weekly-audit-missing/6 Pipeline was successful
ci/crow/cron/weekly-audit-missing/7 Pipeline was successful
ci/crow/cron/weekly-audit-missing/8 Pipeline was successful
ci/crow/cron/weekly-audit-missing/9 Pipeline was successful
ci/crow/cron/weekly-audit-missing/10 Pipeline was successful
ci/crow/cron/weekly-audit-missing/11 Pipeline was successful
ci/crow/cron/weekly-audit-missing/12 Pipeline was successful
ci/crow/cron/weekly-audit-missing/13 Pipeline was successful
ci/crow/cron/weekly-audit-missing/14 Pipeline was successful
ci/crow/cron/weekly-audit-missing/15 Pipeline was successful
ci/crow/cron/weekly-audit-missing/16 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/4 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/1 Pipeline was canceled
ci/crow/cron/weekly-rebuild-missing/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/7 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/8 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/2 Pipeline was canceled
ci/crow/cron/weekly-rebuild-missing/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/10 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/11 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/12 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/9 Pipeline was canceled
ci/crow/cron/weekly-rebuild-missing/14 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline failed
ci/crow/cron/process-updates/11 Pipeline was canceled
ci/crow/cron/process-updates/16 Pipeline was canceled
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/cron/process-updates/18 Pipeline was successful
ci/crow/cron/process-updates/17 Pipeline was canceled
ci/crow/cron/process-updates/1 Pipeline was canceled
ci/crow/cron/process-updates/2 Pipeline was canceled
ci/crow/cron/process-updates/8 Pipeline was canceled
ci/crow/manual/build-all-versions-install-deps/2 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was canceled
ci/crow/cron/process-updates/3 Pipeline was canceled
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was canceled
ci/crow/manual/build-all-versions/7 Pipeline was canceled
ci/crow/manual/build-all-versions/6 Pipeline failed
ci/crow/manual/build-all-versions/8 Pipeline was canceled
ci/crow/manual/build-all-versions/5 Pipeline was canceled
ci/crow/cron/process-updates/4 Pipeline was canceled
ci/crow/cron/process-updates/14 Pipeline was canceled
ci/crow/cron/process-updates/13 Pipeline was successful
Some checks failed
ci/crow/cron/weekly-audit-missing/3 Pipeline was successful
ci/crow/cron/weekly-audit-missing/4 Pipeline was successful
ci/crow/cron/weekly-audit-missing/5 Pipeline was successful
ci/crow/cron/weekly-audit-missing/6 Pipeline was successful
ci/crow/cron/weekly-audit-missing/7 Pipeline was successful
ci/crow/cron/weekly-audit-missing/8 Pipeline was successful
ci/crow/cron/weekly-audit-missing/9 Pipeline was successful
ci/crow/cron/weekly-audit-missing/10 Pipeline was successful
ci/crow/cron/weekly-audit-missing/11 Pipeline was successful
ci/crow/cron/weekly-audit-missing/12 Pipeline was successful
ci/crow/cron/weekly-audit-missing/13 Pipeline was successful
ci/crow/cron/weekly-audit-missing/14 Pipeline was successful
ci/crow/cron/weekly-audit-missing/15 Pipeline was successful
ci/crow/cron/weekly-audit-missing/16 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/3 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/4 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/1 Pipeline was canceled
ci/crow/cron/weekly-rebuild-missing/6 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/7 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/8 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/2 Pipeline was canceled
ci/crow/cron/weekly-rebuild-missing/5 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/10 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/11 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/12 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/13 Pipeline was successful
ci/crow/cron/weekly-rebuild-missing/9 Pipeline was canceled
ci/crow/cron/weekly-rebuild-missing/14 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline failed
ci/crow/cron/process-updates/11 Pipeline was canceled
ci/crow/cron/process-updates/16 Pipeline was canceled
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline was successful
ci/crow/cron/process-updates/18 Pipeline was successful
ci/crow/cron/process-updates/17 Pipeline was canceled
ci/crow/cron/process-updates/1 Pipeline was canceled
ci/crow/cron/process-updates/2 Pipeline was canceled
ci/crow/cron/process-updates/8 Pipeline was canceled
ci/crow/manual/build-all-versions-install-deps/2 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was canceled
ci/crow/cron/process-updates/3 Pipeline was canceled
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was canceled
ci/crow/manual/build-all-versions/7 Pipeline was canceled
ci/crow/manual/build-all-versions/6 Pipeline failed
ci/crow/manual/build-all-versions/8 Pipeline was canceled
ci/crow/manual/build-all-versions/5 Pipeline was canceled
ci/crow/cron/process-updates/4 Pipeline was canceled
ci/crow/cron/process-updates/14 Pipeline was canceled
ci/crow/cron/process-updates/13 Pipeline was successful
BuildKit's default GC caps the ephemeral cache tier — `RUN --mount=type=cache` mounts, local build context, git checkouts — at a hardcoded **512 MB** (shown as `488.3 MiB` in `buildx inspect`). Across our 7-distro build matrix that fills instantly and forces re-downloads of system + R packages on every rebuild. ## Changes - **`docker/buildkitd.toml`** (new) — GC config passed via `--config` when creating the `docker-container` builders `artemis` (amd64) and `gaia` (arm64): - cache-mount tier: **512 MB → 8 GB**, retained 7 days - total cache bounded at **40 GB** with **20 GB min-free** - **`justfile`** — document the `--config docker/buildkitd.toml` flag on the builder-create commands so a recreate does not silently revert to the 512 MB default. ## Notes - Limits are absolute (not `%`) because the two hosts differ ~6× in free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB). The 20 GB min-free is the safety valve on the disk-tight Hetzner host; the old default wanted 64 GiB free, which does not exist there. - Both live builders were already recreated with this config and verified running. Reviewed-on: #112
This commit is contained in:
parent
36d3bc8604
commit
09536cddd6
2 changed files with 45 additions and 2 deletions
41
docker/buildkitd.toml
Normal file
41
docker/buildkitd.toml
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
# BuildKit GC config for the remote buildx builders (artemis/amd64, gaia/arm64).
|
||||
#
|
||||
# Apply when creating the docker-container builders:
|
||||
# docker buildx create --name artemis --driver docker-container \
|
||||
# --config docker/buildkitd.toml ssh://<user@host-amd64>
|
||||
# docker buildx create --name gaia --driver docker-container \
|
||||
# --config docker/buildkitd.toml ssh://<user@host-arm64>
|
||||
#
|
||||
# Why: BuildKit's default GC caps the ephemeral cache tier — RUN
|
||||
# --mount=type=cache mounts, local build context, git checkouts — at a
|
||||
# hardcoded 512 MB (shown as "488.3 MiB" in `buildx inspect`). Across our
|
||||
# 7-distro build matrix that fills instantly and forces re-downloads of
|
||||
# system + R packages every rebuild. The first rule below raises that tier.
|
||||
#
|
||||
# Limits are absolute (not %) on purpose: artemis and gaia have very
|
||||
# different free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB), so a
|
||||
# percentage would mean wildly different real budgets. minFreeSpace = 20 GB
|
||||
# keeps the tight Hetzner host safe while staying modest on the Mac mini.
|
||||
|
||||
[worker.oci]
|
||||
gc = true
|
||||
|
||||
# Tier 1 — ephemeral caches (cache mounts, local context, git checkouts).
|
||||
# Raised from the 512 MB default to 8 GB, retained for 7 days so weekly
|
||||
# rebuilds reuse downloaded packages instead of re-fetching them.
|
||||
[[worker.oci.gcpolicy]]
|
||||
filters = [
|
||||
"type==source.local",
|
||||
"type==exec.cachemount",
|
||||
"type==source.git.checkout",
|
||||
]
|
||||
keepDuration = "168h"
|
||||
maxUsedSpace = "8GB"
|
||||
|
||||
# Tier 2 — everything else (image layers, RUN exec results). Bounds the
|
||||
# whole buildkit cache and always leaves 20 GB free on the host disk.
|
||||
[[worker.oci.gcpolicy]]
|
||||
all = true
|
||||
reservedSpace = "2GB"
|
||||
maxUsedSpace = "40GB"
|
||||
minFreeSpace = "20GB"
|
||||
6
justfile
6
justfile
|
|
@ -10,8 +10,10 @@
|
|||
# - buildx builders named `artemis` (amd64) and `gaia` (arm64), created with the
|
||||
# docker-container driver (runs BuildKit on the remote host's docker daemon over
|
||||
# SSH). The default `remote` driver does NOT work with an ssh:// docker host.
|
||||
# docker buildx create --name artemis --driver docker-container ssh://<user@host-amd64>
|
||||
# docker buildx create --name gaia --driver docker-container ssh://<user@host-arm64>
|
||||
# Pass --config docker/buildkitd.toml so BuildKit's GC keeps a usable cache
|
||||
# (the default caps the cache-mount tier at 512 MB, forcing re-downloads).
|
||||
# docker buildx create --name artemis --driver docker-container --config docker/buildkitd.toml ssh://<user@host-amd64>
|
||||
# docker buildx create --name gaia --driver docker-container --config docker/buildkitd.toml ssh://<user@host-arm64>
|
||||
# - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional)
|
||||
#
|
||||
# Overridable (env or `just VAR=… rebuild …`):
|
||||
|
|
|
|||
Loading…
Reference in a new issue