feat(build): raise BuildKit cache-mount budget for remote builders #112

Merged
pat-s merged 1 commit from fix/buildkit-cache-mount-budget into main 2026-07-08 08:22:04 +00:00
2 changed files with 45 additions and 2 deletions
Showing only changes of commit 474a7fe62d - Show all commits

feat(build): raise BuildKit cache-mount budget for remote builders

BuildKit's default GC caps the ephemeral cache tier (RUN
--mount=type=cache mounts, local context, git checkouts) at a hardcoded
512 MB.
Across the 7-distro build matrix that fills instantly and forces
re-downloads of system and R packages on every rebuild.

Add docker/buildkitd.toml and pass it via --config when creating the
docker-container builders (artemis/amd64, gaia/arm64):

- cache-mount tier raised 512 MB -> 8 GB, retained 7 days
- total cache bounded at 40 GB with 20 GB min-free (safe on the
  disk-tight Hetzner host, modest on the Mac mini)

Limits are absolute rather than percentages because the two hosts differ
~6x in free space. Document the --config flag in the justfile so a future
recreate does not silently revert to the 512 MB default.
Patrick Schratz 2026-07-08 10:20:15 +02:00
No known key found for this signature in database
GPG key ID: 62050D5BC68AB6DC

41
docker/buildkitd.toml Normal file
View file

@ -0,0 +1,41 @@
# BuildKit GC config for the remote buildx builders (artemis/amd64, gaia/arm64).
#
# Apply when creating the docker-container builders:
# docker buildx create --name artemis --driver docker-container \
# --config docker/buildkitd.toml ssh://<user@host-amd64>
# docker buildx create --name gaia --driver docker-container \
# --config docker/buildkitd.toml ssh://<user@host-arm64>
#
# Why: BuildKit's default GC caps the ephemeral cache tier — RUN
# --mount=type=cache mounts, local build context, git checkouts — at a
# hardcoded 512 MB (shown as "488.3 MiB" in `buildx inspect`). Across our
# 7-distro build matrix that fills instantly and forces re-downloads of
# system + R packages every rebuild. The first rule below raises that tier.
#
# Limits are absolute (not %) on purpose: artemis and gaia have very
# different free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB), so a
# percentage would mean wildly different real budgets. minFreeSpace = 20 GB
# keeps the tight Hetzner host safe while staying modest on the Mac mini.
[worker.oci]
gc = true
# Tier 1 — ephemeral caches (cache mounts, local context, git checkouts).
# Raised from the 512 MB default to 8 GB, retained for 7 days so weekly
# rebuilds reuse downloaded packages instead of re-fetching them.
[[worker.oci.gcpolicy]]
filters = [
"type==source.local",
"type==exec.cachemount",
"type==source.git.checkout",
]
keepDuration = "168h"
maxUsedSpace = "8GB"
# Tier 2 — everything else (image layers, RUN exec results). Bounds the
# whole buildkit cache and always leaves 20 GB free on the host disk.
[[worker.oci.gcpolicy]]
all = true
reservedSpace = "2GB"
maxUsedSpace = "40GB"
minFreeSpace = "20GB"

View file

@ -10,8 +10,10 @@
# - buildx builders named `artemis` (amd64) and `gaia` (arm64), created with the
# docker-container driver (runs BuildKit on the remote host's docker daemon over
# SSH). The default `remote` driver does NOT work with an ssh:// docker host.
# docker buildx create --name artemis --driver docker-container ssh://<user@host-amd64>
# docker buildx create --name gaia --driver docker-container ssh://<user@host-arm64>
# Pass --config docker/buildkitd.toml so BuildKit's GC keeps a usable cache
# (the default caps the cache-mount tier at 512 MB, forcing re-downloads).
# docker buildx create --name artemis --driver docker-container --config docker/buildkitd.toml ssh://<user@host-amd64>
# docker buildx create --name gaia --driver docker-container --config docker/buildkitd.toml ssh://<user@host-arm64>
# - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional)
#
# Overridable (env or `just VAR=… rebuild …`):