From 474a7fe62d21147106f3dacb0bcb62a4f1dc0ce7 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 8 Jul 2026 10:20:15 +0200 Subject: [PATCH] feat(build): raise BuildKit cache-mount budget for remote builders BuildKit's default GC caps the ephemeral cache tier (RUN --mount=type=cache mounts, local context, git checkouts) at a hardcoded 512 MB. Across the 7-distro build matrix that fills instantly and forces re-downloads of system and R packages on every rebuild. Add docker/buildkitd.toml and pass it via --config when creating the docker-container builders (artemis/amd64, gaia/arm64): - cache-mount tier raised 512 MB -> 8 GB, retained 7 days - total cache bounded at 40 GB with 20 GB min-free (safe on the disk-tight Hetzner host, modest on the Mac mini) Limits are absolute rather than percentages because the two hosts differ ~6x in free space. Document the --config flag in the justfile so a future recreate does not silently revert to the 512 MB default. --- docker/buildkitd.toml | 41 +++++++++++++++++++++++++++++++++++++++++ justfile | 6 ++++-- 2 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 docker/buildkitd.toml diff --git a/docker/buildkitd.toml b/docker/buildkitd.toml new file mode 100644 index 0000000..ebbeb38 --- /dev/null +++ b/docker/buildkitd.toml @@ -0,0 +1,41 @@ +# BuildKit GC config for the remote buildx builders (artemis/amd64, gaia/arm64). +# +# Apply when creating the docker-container builders: +# docker buildx create --name artemis --driver docker-container \ +# --config docker/buildkitd.toml ssh:// +# docker buildx create --name gaia --driver docker-container \ +# --config docker/buildkitd.toml ssh:// +# +# Why: BuildKit's default GC caps the ephemeral cache tier — RUN +# --mount=type=cache mounts, local build context, git checkouts — at a +# hardcoded 512 MB (shown as "488.3 MiB" in `buildx inspect`). Across our +# 7-distro build matrix that fills instantly and forces re-downloads of +# system + R packages every rebuild. The first rule below raises that tier. +# +# Limits are absolute (not %) on purpose: artemis and gaia have very +# different free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB), so a +# percentage would mean wildly different real budgets. minFreeSpace = 20 GB +# keeps the tight Hetzner host safe while staying modest on the Mac mini. + +[worker.oci] + gc = true + + # Tier 1 — ephemeral caches (cache mounts, local context, git checkouts). + # Raised from the 512 MB default to 8 GB, retained for 7 days so weekly + # rebuilds reuse downloaded packages instead of re-fetching them. + [[worker.oci.gcpolicy]] + filters = [ + "type==source.local", + "type==exec.cachemount", + "type==source.git.checkout", + ] + keepDuration = "168h" + maxUsedSpace = "8GB" + + # Tier 2 — everything else (image layers, RUN exec results). Bounds the + # whole buildkit cache and always leaves 20 GB free on the host disk. + [[worker.oci.gcpolicy]] + all = true + reservedSpace = "2GB" + maxUsedSpace = "40GB" + minFreeSpace = "20GB" diff --git a/justfile b/justfile index 2d9388d..a9a705e 100644 --- a/justfile +++ b/justfile @@ -10,8 +10,10 @@ # - buildx builders named `artemis` (amd64) and `gaia` (arm64), created with the # docker-container driver (runs BuildKit on the remote host's docker daemon over # SSH). The default `remote` driver does NOT work with an ssh:// docker host. -# docker buildx create --name artemis --driver docker-container ssh:// -# docker buildx create --name gaia --driver docker-container ssh:// +# Pass --config docker/buildkitd.toml so BuildKit's GC keeps a usable cache +# (the default caps the cache-mount tier at 512 MB, forcing re-downloads). +# docker buildx create --name artemis --driver docker-container --config docker/buildkitd.toml ssh:// +# docker buildx create --name gaia --driver docker-container --config docker/buildkitd.toml ssh:// # - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional) # # Overridable (env or `just VAR=… rebuild …`): -- 2.54.0