From 09536cddd63cb7f390927dab873799d3736b24c8 Mon Sep 17 00:00:00 2001
From: pat-s <patrick.schratz@gmail.com>
Date: Wed, 8 Jul 2026 08:22:03 +0000
Subject: [PATCH] feat(build): raise BuildKit cache-mount budget for remote
 builders (#112)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

BuildKit's default GC caps the ephemeral cache tier — `RUN --mount=type=cache` mounts, local build context, git checkouts — at a hardcoded **512 MB** (shown as `488.3 MiB` in `buildx inspect`).
Across our 7-distro build matrix that fills instantly and forces re-downloads of system + R packages on every rebuild.

## Changes

- **`docker/buildkitd.toml`** (new) — GC config passed via `--config` when creating the `docker-container` builders `artemis` (amd64) and `gaia` (arm64):
  - cache-mount tier: **512 MB → 8 GB**, retained 7 days
  - total cache bounded at **40 GB** with **20 GB min-free**
- **`justfile`** — document the `--config docker/buildkitd.toml` flag on the builder-create commands so a recreate does not silently revert to the 512 MB default.

## Notes

- Limits are absolute (not `%`) because the two hosts differ ~6× in free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB). The 20 GB min-free is the safety valve on the disk-tight Hetzner host; the old default wanted 64 GiB free, which does not exist there.
- Both live builders were already recreated with this config and verified running.

Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/112
---
 docker/buildkitd.toml | 41 +++++++++++++++++++++++++++++++++++++++++
 justfile              |  6 ++++--
 2 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100644 docker/buildkitd.toml

diff --git a/docker/buildkitd.toml b/docker/buildkitd.toml
new file mode 100644
index 0000000..ebbeb38
--- /dev/null
+++ b/docker/buildkitd.toml
@@ -0,0 +1,41 @@
+# BuildKit GC config for the remote buildx builders (artemis/amd64, gaia/arm64).
+#
+# Apply when creating the docker-container builders:
+#   docker buildx create --name artemis --driver docker-container \
+#     --config docker/buildkitd.toml ssh://<user@host-amd64>
+#   docker buildx create --name gaia    --driver docker-container \
+#     --config docker/buildkitd.toml ssh://<user@host-arm64>
+#
+# Why: BuildKit's default GC caps the ephemeral cache tier — RUN
+# --mount=type=cache mounts, local build context, git checkouts — at a
+# hardcoded 512 MB (shown as "488.3 MiB" in `buildx inspect`). Across our
+# 7-distro build matrix that fills instantly and forces re-downloads of
+# system + R packages every rebuild. The first rule below raises that tier.
+#
+# Limits are absolute (not %) on purpose: artemis and gaia have very
+# different free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB), so a
+# percentage would mean wildly different real budgets. minFreeSpace = 20 GB
+# keeps the tight Hetzner host safe while staying modest on the Mac mini.
+
+[worker.oci]
+  gc = true
+
+  # Tier 1 — ephemeral caches (cache mounts, local context, git checkouts).
+  # Raised from the 512 MB default to 8 GB, retained for 7 days so weekly
+  # rebuilds reuse downloaded packages instead of re-fetching them.
+  [[worker.oci.gcpolicy]]
+    filters = [
+      "type==source.local",
+      "type==exec.cachemount",
+      "type==source.git.checkout",
+    ]
+    keepDuration = "168h"
+    maxUsedSpace = "8GB"
+
+  # Tier 2 — everything else (image layers, RUN exec results). Bounds the
+  # whole buildkit cache and always leaves 20 GB free on the host disk.
+  [[worker.oci.gcpolicy]]
+    all = true
+    reservedSpace = "2GB"
+    maxUsedSpace = "40GB"
+    minFreeSpace = "20GB"
diff --git a/justfile b/justfile
index 2d9388d..a9a705e 100644
--- a/justfile
+++ b/justfile
@@ -10,8 +10,10 @@
 #   - buildx builders named `artemis` (amd64) and `gaia` (arm64), created with the
 #     docker-container driver (runs BuildKit on the remote host's docker daemon over
 #     SSH). The default `remote` driver does NOT work with an ssh:// docker host.
-#       docker buildx create --name artemis --driver docker-container ssh://<user@host-amd64>
-#       docker buildx create --name gaia    --driver docker-container ssh://<user@host-arm64>
+#     Pass --config docker/buildkitd.toml so BuildKit's GC keeps a usable cache
+#     (the default caps the cache-mount tier at 512 MB, forcing re-downloads).
+#       docker buildx create --name artemis --driver docker-container --config docker/buildkitd.toml ssh://<user@host-amd64>
+#       docker buildx create --name gaia    --driver docker-container --config docker/buildkitd.toml ssh://<user@host-arm64>
 #   - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional)
 #
 # Overridable (env or `just VAR=… rebuild …`):