diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5f659e6..0585cdd 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -43,15 +43,13 @@ jobs:
         r_versions=$(python test/get_r_versions.py ${{ github.event.inputs.r_versions }})
         echo "::set-output name=r_versions::$r_versions"
 
-  test:
+  docker-images:
     needs: setup-matrix
     strategy:
       matrix:
         platform: ${{ fromJson(needs.setup-matrix.outputs.platforms) }}
-    env:
-      R_VERSIONS: ${{ join(fromJson(needs.setup-matrix.outputs.r_versions), ' ') }}
     runs-on: ubuntu-latest
-    name: ${{ matrix.platform }} (R ${{ join(fromJson(needs.setup-matrix.outputs.r_versions), ', ') }})
+    name: Docker image (${{ matrix.platform }})
     steps:
       - uses: actions/checkout@v3
 
@@ -75,23 +73,13 @@ jobs:
       # Use docker buildx instead of docker-compose here because cache exporting
       # does not seem to work as of docker-compose v2.6.0 and buildx v0.8.2, even
       # though it works with buildx individually.
-      - name: Build R
+      - name: Build image
         run: |
-          for version in ${{ env.R_VERSIONS }}; do
-            docker buildx build -t r-builds:${{ matrix.platform }} \
-              --file builder/Dockerfile.${{ matrix.platform }} \
-              --cache-from "type=local,src=/tmp/.buildx-cache" \
-              --cache-to "type=local,dest=/tmp/.buildx-cache-new,mode=max" \
-              --load \
-              builder
-            R_VERSION=$version make build-r-${{ matrix.platform }}
-          done
-
-      - name: Test R
-        run: |
-          for version in ${{ env.R_VERSIONS }}; do
-            R_VERSION=$version make test-r-${{ matrix.platform }}
-          done
+          docker buildx build -t r-builds:${{ matrix.platform }} \
+            --file builder/Dockerfile.${{ matrix.platform }} \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
+            --cache-to "type=local,dest=/tmp/.buildx-cache-new,mode=max" \
+            builder
 
       # Temporary workaround for unbounded GHA cache growth with the local cache mode.
       # https://github.com/docker/build-push-action/issues/252
@@ -100,3 +88,42 @@ jobs:
         run: |
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  test:
+    needs: [setup-matrix, docker-images]
+    strategy:
+      matrix:
+        platform: ${{ fromJson(needs.setup-matrix.outputs.platforms) }}
+        r_version: ${{ fromJson(needs.setup-matrix.outputs.r_versions) }}
+    runs-on: ubuntu-latest
+    name: ${{ matrix.platform }} (R ${{ matrix.r_version }})
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+
+      - name: Restore cached Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ matrix.platform }}-buildx-${{ github.sha }}
+          restore-keys: ${{ matrix.platform }}-buildx-
+
+      - name: Load cached Docker image
+        run: |
+          docker buildx build -t r-builds:${{ matrix.platform }} \
+            --file builder/Dockerfile.${{ matrix.platform }} \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
+            --load \
+            builder
+
+      - name: Build R
+        run: |
+          R_VERSION=${{ matrix.r_version }} make build-r-${{ matrix.platform }}
+
+      - name: Test R
+        run: |
+          R_VERSION=${{ matrix.r_version }} make test-r-${{ matrix.platform }}