From 4597879d10475327f009eec4f751184193c1bdd3 Mon Sep 17 00:00:00 2001
From: Greg Lin <glin@rstudio.com>
Date: Wed, 10 Aug 2022 18:54:28 -0500
Subject: [PATCH 1/6] Add GitHub Actions workflow to test R builds

---
 .github/workflows/test.yml | 74 ++++++++++++++++++++++++++++++++++++++
 Makefile                   |  5 ++-
 README.md                  | 25 +++++++++++++
 test/get_platforms.py      | 31 ++++++++++++++++
 test/get_r_versions.py     | 67 ++++++++++++++++++++++++++++++++++
 5 files changed, 201 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/test.yml
 create mode 100644 test/get_platforms.py
 create mode 100644 test/get_r_versions.py

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 0000000..565c63f
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,74 @@
+name: R builds
+
+on:
+  push:
+    paths:
+      - 'builder/**'
+      - 'test/**'
+      - 'Makefile'
+  workflow_dispatch:
+    inputs:
+      platforms:
+        description: |
+          Comma-separated list of platforms. Specify "all" to use all platforms (the default).
+        required: false
+        default: 'all'
+        type: string
+      r_versions:
+        description: |
+          Comma-separated list of R versions. Specify "last-N" to use the
+          last N minor R versions, or "all" to use all minor R versions since R 3.1.
+          Defaults to "last-5".
+        required: false
+        default: 'last-5'
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  setup-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      platforms: ${{ steps.setup-matrix.outputs.platforms }}
+      r_versions: ${{ steps.setup-matrix.outputs.r_versions }}
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up matrix of platforms and R versions
+      id: setup-matrix
+      run: |
+        platforms=$(python test/get_platforms.py ${{ github.event.inputs.platforms }})
+        echo "::set-output name=platforms::$platforms"
+        r_versions=$(python test/get_r_versions.py ${{ github.event.inputs.r_versions }})
+        echo "::set-output name=r_versions::$r_versions"
+
+  test:
+    needs: setup-matrix
+    strategy:
+      matrix:
+        platform: ${{ fromJson(needs.setup-matrix.outputs.platforms) }}
+    env:
+      R_VERSIONS: ${{ join(fromJson(needs.setup-matrix.outputs.r_versions), ' ') }}
+    runs-on: ubuntu-latest
+    name: ${{ matrix.platform }} (R ${{ join(fromJson(needs.setup-matrix.outputs.r_versions), ', ') }})
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: docker
+          install: true
+
+      - name: Build R
+        run: |
+          for version in ${{ env.R_VERSIONS }}; do
+            R_VERSION=$version make build-r-${{ matrix.platform }}
+          done
+
+      - name: Test R
+        run: |
+          for version in ${{ env.R_VERSIONS }}; do
+            R_VERSION=$version make test-r-${{ matrix.platform }}
+          done
diff --git a/Makefile b/Makefile
index f0f7492..186b40c 100644
--- a/Makefile
+++ b/Makefile
@@ -59,6 +59,9 @@ $(foreach platform,$(PLATFORMS), \
     $(eval $(GEN_TARGETS)) \
 )
 
+print-platforms:
+	@echo $(PLATFORMS)
+
 # Helper for launching a bash session on a docker image of your choice. Defaults
 # to "ubuntu:xenial".
 TARGET_IMAGE?=ubuntu:xenial
@@ -68,4 +71,4 @@ bash:
 		-w /r-builds \
 		${TARGET_IMAGE} /bin/bash
 
-.PHONY: deps docker-build docker-push docker-down docker-build-package docker-shell-package-env ecr-login fetch-serverless-custom-file serverless-deploy
+.PHONY: deps docker-build docker-push docker-down docker-build-package docker-shell-package-env ecr-login fetch-serverless-custom-file print-platforms serverless-deploy
diff --git a/README.md b/README.md
index 2ee37c2..ba42a2f 100644
--- a/README.md
+++ b/README.md
@@ -226,6 +226,21 @@ environment:
 
 In order for the makefile to push these new platforms to ECR, add them to the PLATFORMS variable near the top of the Makefile
 
+### test/docker-compose.yml
+
+A new service in the `test/docker-compose.yml` file named according to the `platform-version` and containing the proper entries:
+
+```yaml
+  ubuntu-2204:
+    image: ubuntu:jammy
+    command: /r-builds/test/test-apt.sh
+    environment:
+      - OS_IDENTIFIER=ubuntu-2204
+      - R_VERSION=${R_VERSION}
+    volumes:
+      - ../:/r-builds
+```
+
 ### Submit a Pull Request
 
 Once you've followed the steps above, submit a pull request. On successful merge, builds for this platform will begin to be available from the CDN.
@@ -244,6 +259,16 @@ serverless invoke stepf -n rBuilds -d '{"force": true, "versions": ["3.6.3", "4.
 
 ## Testing
 
+Tests are automatically run on each push that changes a file in `builder/`, `test/`, or the `Makefile`.
+These tests validate that R was correctly configured, built, and packaged. By default, the tests run
+for the last 5 minor R versions on each platform.
+
+To run the tests manually, you can navigate to the [GitHub Actions workflow page](https://github.com/rstudio/r-builds/actions/workflows/test.yml)
+and use "Run workflow" to run the tests from a custom branch, list of platforms, and list of R versions.
+
+To skip the tests, add `[skip ci]` to your commit message. See [Skipping workflow runs](https://docs.github.com/en/actions/managing-workflow-runs/skipping-workflow-runs)
+for more information.
+
 To test the R builds locally, you can use the `build-r-$PLATFORM` and `test-r-$PLATFORM`
 targets to build R and run the tests. The tests use the quick install script to install R,
 using a locally built R if present, or otherwise a build from the CDN.
diff --git a/test/get_platforms.py b/test/get_platforms.py
new file mode 100644
index 0000000..20d7e6d
--- /dev/null
+++ b/test/get_platforms.py
@@ -0,0 +1,31 @@
+import argparse
+import json
+import subprocess
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Print R-builds platforms as JSON.")
+    parser.add_argument(
+        'platforms',
+        type=str,
+        nargs='?',
+        default='all',
+        help='Comma-separated list of platforms. Specify "all" to use all platforms (the default).'
+    )
+    args = parser.parse_args()
+    platforms = _get_platforms(which=args.platforms)
+    print(json.dumps(platforms))
+
+
+def _get_platforms(which='all'):
+    supported_platforms = subprocess.check_output(['make', 'print-platforms'], text=True)
+    supported_platforms = supported_platforms.split()
+    if which == 'all':
+        return supported_platforms
+    platforms = which.split(',')
+    platforms = [p for p in platforms if p in supported_platforms]
+    return platforms
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/get_r_versions.py b/test/get_r_versions.py
new file mode 100644
index 0000000..0a2ebd9
--- /dev/null
+++ b/test/get_r_versions.py
@@ -0,0 +1,67 @@
+import argparse
+import json
+import re
+import urllib.request
+
+VERSIONS_URL = 'https://cdn.rstudio.com/r/versions.json'
+
+# Minimum R version for "all"
+MIN_ALL_VERSION = '3.1.0'
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Print R-builds R versions as JSON.")
+    parser.add_argument(
+        'versions',
+        type=str,
+        nargs='?',
+        default='last-5',
+        help="""Comma-separated list of R versions. Specify "last-N" to use the
+            last N minor R versions, or "all" to use all minor R versions since R 3.1.
+            Defaults to "last-5".
+            """
+    )
+    args = parser.parse_args()
+    versions = _get_versions(which=args.versions)
+    print(json.dumps(versions))
+
+
+def _get_versions(which='all'):
+    supported_versions = sorted(_get_supported_versions(), reverse=True)
+    
+    last_n_versions = None
+    if which.startswith('last-'):
+        last_n_versions = int(which.replace('last-', ''))
+    elif which != 'all':
+        versions = which.split(',')
+        versions = [v for v in versions if v in supported_versions]
+        return versions
+
+    versions = {}
+    for ver in supported_versions:
+        # Skip unreleased versions (e.g., devel, next)
+        if not re.match(r'[\d.]', ver):
+            continue
+        if ver < MIN_ALL_VERSION:
+            continue
+        minor_ver = tuple(ver.split('.')[0:2])
+        if minor_ver not in versions:
+            versions[minor_ver] = ver
+    versions = sorted(list(versions.values()), reverse=True)
+
+    if last_n_versions:
+        return versions[0:last_n_versions]
+
+    return versions
+
+
+def _get_supported_versions():
+    request = urllib.request.Request(VERSIONS_URL)
+    response = urllib.request.urlopen(request)
+    data = response.read()
+    result = json.loads(data)
+    return result['r_versions']
+
+
+if __name__ == '__main__':
+    main()

From 5576778f096bf48b5d81b1d6628f5dab1d7948bb Mon Sep 17 00:00:00 2001
From: Greg Lin <glin@rstudio.com>
Date: Mon, 15 Aug 2022 14:49:43 -0500
Subject: [PATCH 2/6] Remove deprecated version field from docker-compose.yml

---
 builder/docker-compose.yml | 2 --
 test/docker-compose.yml    | 2 --
 2 files changed, 4 deletions(-)

diff --git a/builder/docker-compose.yml b/builder/docker-compose.yml
index 9eed916..0ab026c 100644
--- a/builder/docker-compose.yml
+++ b/builder/docker-compose.yml
@@ -1,5 +1,3 @@
-version: '2.0'
-
 services:
   ubuntu-1804:
     command: ./build.sh
diff --git a/test/docker-compose.yml b/test/docker-compose.yml
index 57215bd..2bb5760 100644
--- a/test/docker-compose.yml
+++ b/test/docker-compose.yml
@@ -1,5 +1,3 @@
-version: '2.0'
-
 services:
   ubuntu-1804:
     image: ubuntu:bionic

From a1f46060fa2be4787e7532a8284cd2079876f6f4 Mon Sep 17 00:00:00 2001
From: Greg Lin <glin@rstudio.com>
Date: Mon, 15 Aug 2022 18:14:43 -0500
Subject: [PATCH 3/6] Try local Docker image caching in GHA

---
 .github/workflows/test.yml | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 565c63f..5f659e6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -58,12 +58,32 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:
-          driver: docker
           install: true
 
+      # Enable Docker layer caching without having to push to a registry.
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#local-cache
+      # This may eventually be migrated to the GitHub Actions cache backend,
+      # which is still considered experimental.
+      # https://github.com/moby/buildkit#github-actions-cache-experimental
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ matrix.platform }}-buildx-${{ github.sha }}
+          restore-keys: ${{ matrix.platform }}-buildx-
+
+      # Use docker buildx instead of docker-compose here because cache exporting
+      # does not seem to work as of docker-compose v2.6.0 and buildx v0.8.2, even
+      # though it works with buildx individually.
       - name: Build R
         run: |
           for version in ${{ env.R_VERSIONS }}; do
+            docker buildx build -t r-builds:${{ matrix.platform }} \
+              --file builder/Dockerfile.${{ matrix.platform }} \
+              --cache-from "type=local,src=/tmp/.buildx-cache" \
+              --cache-to "type=local,dest=/tmp/.buildx-cache-new,mode=max" \
+              --load \
+              builder
             R_VERSION=$version make build-r-${{ matrix.platform }}
           done
 
@@ -72,3 +92,11 @@ jobs:
           for version in ${{ env.R_VERSIONS }}; do
             R_VERSION=$version make test-r-${{ matrix.platform }}
           done
+
+      # Temporary workaround for unbounded GHA cache growth with the local cache mode.
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

From 225b7da1f79b7cb1757ab13ff98f64398dcf243c Mon Sep 17 00:00:00 2001
From: Greg Lin <glin@rstudio.com>
Date: Tue, 16 Aug 2022 17:09:05 -0500
Subject: [PATCH 4/6] Try parallelizing R version builds

---
 .github/workflows/test.yml | 67 ++++++++++++++++++++++++++------------
 1 file changed, 47 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5f659e6..0585cdd 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -43,15 +43,13 @@ jobs:
         r_versions=$(python test/get_r_versions.py ${{ github.event.inputs.r_versions }})
         echo "::set-output name=r_versions::$r_versions"
 
-  test:
+  docker-images:
     needs: setup-matrix
     strategy:
       matrix:
         platform: ${{ fromJson(needs.setup-matrix.outputs.platforms) }}
-    env:
-      R_VERSIONS: ${{ join(fromJson(needs.setup-matrix.outputs.r_versions), ' ') }}
     runs-on: ubuntu-latest
-    name: ${{ matrix.platform }} (R ${{ join(fromJson(needs.setup-matrix.outputs.r_versions), ', ') }})
+    name: Docker image (${{ matrix.platform }})
     steps:
       - uses: actions/checkout@v3
 
@@ -75,23 +73,13 @@ jobs:
       # Use docker buildx instead of docker-compose here because cache exporting
       # does not seem to work as of docker-compose v2.6.0 and buildx v0.8.2, even
       # though it works with buildx individually.
-      - name: Build R
+      - name: Build image
         run: |
-          for version in ${{ env.R_VERSIONS }}; do
-            docker buildx build -t r-builds:${{ matrix.platform }} \
-              --file builder/Dockerfile.${{ matrix.platform }} \
-              --cache-from "type=local,src=/tmp/.buildx-cache" \
-              --cache-to "type=local,dest=/tmp/.buildx-cache-new,mode=max" \
-              --load \
-              builder
-            R_VERSION=$version make build-r-${{ matrix.platform }}
-          done
-
-      - name: Test R
-        run: |
-          for version in ${{ env.R_VERSIONS }}; do
-            R_VERSION=$version make test-r-${{ matrix.platform }}
-          done
+          docker buildx build -t r-builds:${{ matrix.platform }} \
+            --file builder/Dockerfile.${{ matrix.platform }} \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
+            --cache-to "type=local,dest=/tmp/.buildx-cache-new,mode=max" \
+            builder
 
       # Temporary workaround for unbounded GHA cache growth with the local cache mode.
       # https://github.com/docker/build-push-action/issues/252
@@ -100,3 +88,42 @@ jobs:
         run: |
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  test:
+    needs: [setup-matrix, docker-images]
+    strategy:
+      matrix:
+        platform: ${{ fromJson(needs.setup-matrix.outputs.platforms) }}
+        r_version: ${{ fromJson(needs.setup-matrix.outputs.r_versions) }}
+    runs-on: ubuntu-latest
+    name: ${{ matrix.platform }} (R ${{ matrix.r_version }})
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+
+      - name: Restore cached Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ matrix.platform }}-buildx-${{ github.sha }}
+          restore-keys: ${{ matrix.platform }}-buildx-
+
+      - name: Load cached Docker image
+        run: |
+          docker buildx build -t r-builds:${{ matrix.platform }} \
+            --file builder/Dockerfile.${{ matrix.platform }} \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
+            --load \
+            builder
+
+      - name: Build R
+        run: |
+          R_VERSION=${{ matrix.r_version }} make build-r-${{ matrix.platform }}
+
+      - name: Test R
+        run: |
+          R_VERSION=${{ matrix.r_version }} make test-r-${{ matrix.platform }}

From 534f225b0cdaed8f067758e545a43adce2391298 Mon Sep 17 00:00:00 2001
From: Greg Lin <glin@rstudio.com>
Date: Tue, 16 Aug 2022 18:35:54 -0500
Subject: [PATCH 5/6] Trigger workflow on workflow changes

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0585cdd..8042705 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -6,6 +6,7 @@ on:
       - 'builder/**'
       - 'test/**'
       - 'Makefile'
+      - '.github/workflows/test.yml'
   workflow_dispatch:
     inputs:
       platforms:

From b863122cd4f0f23db471f7f480a6220dd008d58d Mon Sep 17 00:00:00 2001
From: Greg Lin <glin@rstudio.com>
Date: Wed, 17 Aug 2022 12:10:19 -0500
Subject: [PATCH 6/6] Disable fail-fast on matrix jobs since there seem to be a
 lot of transient errors

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8042705..4eb5c69 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -93,6 +93,7 @@ jobs:
   test:
     needs: [setup-matrix, docker-images]
     strategy:
+      fail-fast: false
       matrix:
         platform: ${{ fromJson(needs.setup-matrix.outputs.platforms) }}
         r_version: ${{ fromJson(needs.setup-matrix.outputs.r_versions) }}