Merge branch 'master' into jc-ubuntu-20

README.md

@@ -174,7 +174,7 @@ Create a `builder/Dockerfile.platform-version` (where `platform-version` is `ubu
 
 1. an `OS_IDENTIFIER` env with the `platform-version`.
 2. a step which ensures the R source build dependencies are installed
-3. The `awscli`, most likely installed via `pip` for uploading tarballs to S3
+3. The `awscli`, 1.17.10+ if installed via `pip`, for uploading tarballs to S3
 4. `COPY` and `ENTRYPOINT` for the `build.sh` file in `builder/`.
 
 ### docker-compose.yml

builder/Dockerfile.ubuntu-1604

@@ -6,7 +6,7 @@ RUN set -x \
   && sed -i "s|# deb-src|deb-src|g" /etc/apt/sources.list \
   && export DEBIAN_FRONTEND=noninteractive \
   && apt-get update \
-  && apt-get install -y libcurl4-openssl-dev libicu-dev libopenblas-base wget python-pip ruby ruby-dev \
+  && apt-get install -y libcurl4-openssl-dev libicu-dev libopenblas-base libpcre2-dev wget python-pip ruby ruby-dev \
   && apt-get build-dep -y r-base
 
 RUN pip install awscli

builder/Dockerfile.ubuntu-1804

@@ -6,7 +6,7 @@ RUN set -x \
   && sed -i "s|# deb-src|deb-src|g" /etc/apt/sources.list \
   && export DEBIAN_FRONTEND=noninteractive \
   && apt-get update \
-  && apt-get install -y libcurl4-openssl-dev libicu-dev libopenblas-base wget python-pip ruby ruby-dev \
+  && apt-get install -y libcurl4-openssl-dev libicu-dev libopenblas-base libpcre2-dev wget python-pip ruby ruby-dev \
   && apt-get build-dep -y r-base
 
 RUN pip install awscli

builder/build.sh

@@ -40,7 +40,7 @@ archive_r() {
 
 fetch_r_source() {
   echo "Downloading R-${1}"
-  wget -q ${CRAN}/src/base/R-3/R-${1}.tar.gz -O /tmp/R-${1}.tar.gz
+  wget -q "${CRAN}/src/base/R-`echo ${1}| awk 'BEGIN {FS="."} {print $1}'`/R-${1}.tar.gz" -O /tmp/R-${1}.tar.gz
   echo "Extracting R-${1}"
   tar xf /tmp/R-${1}.tar.gz -C /tmp
   rm /tmp/R-${1}.tar.gz

builder/package.ubuntu-1604

@@ -33,6 +33,7 @@ fpm \
   -d libpango-1.0-0 \
   -d libpangocairo-1.0-0 \
   -d libpaper-utils \
+  -d libpcre2-8-0 \
   -d libpcre3 \
   -d libpng16-16 \
   -d libreadline6 \

builder/package.ubuntu-1804

@@ -33,6 +33,7 @@ fpm \
   -d libpango-1.0-0 \
   -d libpangocairo-1.0-0 \
   -d libpaper-utils \
+  -d libpcre2-8-0 \
   -d libpcre3 \
   -d libpng16-16 \
   -d libreadline7 \

handler.py

@@ -5,7 +5,8 @@ import json
 import boto3
 import botocore
 
-CRAN_SRC_R_URL = 'https://cran.rstudio.com/src/base/R-3/'
+CRAN_SRC_R3_URL = 'https://cran.rstudio.com/src/base/R-3/'
+CRAN_SRC_R4_URL = 'https://cran.rstudio.com/src/base/R-4/'
 batch_client = boto3.client('batch', region_name='us-east-1')
 
 
@@ -23,9 +24,9 @@ def _persist_r_versions(data):
     obj.put(Body=json.dumps(data), ContentType='application/json')
 
 
-def _cran_r_versions():
+def _cran_r_versions(url):
     """Perform a lookup of CRAN-known R version."""
-    r = requests.get(CRAN_SRC_R_URL)
+    r = requests.get(url)
     soup = BeautifulSoup(r.text, 'html.parser')
     r_versions = []
     for link in soup.find_all('a'):
@@ -34,6 +35,14 @@ def _cran_r_versions():
             v = href.replace('.tar.gz', '').replace('R-', '')
             if '-revised' not in v:  # reject 3.2.4-revised
                 r_versions.append(v)
+    return r_versions
+
+
+def _cran_all_r_versions():
+    """Perform a lookup of CRAN-known R version."""
+    r_versions = []
+    r_versions.extend(_cran_r_versions(CRAN_SRC_R3_URL))
+    r_versions.extend(_cran_r_versions(CRAN_SRC_R4_URL))
     return {'r_versions': r_versions}
 
 
@@ -86,13 +95,13 @@ def _submit_job(version, platform):
 
 
 def _versions_to_build(force):
-    cran_versions = _cran_r_versions()['r_versions']
+    cran_versions = _cran_all_r_versions()['r_versions']
     known_versions = _known_r_versions()['r_versions']
     new_versions = _compare_versions(cran_versions, known_versions)
 
     if len(new_versions) > 0:
         print('New R Versions found: %s' % new_versions)
-        _persist_r_versions(_cran_r_versions())
+        _persist_r_versions(_cran_all_r_versions())
 
     if force in [True, 'True', 'true']:
         return cran_versions