From 833b44f31591152d33aa2ee0c2efa3725f460136 Mon Sep 17 00:00:00 2001 From: Jonathan Curran Date: Thu, 23 Apr 2020 08:40:24 -0600 Subject: [PATCH 1/4] Add Ubuntu 20 support --- Makefile | 2 +- README.md | 5 +++- builder/Dockerfile.ubuntu-2004 | 23 +++++++++++++++ builder/docker-compose.yml | 11 +++++++ builder/package.ubuntu-2004 | 53 ++++++++++++++++++++++++++++++++++ serverless-resources.yml | 12 ++++++++ serverless.yml | 4 ++- 7 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 builder/Dockerfile.ubuntu-2004 create mode 100644 builder/package.ubuntu-2004 diff --git a/Makefile b/Makefile index 9cd493c..329addf 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PLATFORMS := ubuntu-1604 ubuntu-1804 debian-9 centos-6 centos-7 centos-8 opensuse-42 opensuse-15 +PLATFORMS := ubuntu-1604 ubuntu-1804 ubuntu-2004 debian-9 centos-6 centos-7 centos-8 opensuse-42 opensuse-15 SLS_BINARY ?= ./node_modules/serverless/bin/serverless deps: diff --git a/README.md b/README.md index 4fb898f..1170fd7 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ bug, or ask questions on [RStudio Community](https://community.rstudio.com). ## Supported Platforms R binaries are built for the following Linux operating systems: -- Ubuntu 16.04, 18.04 +- Ubuntu 16.04, 18.04, 20.04 - Debian 9 - CentOS / Red Hat Enterprise Linux 6, 7, 8 - openSUSE 42.3, 15.0 @@ -55,6 +55,9 @@ wget https://cdn.rstudio.com/r/ubuntu-1604/pkgs/r-${R_VERSION}_1_amd64.deb # Ubuntu 18.04 wget https://cdn.rstudio.com/r/ubuntu-1804/pkgs/r-${R_VERSION}_1_amd64.deb +# Ubuntu 20.04 +wget https://cdn.rstudio.com/r/ubuntu-2004/pkgs/r-${R_VERSION}_1_amd64.deb + # Debian 9 wget https://cdn.rstudio.com/r/debian-9/pkgs/r-${R_VERSION}_1_amd64.deb ``` diff --git a/builder/Dockerfile.ubuntu-2004 b/builder/Dockerfile.ubuntu-2004 new file mode 100644 index 0000000..08db4f2 --- /dev/null +++ b/builder/Dockerfile.ubuntu-2004 @@ -0,0 +1,23 @@ +FROM ubuntu:focal + +ENV OS_IDENTIFIER ubuntu-2004 + +RUN set -x \ + && sed -i "s|# deb-src|deb-src|g" /etc/apt/sources.list \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get install -y libcurl4-openssl-dev libicu-dev libopenblas-base wget python3-pip ruby ruby-dev \ + && apt-get build-dep -y r-base + +RUN pip3 install awscli + +RUN gem install fpm + +RUN chmod 0777 /opt + +# Override the default pager used by R +ENV PAGER /usr/bin/pager + +COPY package.ubuntu-2004 /package.sh +COPY build.sh . +ENTRYPOINT ./build.sh diff --git a/builder/docker-compose.yml b/builder/docker-compose.yml index 7b6eaf4..82ad921 100644 --- a/builder/docker-compose.yml +++ b/builder/docker-compose.yml @@ -23,6 +23,17 @@ services: image: r-builds:ubuntu-1804 volumes: - ./integration/tmp:/tmp/output + ubuntu-2004: + command: ./build.sh + environment: + - R_VERSION=${R_VERSION} + - LOCAL_STORE=/tmp/output + build: + context: . + dockerfile: Dockerfile.ubuntu-2004 + image: r-builds:ubuntu-2004 + volumes: + - ./integration/tmp:/tmp/output debian-9: command: ./build.sh environment: diff --git a/builder/package.ubuntu-2004 b/builder/package.ubuntu-2004 new file mode 100644 index 0000000..9b03cd7 --- /dev/null +++ b/builder/package.ubuntu-2004 @@ -0,0 +1,53 @@ +#!/bin/bash + +if [[ ! -d /tmp/output/ubuntu-2004 ]]; then + mkdir -p /tmp/output/ubuntu-2004 +fi + +fpm \ + -s dir \ + -t deb \ + -v 1 \ + -n R-${R_VERSION} \ + --vendor "RStudio Inc." \ + --deb-priority "optional" \ + --deb-field 'Bugs: https://github.com/rstudio/r-builds/issues' \ + --url "http://www.r-project.org/" \ + --description "GNU R statistical computation and graphics system" \ + --maintainer "RStudio Inc https://github.com/rstudio/r-builds" \ + --license "GPL-2" \ + -p /tmp/output/ubuntu-2004/ \ + -d g++ \ + -d gcc \ + -d gfortran \ + -d libbz2-1.0 \ + -d libc6 \ + -d libcairo2 \ + -d libcurl4 \ + -d libglib2.0-0 \ + -d libgomp1 \ + -d libicu66 \ + -d libjpeg8 \ + -d liblzma5 \ + -d libopenblas-dev \ + -d libpango-1.0-0 \ + -d libpangocairo-1.0-0 \ + -d libpaper-utils \ + -d libpcre3 \ + -d libpng16-16 \ + -d libreadline8 \ + -d libtcl8.6 \ + -d libtiff5 \ + -d libtk8.6 \ + -d libx11-6 \ + -d libxt6 \ + -d make \ + -d ucf \ + -d unzip \ + -d zip \ + -d zlib1g \ + /opt/R/${R_VERSION} + +shopt -s extglob +export PKG_FILE=$(ls /tmp/output/ubuntu-2004/[rR]-${R_VERSION}*.@(deb|rpm) | head -1) + diff --git a/serverless-resources.yml b/serverless-resources.yml index 3a3f6b6..dcca0fa 100644 --- a/serverless-resources.yml +++ b/serverless-resources.yml @@ -155,6 +155,18 @@ rBuildsBatchJobDefinitionUbuntu1804: JobRoleArn: "Fn::GetAtt": [ rBuildsEcsTaskIamRole, Arn ] Image: "#{AWS::AccountId}.dkr.ecr.#{AWS::Region}.amazonaws.com/r-builds:ubuntu-1804" +rBuildsBatchJobDefinitionUbuntu2004: + Type: AWS::Batch::JobDefinition + Properties: + Type: container + ContainerProperties: + Command: + - ./build.sh + Vcpus: 4 + Memory: 4096 + JobRoleArn: + "Fn::GetAtt": [ rBuildsEcsTaskIamRole, Arn ] + Image: "#{AWS::AccountId}.dkr.ecr.#{AWS::Region}.amazonaws.com/r-builds:ubuntu-2004" rBuildsBatchJobDefinitionDebian9: Type: AWS::Batch::JobDefinition Properties: diff --git a/serverless.yml b/serverless.yml index 77089e0..cf75280 100644 --- a/serverless.yml +++ b/serverless.yml @@ -39,6 +39,8 @@ provider: Ref: rBuildsBatchJobDefinitionUbuntu1604 JOB_DEFINITION_ARN_ubuntu_1804: Ref: rBuildsBatchJobDefinitionUbuntu1804 + JOB_DEFINITION_ARN_ubuntu_2004: + Ref: rBuildsBatchJobDefinitionUbuntu2004 JOB_DEFINITION_ARN_debian_9: Ref: rBuildsBatchJobDefinitionDebian9 JOB_DEFINITION_ARN_centos_6: @@ -51,7 +53,7 @@ provider: Ref: rBuildsBatchJobDefinitionOpensuse42 JOB_DEFINITION_ARN_opensuse_15: Ref: rBuildsBatchJobDefinitionOpensuse15 - SUPPORTED_PLATFORMS: ubuntu-1604,ubuntu-1804,debian-9,centos-6,centos-7,centos-8,opensuse-42,opensuse-15 + SUPPORTED_PLATFORMS: ubuntu-1604,ubuntu-1804,ubuntu-2004,debian-9,centos-6,centos-7,centos-8,opensuse-42,opensuse-15 functions: queueBuilds: From c30d2f734ea5ec50f5b2d2e3a02db12f500730d3 Mon Sep 17 00:00:00 2001 From: Jonathan Curran Date: Fri, 24 Apr 2020 15:29:11 -0600 Subject: [PATCH 2/4] s/Inc/PBC/ --- builder/package.ubuntu-2004 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/package.ubuntu-2004 b/builder/package.ubuntu-2004 index 9b03cd7..8a061fb 100644 --- a/builder/package.ubuntu-2004 +++ b/builder/package.ubuntu-2004 @@ -9,12 +9,12 @@ fpm \ -t deb \ -v 1 \ -n R-${R_VERSION} \ - --vendor "RStudio Inc." \ + --vendor "RStudio PBC" \ --deb-priority "optional" \ --deb-field 'Bugs: https://github.com/rstudio/r-builds/issues' \ --url "http://www.r-project.org/" \ --description "GNU R statistical computation and graphics system" \ - --maintainer "RStudio Inc https://github.com/rstudio/r-builds" \ + --maintainer "RStudio, PBC https://github.com/rstudio/r-builds" \ --license "GPL-2" \ -p /tmp/output/ubuntu-2004/ \ -d g++ \ From 9471eaab3b717d9e8448736cd2d9474bb3bd610a Mon Sep 17 00:00:00 2001 From: Jonathan Curran Date: Mon, 27 Apr 2020 14:45:48 -0600 Subject: [PATCH 3/4] pcre --- builder/Dockerfile.ubuntu-2004 | 2 +- builder/package.ubuntu-2004 | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/builder/Dockerfile.ubuntu-2004 b/builder/Dockerfile.ubuntu-2004 index 08db4f2..8181332 100644 --- a/builder/Dockerfile.ubuntu-2004 +++ b/builder/Dockerfile.ubuntu-2004 @@ -6,7 +6,7 @@ RUN set -x \ && sed -i "s|# deb-src|deb-src|g" /etc/apt/sources.list \ && export DEBIAN_FRONTEND=noninteractive \ && apt-get update \ - && apt-get install -y libcurl4-openssl-dev libicu-dev libopenblas-base wget python3-pip ruby ruby-dev \ + && apt-get install -y libcurl4-openssl-dev libicu-dev libopenblas-base libpcre2-dev wget python3-pip ruby ruby-dev \ && apt-get build-dep -y r-base RUN pip3 install awscli diff --git a/builder/package.ubuntu-2004 b/builder/package.ubuntu-2004 index 8a061fb..18a60f0 100644 --- a/builder/package.ubuntu-2004 +++ b/builder/package.ubuntu-2004 @@ -9,7 +9,7 @@ fpm \ -t deb \ -v 1 \ -n R-${R_VERSION} \ - --vendor "RStudio PBC" \ + --vendor "RStudio, PBC" \ --deb-priority "optional" \ --deb-field 'Bugs: https://github.com/rstudio/r-builds/issues' \ --url "http://www.r-project.org/" \ @@ -33,6 +33,7 @@ fpm \ -d libpango-1.0-0 \ -d libpangocairo-1.0-0 \ -d libpaper-utils \ + -d libpcre2-8-0 \ -d libpcre3 \ -d libpng16-16 \ -d libreadline8 \ From 48c1c0faf4e226d9859011720f373e7c5415f542 Mon Sep 17 00:00:00 2001 From: Jonathan Curran Date: Mon, 27 Apr 2020 17:18:50 -0600 Subject: [PATCH 4/4] Ignore ubuntu 20 for zlib build bug --- handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handler.py b/handler.py index f146940..82c8ce0 100644 --- a/handler.py +++ b/handler.py @@ -125,7 +125,7 @@ def queue_builds(event, context): # In R 3.3.0, 3.3.1, and 3.3.2, the configure script check for the # zlib version fails to handle versions longer than 5 characters. # Skip builds affected by this bug. - if platform in ['ubuntu-1804', 'opensuse-15', 'centos-8'] and version in ['3.3.0', '3.3.1', '3.3.2']: + if platform in ['ubuntu-2004', 'ubuntu-1804', 'opensuse-15', 'centos-8'] and version in ['3.3.0', '3.3.1', '3.3.2']: continue job_ids.append(_submit_job(version, platform)) event['jobIds'] = job_ids