diff --git a/Makefile b/Makefile index dbf4696..716919d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PLATFORMS := ubuntu-1804 ubuntu-2004 ubuntu-2204 debian-9 debian-10 centos-7 centos-8 opensuse-42 opensuse-152 opensuse-153 +PLATFORMS := ubuntu-1804 ubuntu-2004 ubuntu-2204 debian-9 debian-10 debian-11 centos-7 centos-8 opensuse-42 opensuse-152 opensuse-153 SLS_BINARY ?= ./node_modules/serverless/bin/serverless deps: diff --git a/README.md b/README.md index 9affd91..b20597b 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,9 @@ wget https://cdn.rstudio.com/r/debian-9/pkgs/r-${R_VERSION}_1_amd64.deb # Debian 10 wget https://cdn.rstudio.com/r/debian-10/pkgs/r-${R_VERSION}_1_amd64.deb + +# Debian 11 +wget https://cdn.rstudio.com/r/debian-11/pkgs/r-${R_VERSION}_1_amd64.deb ``` Then install the package: diff --git a/builder/Dockerfile.debian-11 b/builder/Dockerfile.debian-11 new file mode 100644 index 0000000..ba4d1bd --- /dev/null +++ b/builder/Dockerfile.debian-11 @@ -0,0 +1,27 @@ +FROM debian:bullseye + +ENV OS_IDENTIFIER debian-11 + +RUN set -x \ + && export DEBIAN_FRONTEND=noninteractive \ + && echo 'deb-src http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list \ + && apt-get update \ + && apt-get install -y gcc libcurl4-openssl-dev libicu-dev \ + libopenblas-base libpcre2-dev make python3-pip ruby ruby-dev wget \ + && apt-get build-dep -y r-base + +RUN pip3 install awscli + +RUN chmod 0777 /opt + +RUN gem install fpm + +# Override the default pager used by R +ENV PAGER /usr/bin/pager + +# R 3.x requires PCRE2 for Pango support on Debian 11 +ENV INCLUDE_PCRE2_IN_R_3 yes + +COPY package.debian-11 /package.sh +COPY build.sh . +ENTRYPOINT ./build.sh diff --git a/builder/build.sh b/builder/build.sh index 8f77a36..dcdaf59 100755 --- a/builder/build.sh +++ b/builder/build.sh @@ -65,12 +65,34 @@ compile_r() { build_flag='' fi + # R 3.6.1 and below require additional compiler flags for GCC 10 and above + # (e.g., on Debian 11). Add -fcommon to CFLAGS to work around issues with new + # default of -fno-common for C (fixed in R 3.6.2). Add -fallow-argument-mismatch + # to FFLAGS to work around issues with changes to argument mismatch checking + # in Fortran (fixed in R 3.6.2, but not mentioned in NEWS). + # https://cran.r-project.org/doc/manuals/r-release/NEWS.3.html + # https://cran.r-project.org/doc/manuals/r-release/R-admin.html#Using-Fortran + # https://gcc.gnu.org/gcc-10/porting_to.html + gcc_major_version=$(gcc -dumpversion | cut -d '.' -f 1) + if _version_is_less_than "${R_VERSION}" 3.6.2 && _version_is_greater_than "${gcc_major_version}" 9; then + # Default CFLAGS/FFLAGS for all R 3.x versions is '-g -O2' when using GCC + export CFLAGS='-g -O2 -fcommon' + export FFLAGS='-g -O2 -fallow-argument-mismatch' + echo "Setting CFLAGS: ${CFLAGS}" + echo "Setting FFLAGS: ${FFLAGS}" + fi + # Avoid a PCRE2 dependency for R 3.5 and 3.6. R 3.x uses PCRE1, but R 3.5+ # will link against PCRE2 if present, although it is not actually used. # Since there's no way to disable this in the configure script, and we need # PCRE2 for R 4.x, we hide PCRE2 from the configure script by temporarily # removing the pkg-config file and pcre2-config script. - if [[ "${1}" =~ ^3 ]] && pkg-config --exists libpcre2-8; then + # + # The INCLUDE_PCRE2_IN_R_3 environment variable can be set to include PCRE2 + # in R 3.x builds, for distributions where PCRE2 is always required. + # In Debian 11, Pango now depends on PCRE2, so R 3.x will not be compiled with + # Pango support if the PCRE2 pkg-config file is missing. + if [[ "${1}" =~ ^3 ]] && pkg-config --exists libpcre2-8 && [ -z "$INCLUDE_PCRE2_IN_R_3" ]; then mkdir -p /tmp/pcre2 pc_dir=$(pkg-config --variable pcfiledir libpcre2-8) mv ${pc_dir}/libpcre2-8.pc /tmp/pcre2 @@ -145,6 +167,10 @@ _version_is_greater_than() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1" } +_version_is_less_than() { + test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$2" +} + ###### RUN R COMPILE PROCEDURE ###### set_up_environment fetch_r_source $R_VERSION diff --git a/builder/docker-compose.yml b/builder/docker-compose.yml index 9cdcdb4..c1335ef 100644 --- a/builder/docker-compose.yml +++ b/builder/docker-compose.yml @@ -56,6 +56,17 @@ services: image: r-builds:debian-10 volumes: - ./integration/tmp:/tmp/output + debian-11: + command: ./build.sh + environment: + - R_VERSION=${R_VERSION} + - LOCAL_STORE=/tmp/output + build: + context: . + dockerfile: Dockerfile.debian-11 + image: r-builds:debian-11 + volumes: + - ./integration/tmp:/tmp/output centos-7: command: ./build.sh environment: diff --git a/builder/package.debian-11 b/builder/package.debian-11 new file mode 100644 index 0000000..27f1d0f --- /dev/null +++ b/builder/package.debian-11 @@ -0,0 +1,59 @@ +#!/bin/bash + +if [[ ! -d /tmp/output/debian-11 ]]; then + mkdir -p /tmp/output/debian-11 +fi + +# R 3.x requires PCRE1. On Debian 11, R 3.x also requires PCRE2 for Pango support. +pcre_lib_flags='-d libpcre2-dev' +if [[ "${R_VERSION}" =~ ^3 ]]; then + pcre_lib_flags='-d libpcre2-dev -d libpcre3-dev' +fi + +fpm \ + -s dir \ + -t deb \ + -v 1 \ + -n R-${R_VERSION} \ + --vendor "RStudio, PBC" \ + --deb-priority "optional" \ + --deb-field 'Bugs: https://github.com/rstudio/r-builds/issues' \ + --url "http://www.r-project.org/" \ + --description "GNU R statistical computation and graphics system" \ + --maintainer "RStudio, PBC https://github.com/rstudio/r-builds" \ + --license "GPL-2" \ + -p /tmp/output/debian-11/ \ + -d ca-certificates \ + -d g++ \ + -d gcc \ + -d gfortran \ + -d libbz2-dev \ + -d libc6 \ + -d libcairo2 \ + -d libcurl4-openssl-dev \ + -d libglib2.0-0 \ + -d libgomp1 \ + -d libicu-dev \ + -d liblzma-dev \ + -d libopenblas-dev \ + -d libpango-1.0-0 \ + -d libpangocairo-1.0-0 \ + -d libpaper-utils \ + ${pcre_lib_flags} \ + -d libpng16-16 \ + -d libreadline8 \ + -d libtcl8.6 \ + -d libtiff5 \ + -d libtk8.6 \ + -d libx11-6 \ + -d libxt6 \ + -d make \ + -d ucf \ + -d unzip \ + -d zip \ + -d zlib1g-dev \ + /opt/R/${R_VERSION} + +shopt -s extglob +export PKG_FILE=$(ls /tmp/output/debian-11/[rR]-${R_VERSION}*.@(deb|rpm) | head -1) + diff --git a/handler.py b/handler.py index 34d0b51..56969cc 100644 --- a/handler.py +++ b/handler.py @@ -136,6 +136,7 @@ def queue_builds(event, context): 'opensuse-153', 'centos-8', 'debian-10', + 'debian-11', ] and version in ['3.3.0', '3.3.1', '3.3.2']: continue job_ids.append(_submit_job(version, platform)) diff --git a/serverless-resources.yml b/serverless-resources.yml index 6496b39..9fae387 100644 --- a/serverless-resources.yml +++ b/serverless-resources.yml @@ -201,6 +201,20 @@ rBuildsBatchJobDefinitionDebian10: Image: "#{AWS::AccountId}.dkr.ecr.#{AWS::Region}.amazonaws.com/r-builds:debian-10" Timeout: AttemptDurationSeconds: 7200 +rBuildsBatchJobDefinitionDebian11: + Type: AWS::Batch::JobDefinition + Properties: + Type: container + ContainerProperties: + Command: + - ./build.sh + Vcpus: 4 + Memory: 4096 + JobRoleArn: + "Fn::GetAtt": [ rBuildsEcsTaskIamRole, Arn ] + Image: "#{AWS::AccountId}.dkr.ecr.#{AWS::Region}.amazonaws.com/r-builds:debian-11" + Timeout: + AttemptDurationSeconds: 7200 rBuildsBatchJobDefinitionCentos7: Type: AWS::Batch::JobDefinition Properties: diff --git a/serverless.yml b/serverless.yml index fdb9bab..400f18c 100644 --- a/serverless.yml +++ b/serverless.yml @@ -45,6 +45,8 @@ provider: Ref: rBuildsBatchJobDefinitionDebian9 JOB_DEFINITION_ARN_debian_10: Ref: rBuildsBatchJobDefinitionDebian10 + JOB_DEFINITION_ARN_debian_11: + Ref: rBuildsBatchJobDefinitionDebian11 JOB_DEFINITION_ARN_centos_7: Ref: rBuildsBatchJobDefinitionCentos7 JOB_DEFINITION_ARN_centos_8: @@ -55,7 +57,7 @@ provider: Ref: rBuildsBatchJobDefinitionOpensuse152 JOB_DEFINITION_ARN_opensuse_153: Ref: rBuildsBatchJobDefinitionOpensuse153 - SUPPORTED_PLATFORMS: ubuntu-1804,ubuntu-2004,ubuntu-2204,debian-9,debian-10,centos-7,centos-8,opensuse-42,opensuse-152,opensuse-153 + SUPPORTED_PLATFORMS: ubuntu-1804,ubuntu-2004,ubuntu-2204,debian-9,debian-10,debian-11,centos-7,centos-8,opensuse-42,opensuse-152,opensuse-153 functions: queueBuilds: