From 6c9247cfead7d302bbed9a16d86228d862651398 Mon Sep 17 00:00:00 2001 From: Colin Rundel Date: Tue, 25 Jul 2023 17:35:32 -0400 Subject: [PATCH] Attempt at adding builds for Fedora 37 and 38 --- Makefile | 2 +- README.md | 20 ++++++ builder/Dockerfile.fedora-37 | 86 ++++++++++++++++++++++++++ builder/Dockerfile.fedora-38 | 86 ++++++++++++++++++++++++++ builder/docker-compose.yml | 24 ++++++++ builder/package.fedora-37 | 116 +++++++++++++++++++++++++++++++++++ builder/package.fedora-38 | 116 +++++++++++++++++++++++++++++++++++ serverless-resources.yml | 28 +++++++++ serverless.yml | 6 +- test/docker-compose.yml | 16 +++++ 10 files changed, 498 insertions(+), 2 deletions(-) create mode 100644 builder/Dockerfile.fedora-37 create mode 100644 builder/Dockerfile.fedora-38 create mode 100644 builder/package.fedora-37 create mode 100644 builder/package.fedora-38 diff --git a/Makefile b/Makefile index 1cdad07..4c9daf8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PLATFORMS := ubuntu-2004 ubuntu-2204 debian-10 debian-11 debian-12 centos-7 centos-8 rhel-9 opensuse-154 +PLATFORMS := ubuntu-2004 ubuntu-2204 debian-10 debian-11 debian-12 centos-7 centos-8 rhel-9 opensuse-154 fedora-37 fedora-38 SLS_BINARY ?= ./node_modules/serverless/bin/serverless.js deps: diff --git a/README.md b/README.md index 7503c0a..e7513de 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ R binaries are built for the following Linux operating systems: - Red Hat Enterprise Linux 7, 8, 9 - openSUSE 15.4 - SUSE Linux Enterprise 15 SP4 +- Fedora 37, 38 Operating systems are supported until their vendor end-of-support dates, which can be found on the [Posit Platform Support](https://posit.co/about/platform-support/) @@ -146,6 +147,25 @@ Then install the package: sudo zypper --no-gpg-checks install R-${R_VERSION}-1-1.x86_64.rpm ``` +Download the deb package: +```bash +# Debian 10 +curl -O https://cdn.posit.co/r/fedora-37/pkgs/r-${R_VERSION}_1_amd64.rpm + +# Debian 11 +curl -O https://cdn.posit.co/r/fedora-38/pkgs/r-${R_VERSION}_1_amd64.rpm + +# Debian 12 +curl -O https://cdn.posit.co/r/debian-12/pkgs/r-${R_VERSION}_1_amd64.deb +``` + +Then install the package: +```bash +sudo dnf install r-${R_VERSION}_1_amd64.rpm +``` + + + ### Verify R installation Test that R was successfully installed by running: diff --git a/builder/Dockerfile.fedora-37 b/builder/Dockerfile.fedora-37 new file mode 100644 index 0000000..1d62365 --- /dev/null +++ b/builder/Dockerfile.fedora-37 @@ -0,0 +1,86 @@ +FROM fedora:37 + +ENV OS_IDENTIFIER fedora-37 + +RUN dnf -y upgrade \ + && dnf -y install dnf-plugins-core \ + && dnf -y install \ + autoconf \ + automake \ + bzip2-devel \ + cairo-devel \ + flexiblas-devel \ + gcc-c++ \ + gcc-gfortran \ + java-11-openjdk-devel \ + libICE-devel \ + libSM-devel \ + libX11-devel \ + libXmu-devel \ + libXt-devel \ + libcurl-devel \ + libicu-devel \ + libjpeg-devel \ + libpng-devel \ + libtiff-devel \ + libtool \ + make \ + ncurses-devel \ + pango-devel \ + pcre-devel \ + pcre2-devel \ + readline-devel \ + rpm-build \ + tcl-devel \ + tex \ + texinfo-tex \ + texlive-collection-latexrecommended \ + tk-devel \ + unzip \ + valgrind-devel \ + which \ + wget \ + xz-devel \ + zlib-devel \ + && dnf clean all + +# Install AWS CLI +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + ./aws/install && \ + rm -rf aws awscliv2.zip + +RUN if [ "$(arch)" == "aarch64" ]; then echo arm64; else echo amd64; fi > /tmp/arch + +RUN curl -LO "https://github.com/goreleaser/nfpm/releases/download/v2.18.1/nfpm_$(cat /tmp/arch).rpm" && \ + dnf install -y "nfpm_$(cat /tmp/arch).rpm" && \ + rm "nfpm_$(cat /tmp/arch).rpm" + +RUN chmod 0777 /opt + +# Configure flags for RHEL 9 that don't use the defaults in build.sh +ENV CONFIGURE_OPTIONS="\ + --enable-R-shlib \ + --with-tcltk \ + --enable-memory-profiling \ + --with-x \ + --with-system-valgrind-headers \ + --with-tcl-config=/usr/lib64/tclConfig.sh \ + --with-tk-config=/usr/lib64/tkConfig.sh \ + --enable-prebuilt-html" + +# Make sure that patching Java does not break R. +# R's default JAVA_HOME path includes the exact Java version on CentOS/RHEL, which +# requires users to run `R CMD javareconf` even on minor/patch upgrades. Use the +# major version symlink to avoid this. +# https://cran.r-project.org/doc/manuals/r-release/R-admin.html#Java-support +# https://solutions.posit.co/envs-pkgs/using-rjava/ +ENV JAVA_HOME=/usr/lib/jvm/jre-11-openjdk + +# R 3.x requires PCRE2 for Pango support on RHEL 9 +ENV INCLUDE_PCRE2_IN_R_3 yes + +COPY package.fedora-37 /package.sh +COPY build.sh . +COPY patches /patches +ENTRYPOINT ./build.sh diff --git a/builder/Dockerfile.fedora-38 b/builder/Dockerfile.fedora-38 new file mode 100644 index 0000000..103adbe --- /dev/null +++ b/builder/Dockerfile.fedora-38 @@ -0,0 +1,86 @@ +FROM fedora:38 + +ENV OS_IDENTIFIER fedora-38 + +RUN dnf -y upgrade \ + && dnf -y install dnf-plugins-core \ + && dnf -y install \ + autoconf \ + automake \ + bzip2-devel \ + cairo-devel \ + flexiblas-devel \ + gcc-c++ \ + gcc-gfortran \ + java-11-openjdk-devel \ + libICE-devel \ + libSM-devel \ + libX11-devel \ + libXmu-devel \ + libXt-devel \ + libcurl-devel \ + libicu-devel \ + libjpeg-devel \ + libpng-devel \ + libtiff-devel \ + libtool \ + make \ + ncurses-devel \ + pango-devel \ + pcre-devel \ + pcre2-devel \ + readline-devel \ + rpm-build \ + tcl-devel \ + tex \ + texinfo-tex \ + texlive-collection-latexrecommended \ + tk-devel \ + unzip \ + valgrind-devel \ + which \ + wget \ + xz-devel \ + zlib-devel \ + && dnf clean all + +# Install AWS CLI +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + ./aws/install && \ + rm -rf aws awscliv2.zip + +RUN if [ "$(arch)" == "aarch64" ]; then echo arm64; else echo amd64; fi > /tmp/arch + +RUN curl -LO "https://github.com/goreleaser/nfpm/releases/download/v2.18.1/nfpm_$(cat /tmp/arch).rpm" && \ + dnf install -y "nfpm_$(cat /tmp/arch).rpm" && \ + rm "nfpm_$(cat /tmp/arch).rpm" + +RUN chmod 0777 /opt + +# Configure flags for RHEL 9 that don't use the defaults in build.sh +ENV CONFIGURE_OPTIONS="\ + --enable-R-shlib \ + --with-tcltk \ + --enable-memory-profiling \ + --with-x \ + --with-system-valgrind-headers \ + --with-tcl-config=/usr/lib64/tclConfig.sh \ + --with-tk-config=/usr/lib64/tkConfig.sh \ + --enable-prebuilt-html" + +# Make sure that patching Java does not break R. +# R's default JAVA_HOME path includes the exact Java version on CentOS/RHEL, which +# requires users to run `R CMD javareconf` even on minor/patch upgrades. Use the +# major version symlink to avoid this. +# https://cran.r-project.org/doc/manuals/r-release/R-admin.html#Java-support +# https://solutions.posit.co/envs-pkgs/using-rjava/ +ENV JAVA_HOME=/usr/lib/jvm/jre-11-openjdk + +# R 3.x requires PCRE2 for Pango support on RHEL 9 +ENV INCLUDE_PCRE2_IN_R_3 yes + +COPY package.fedora-38 /package.sh +COPY build.sh . +COPY patches /patches +ENTRYPOINT ./build.sh diff --git a/builder/docker-compose.yml b/builder/docker-compose.yml index 8f3bd3d..b8b0407 100644 --- a/builder/docker-compose.yml +++ b/builder/docker-compose.yml @@ -107,3 +107,27 @@ services: image: r-builds:opensuse-154 volumes: - ./integration/tmp:/tmp/output + fedora-37: + command: ./build.sh + environment: + - R_VERSION=${R_VERSION} + - R_INSTALL_PATH=${R_INSTALL_PATH} + - LOCAL_STORE=/tmp/output + build: + context: . + dockerfile: Dockerfile.fedora-37 + image: r-builds:fedora-37 + volumes: + - ./integration/tmp:/tmp/output + fedora-38: + command: ./build.sh + environment: + - R_VERSION=${R_VERSION} + - R_INSTALL_PATH=${R_INSTALL_PATH} + - LOCAL_STORE=/tmp/output + build: + context: . + dockerfile: Dockerfile.fedora-38 + image: r-builds:fedora-38 + volumes: + - ./integration/tmp:/tmp/output \ No newline at end of file diff --git a/builder/package.fedora-37 b/builder/package.fedora-37 new file mode 100644 index 0000000..1c3d03b --- /dev/null +++ b/builder/package.fedora-37 @@ -0,0 +1,116 @@ +#!/bin/bash + +if [[ ! -d /tmp/output/${OS_IDENTIFIER} ]]; then + mkdir -p "/tmp/output/${OS_IDENTIFIER}" +fi + +# R 3.x requires PCRE1. On RHEL 9, R 3.x also requires PCRE2 for Pango support. +pcre_libs='- pcre2-devel' +if [[ "${R_VERSION}" =~ ^3 ]]; then + pcre_libs='- pcre2-devel +- pcre-devel' +fi + +# On RHEL and SUSE, we link R against the internal shared BLAS to make the +# R binaries more portable and allow users to switch BLAS implementations without +# having to recompile R. We default to OpenBLAS, but users may prefer other implementations. +# +# Binary packages built against the shared BLAS are also more portable and may be used +# with the default R distributed by RHEL/SUSE, or other R installations using +# shared BLAS and configured with a different BLAS (such as Microsoft R Open with MKL). +# This is especially important for Posit Package Manager's binary packages. +# +# However, EPEL 9's R now links externally against FlexiBLAS, which provides a +# native BLAS switching mechanism for Fedora/RHEL. Starting with R 4.3.0, we also +# link to external FlexiBLAS for compatibility. +# https://fedoraproject.org/wiki/Changes/FlexiBLAS_as_BLAS/LAPACK_manager +# +# On Ubuntu/Debian, we link R against the external BLAS instead (--with-blas/--with-lapack), +# as those distributions use the alternatives system to swap BLAS libraries at runtime. +# The default R distributions on Ubuntu/Debian use the external BLAS, so we do as well +# for portability. +# +# https://cran.r-project.org/doc/manuals/r-release/R-admin.html#Shared-BLAS +if [[ "$("${R_INSTALL_PATH}/bin/R" CMD config BLAS_LIBS)" == "-lflexiblas" ]]; then + blas_lib='flexiblas-devel' + + # Create postremove script to remove empty directories, as nFPM doesn't include them in the RPM files. + cat <> /after-remove.sh + if [ -d ${R_INSTALL_PATH} ]; then + rm -r ${R_INSTALL_PATH} + fi +EOF + + scripts="scripts: + postremove: /after-remove.sh +" +else + blas_lib='openblas-devel' + + # Create post-install script required for OpenBLAS. + cat <> /post-install.sh + mv ${R_INSTALL_PATH}/lib/R/lib/libRblas.so ${R_INSTALL_PATH}/lib/R/lib/libRblas.so.keep + ln -s /usr/lib64/libopenblasp.so ${R_INSTALL_PATH}/lib/R/lib/libRblas.so +EOF + + # Create after-remove script to remove internal BLAS, which won't be cleaned up automatically. + cat <> /after-remove.sh + if [ -d ${R_INSTALL_PATH} ]; then + rm -r ${R_INSTALL_PATH} + fi +EOF + + scripts="scripts: + postinstall: /post-install.sh + postremove: /after-remove.sh +" +fi + +if [ "$(arch)" == "aarch64" ]; then echo arm64; else echo amd64; fi > /tmp/arch + +cat < /tmp/nfpm.yml +name: R-${R_VERSION} +version: 1 +version_schema: none +arch: $(cat /tmp/arch) +release: 1 +maintainer: Posit Software, PBC +description: | + GNU R statistical computation and graphics system +vendor: Posit Software, PBC +homepage: https://www.r-project.org +license: GPLv2+ +depends: +- bzip2-devel +- gcc +- gcc-c++ +- gcc-gfortran +- libcurl-devel +- libicu-devel +- libSM +- libtiff +- libXmu +- libXt +- make +- ${blas_lib} +- pango +${pcre_libs} +- tcl +- tk +- unzip +- which +- xz-devel +- zip +- zlib-devel +contents: +- src: ${R_INSTALL_PATH} + dst: ${R_INSTALL_PATH} +${scripts} +EOF + +nfpm package \ + -f /tmp/nfpm.yml \ + -p rpm \ + -t "/tmp/output/${OS_IDENTIFIER}" + +export PKG_FILE=$(ls /tmp/output/${OS_IDENTIFIER}/R-${R_VERSION}*.rpm | head -1) diff --git a/builder/package.fedora-38 b/builder/package.fedora-38 new file mode 100644 index 0000000..1c3d03b --- /dev/null +++ b/builder/package.fedora-38 @@ -0,0 +1,116 @@ +#!/bin/bash + +if [[ ! -d /tmp/output/${OS_IDENTIFIER} ]]; then + mkdir -p "/tmp/output/${OS_IDENTIFIER}" +fi + +# R 3.x requires PCRE1. On RHEL 9, R 3.x also requires PCRE2 for Pango support. +pcre_libs='- pcre2-devel' +if [[ "${R_VERSION}" =~ ^3 ]]; then + pcre_libs='- pcre2-devel +- pcre-devel' +fi + +# On RHEL and SUSE, we link R against the internal shared BLAS to make the +# R binaries more portable and allow users to switch BLAS implementations without +# having to recompile R. We default to OpenBLAS, but users may prefer other implementations. +# +# Binary packages built against the shared BLAS are also more portable and may be used +# with the default R distributed by RHEL/SUSE, or other R installations using +# shared BLAS and configured with a different BLAS (such as Microsoft R Open with MKL). +# This is especially important for Posit Package Manager's binary packages. +# +# However, EPEL 9's R now links externally against FlexiBLAS, which provides a +# native BLAS switching mechanism for Fedora/RHEL. Starting with R 4.3.0, we also +# link to external FlexiBLAS for compatibility. +# https://fedoraproject.org/wiki/Changes/FlexiBLAS_as_BLAS/LAPACK_manager +# +# On Ubuntu/Debian, we link R against the external BLAS instead (--with-blas/--with-lapack), +# as those distributions use the alternatives system to swap BLAS libraries at runtime. +# The default R distributions on Ubuntu/Debian use the external BLAS, so we do as well +# for portability. +# +# https://cran.r-project.org/doc/manuals/r-release/R-admin.html#Shared-BLAS +if [[ "$("${R_INSTALL_PATH}/bin/R" CMD config BLAS_LIBS)" == "-lflexiblas" ]]; then + blas_lib='flexiblas-devel' + + # Create postremove script to remove empty directories, as nFPM doesn't include them in the RPM files. + cat <> /after-remove.sh + if [ -d ${R_INSTALL_PATH} ]; then + rm -r ${R_INSTALL_PATH} + fi +EOF + + scripts="scripts: + postremove: /after-remove.sh +" +else + blas_lib='openblas-devel' + + # Create post-install script required for OpenBLAS. + cat <> /post-install.sh + mv ${R_INSTALL_PATH}/lib/R/lib/libRblas.so ${R_INSTALL_PATH}/lib/R/lib/libRblas.so.keep + ln -s /usr/lib64/libopenblasp.so ${R_INSTALL_PATH}/lib/R/lib/libRblas.so +EOF + + # Create after-remove script to remove internal BLAS, which won't be cleaned up automatically. + cat <> /after-remove.sh + if [ -d ${R_INSTALL_PATH} ]; then + rm -r ${R_INSTALL_PATH} + fi +EOF + + scripts="scripts: + postinstall: /post-install.sh + postremove: /after-remove.sh +" +fi + +if [ "$(arch)" == "aarch64" ]; then echo arm64; else echo amd64; fi > /tmp/arch + +cat < /tmp/nfpm.yml +name: R-${R_VERSION} +version: 1 +version_schema: none +arch: $(cat /tmp/arch) +release: 1 +maintainer: Posit Software, PBC +description: | + GNU R statistical computation and graphics system +vendor: Posit Software, PBC +homepage: https://www.r-project.org +license: GPLv2+ +depends: +- bzip2-devel +- gcc +- gcc-c++ +- gcc-gfortran +- libcurl-devel +- libicu-devel +- libSM +- libtiff +- libXmu +- libXt +- make +- ${blas_lib} +- pango +${pcre_libs} +- tcl +- tk +- unzip +- which +- xz-devel +- zip +- zlib-devel +contents: +- src: ${R_INSTALL_PATH} + dst: ${R_INSTALL_PATH} +${scripts} +EOF + +nfpm package \ + -f /tmp/nfpm.yml \ + -p rpm \ + -t "/tmp/output/${OS_IDENTIFIER}" + +export PKG_FILE=$(ls /tmp/output/${OS_IDENTIFIER}/R-${R_VERSION}*.rpm | head -1) diff --git a/serverless-resources.yml b/serverless-resources.yml index cd846bf..9e8f6db 100644 --- a/serverless-resources.yml +++ b/serverless-resources.yml @@ -257,6 +257,34 @@ rBuildsBatchJobDefinitionOpensuse154: Image: "#{AWS::AccountId}.dkr.ecr.#{AWS::Region}.amazonaws.com/r-builds:opensuse-154" Timeout: AttemptDurationSeconds: 7200 +rBuildsBatchJobDefinitionFedora37: + Type: AWS::Batch::JobDefinition + Properties: + Type: container + ContainerProperties: + Command: + - ./build.sh + Vcpus: 4 + Memory: 4096 + JobRoleArn: + "Fn::GetAtt": [ rBuildsEcsTaskIamRole, Arn ] + Image: "#{AWS::AccountId}.dkr.ecr.#{AWS::Region}.amazonaws.com/r-builds:fedora-37" + Timeout: + AttemptDurationSeconds: 7200 +rBuildsBatchJobDefinitionFedora38: + Type: AWS::Batch::JobDefinition + Properties: + Type: container + ContainerProperties: + Command: + - ./build.sh + Vcpus: 4 + Memory: 4096 + JobRoleArn: + "Fn::GetAtt": [ rBuildsEcsTaskIamRole, Arn ] + Image: "#{AWS::AccountId}.dkr.ecr.#{AWS::Region}.amazonaws.com/r-builds:fedora-38" + Timeout: + AttemptDurationSeconds: 7200 # step function cloudwatch event trigger resources rBuildsEventRuleIamRole: diff --git a/serverless.yml b/serverless.yml index 3b4af82..f857a4e 100644 --- a/serverless.yml +++ b/serverless.yml @@ -63,7 +63,11 @@ provider: Ref: rBuildsBatchJobDefinitionRhel9 JOB_DEFINITION_ARN_opensuse_154: Ref: rBuildsBatchJobDefinitionOpensuse154 - SUPPORTED_PLATFORMS: ubuntu-2004,ubuntu-2204,debian-10,debian-11,debian-12,centos-7,centos-8,rhel-9,opensuse-154 + JOB_DEFINITION_ARN_fedora_37: + Ref: rBuildsBatchJobDefinitionFedora37 + JOB_DEFINITION_ARN_fedora_38: + Ref: rBuildsBatchJobDefinitionFedora38 + SUPPORTED_PLATFORMS: ubuntu-2004,ubuntu-2204,debian-10,debian-11,debian-12,centos-7,centos-8,rhel-9,opensuse-154,fedora-37,fedora-38 functions: queueBuilds: diff --git a/test/docker-compose.yml b/test/docker-compose.yml index f44e77e..4ba742b 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -71,3 +71,19 @@ services: - R_VERSION=${R_VERSION} volumes: - ../:/r-builds + fedora-37: + image: fedora:37 + command: /r-builds/test/test-yum.sh + environment: + - OS_IDENTIFIER=fedora-37 + - R_VERSION=${R_VERSION} + volumes: + - ../:/r-builds + fedora-38: + image: fedora:38 + command: /r-builds/test/test-yum.sh + environment: + - OS_IDENTIFIER=fedora-38 + - R_VERSION=${R_VERSION} + volumes: + - ../:/r-builds