diff --git a/Justfile b/Justfile index 72988b1..8b13789 100644 --- a/Justfile +++ b/Justfile @@ -1,36 +1 @@ -# just image alpine 3.20 -# just image alpine 3.21 4.4.3 -# just image alpine 3.22 4.5.0 -# just image ubuntu jammy 4.4.3 -# just image ubuntu noble 4.4.3 -# just image redhat 8 4.4.3 -# just image redhat 9 4.4.3 -# just image redhat 10 -image OS OS_VERSION R_VERSION : - if echo "{{OS}}" | grep -qi "redhat"; then \ - docker buildx build --progress plain --build-arg R_VERSION={{R_VERSION}} --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg OS_VERSION={{OS_VERSION}} --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" --platform linux/amd64,linux/arm64 -f docker/Containerfile-{{OS}} -t devxygmbh/rpkgs-build-env-{{OS}}:{{OS_VERSION}}-{{R_VERSION}} --push .; \ - else \ - docker buildx build --progress plain --build-arg R_VERSION={{R_VERSION}} --build-arg OS_VERSION={{OS_VERSION}} --build-arg GITHUB_PAT="$GITHUB_PAT" -f docker/Containerfile-{{OS}} --platform linux/amd64,linux/arm64 -t devxygmbh/rpkgs-build-env-{{OS}}:{{OS_VERSION}}-{{R_VERSION}} --push .; \ - fi -### JAGS takes forever to build in CI through buildx. Building the binary once and injecting it into the final image -# just image-jags alpine 3.20 -# just image-jags alpine 3.21 -# just image-jags alpine 3.22 -# just image-jags ubuntu jammy -# just image-jags ubuntu noble -# just image-jags redhat 8 -# just image-jags redhat 9 -# just image-jags redhat 10 -image-jags OS OS_VERSION: - if echo "{{OS}}" | grep -qi "redhat"; then \ - docker buildx build --progress plain --platform linux/amd64,linux/arm64 --build-arg OS_VERSION={{OS_VERSION}} --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" -f docker/Containerfile-{{OS}}-jags --push -t devxygmbh/{{OS}}-jags:{{OS_VERSION}} .; \ - else \ - docker buildx build --progress plain --platform linux/amd64,linux/arm64 --build-arg OS_VERSION={{OS_VERSION}} -f docker/Containerfile-{{OS}}-jags --push -t devxygmbh/{{OS}}-jags:{{OS_VERSION}} .;\ - fi - -preview-docs: - cd docs && quarto preview - -publish-docs: - cd docs && quarto render && rclone sync ./_site hetzner-s3://devxy-r-package-binaries-docs diff --git a/docker/Containerfile-alpine b/docker/Containerfile-alpine deleted file mode 100644 index 58f7758..0000000 --- a/docker/Containerfile-alpine +++ /dev/null @@ -1,64 +0,0 @@ -ARG OS_VERSION -FROM devxygmbh/alpine-jags:${OS_VERSION} AS jags - -ARG OS_VERSION -FROM alpine:${OS_VERSION} AS final - -ARG R_VERSION=4.4.3 -ENV PATH=/opt/R/${R_VERSION}/bin:$PATH -ENV TZ="Europe/Berlin" -ENV R_LIBS_USER=/mnt/cache/R-pkgs -ENV R_PKG_CACHE_DIR=/mnt/cache/pkgcache - -ENV NCPUS=2 - -ARG GITHUB_PAT - -COPY --link --from=jags /JAGS-4.3.2 /JAGS-4.3.2 - -# Background: JAGS needs lapack & lapack-dev but these conflict with openblas-dev which is required by R-dev -# Solution: first compile JAGS, then remove lapack & lapack-dev again -# # xvfb: for graphical pkgs requiring tcl, like gWidgets2tcltk -RUN apk add -q --no-cache curl tar make blas blas-dev gcc gfortran g++ htop git cmake tk-dev xvfb xvfb-run xorg-server xauth xinit xz libdeflate libdeflate-dev - -### CUSTOM DEPENDENCIES - -# linux-headers: ps -# libxml2-dev: xml2 -# libgit2-dev: gert -# libpq-dev: RPostgres -# libfftw3-dev: ravetools -> actually auto-detect but somehow doesn't work in container? -RUN apk add -q --no-cache R-dev curl ca-certificates bash g++ tzdata openjdk17 tar sed patch openblas-dev pkgconfig linux-headers ccache libxml2-dev libgit2-dev libpq-dev pkgconfig fftw-dev automake autoconf libtool elfutils-dev -# - Chromium should be auto-installed once pkgdepends merges the latest system-requirements rule changes -RUN apk add -q --no-cache chromium - -# https://github.com/RcppCore/Rcpp/issues/448#issuecomment-220148774 -# LDFLAGS+=-fPIC -> arrow (https://github.com/r-hub/r-minimal/blob/main/examples/arrow/Dockerfile) -# dbarts: CFLAGS += -flax-vector-conversions (https://github.com/vdorie/dbarts/issues/66) -# CXXFLAGS: -> Rcpp -# -DDMLC_USE_FOPEN64=0 -> xgboost (https://github.com/dmlc/xgboost/issues/8595) -# can't set ccache for gfortran because of https://gitlab.com/roigrp/solver/highs/-/issues/4 -RUN mkdir -p /root/.R && bash -c 'echo -e "CXX_STD = CXX14\nCXXFLAGS=-D__MUSL__ -DDMLC_USE_FOPEN64=0\nCPPFLAGS=-D__MUSL__ -DDMLC_USE_FOPEN64=0\nCFLAGS += -flax-vector-conversions\nLDFLAGS+=-fPIC\nCC=ccache gcc\nCPP=gcc -E\nCXXCPP=gcc -e\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=gfortran\nFC=gfortran" > /root/.R/Makevars' - -# R repos -RUN bash -c 'echo -e "options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(CRAN = \"https://cloud.r-project.org\", INLA = \"https://inla.r-inla-download.org/R/stable\")))" > /root/.Rprofile' -RUN bash -c 'echo -e "PKG_SYSREQS=TRUE\nPKG_SYSREQS_VERBOSE=TRUE" > ~/.Renviron' - -### INSTALL R - -ARG OS_VERSION -RUN export OS_VERSION_TRIM=$(echo $OS_VERSION | tr -d '.') && \ - curl -v -O "https://devxy-r-builds.fsn1.your-objectstorage.com/alpine${OS_VERSION_TRIM}/r-${R_VERSION}_1_$(arch).apk" -RUN apk add --allow-untrusted "r-${R_VERSION}_1_$(arch).apk" && rm "r-${R_VERSION}_1_$(arch).apk" - -RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - -# these dirs will later be used by the CI with existing mounts -RUN mkdir -p /mnt/cache/packages /mnt/cache/pkgcache /mnt/cache/R-pkgs - -# encoding -> https://github.com/r-hub/r-minimal/blob/fa9c117313e05860a0d20b113e5f794c51be8b8c/Dockerfile#L84 -RUN sed -i 's/,//g' /opt/R/${R_VERSION}/lib/R/library/utils/iconvlist - -RUN R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git", dependencies = TRUE)' - -CMD ["/bin/bash"] diff --git a/docker/Containerfile-alpine-jags b/docker/Containerfile-alpine-jags deleted file mode 100644 index 5e6cc72..0000000 --- a/docker/Containerfile-alpine-jags +++ /dev/null @@ -1,11 +0,0 @@ -ARG VERSION -FROM alpine:${VERSION} AS jags - -ARG JAGS_VERSION=4.3.2 - -RUN apk add -q --no-cache curl tar make lapack lapack-dev blas blas-dev gcc gfortran g++ git cmake tk-dev xz - -# These usually are not packaged in OS repositories and need to be installed from source -# - JAGS (needed for runjags which itself is again a dependency of many packages) -RUN curl -s -L -O https://sourceforge.net/projects/mcmc-jags/files/JAGS/4.x/Source/JAGS-${JAGS_VERSION}.tar.gz && \ - tar -xzf JAGS-${JAGS_VERSION}.tar.gz && cd JAGS-${JAGS_VERSION} && ./configure --enable-silent-rules && make 2>&1 diff --git a/docker/Containerfile-redhat b/docker/Containerfile-redhat deleted file mode 100644 index 68aa291..0000000 --- a/docker/Containerfile-redhat +++ /dev/null @@ -1,68 +0,0 @@ -ARG OS_VERSION -FROM devxygmbh/redhat-jags:${OS_VERSION} AS jags - -ARG OS_VERSION -FROM redhat/ubi${OS_VERSION} AS final - -ARG R_VERSION -ARG RED_HAT_DEV_PW -ENV PATH=/opt/R/${R_VERSION}/bin:$PATH -ENV TZ="Europe/Berlin" -ENV R_LIBS_USER=/mnt/cache/R-pkgs -ENV R_PKG_CACHE_DIR=/mnt/cache/pkgcache - -ENV NCPUS=2 - -ARG GITHUB_PAT - -COPY --link --from=jags /JAGS-4.3.2 /JAGS-4.3.2 - -ARG OS_VERSION -RUN subscription-manager register --username mail@devxy.io --password "$RED_HAT_DEV_PW" && subscription-manager repos --enable codeready-builder-for-rhel-${OS_VERSION}-$(arch)-rpms -RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-${OS_VERSION}.noarch.rpm && dnf install -y epel-release -# xvfb: for graphical pkgs requiring tcl, like gWidgets2tcltk -# ImageMagick-c++-devel ImageMagick-devel: boths are not auto-installed via pak sysreqs -# mecab-devel: RcppMeCab -> no autodetection yet -# libfftw3-dev: ravetools -> actually auto-detect but somehow doesn't work in container? -ARG OS_VERSION -RUN if [ "$OS_VERSION" = "8" ]; then \ - dnf install -y -q make lapack-devel git gcc gcc-c++ fontconfig-devel fribidi-devel libjpeg-turbo-devel libtiff-devel postgresql-devel htop libgit2_1.7-devel libxml2-devel hiredis ccache cmake xorg-x11-server-Xvfb ImageMagick-c++-devel ImageMagick-devel xz pkg-config mecab-devel fftw-devel tk-devel automake readline-devel; \ - elif [ "$OS_VERSION" = "9" ] || [ "$OS_VERSION" = "10" ]; then \ - dnf install -y -q make lapack-devel git gcc gcc-c++ fontconfig-devel fribidi-devel libjpeg-turbo-devel libtiff-devel postgresql-devel htop libgit2-devel libxml2-devel hiredis ccache cmake xorg-x11-server-Xvfb ImageMagick-c++-devel ImageMagick-devel xz pkg-config mecab-devel fftw-devel tk-devel automake readline-devel; \ - fi; - -### CUSTOM DEPENDENCIES -# - Chromium should be auto-installed once pkgdepends merges the latest system-requirements rule changes -RUN dnf install -y chromium - -### INSTALL R -RUN curl -O "https://devxy-r-builds.fsn1.your-objectstorage.com/el${OS_VERSION}/R-${R_VERSION}-1-1.$(arch).rpm" -RUN dnf install -q -y "R-${R_VERSION}-1-1.$(arch).rpm" && rm "R-${R_VERSION}-1-1.$(arch).rpm" -RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - -RUN if [ "$OS_VERSION" = "8" ]; then \ - export CXX_STD=14; \ - elif [ "$OS_VERSION" = "9" ]; then \ - export CXX_STD=17; \ - elif [ "$OS_VERSION" = "10" ]; then \ - export CXX_STD=20; \ - fi; \ - if [ "$(arch)" = "x86_64" ]; then \ - export ARCH=amd64; \ - elif [ "$(arch)" = "aarch64" ]; then \ - export ARCH=arm64; \ - fi; \ - mkdir -p /root/.R && bash -c 'echo -e "CXX_STD = ${CXX_STD}\nCFLAGS += -flax-vector-conversions\nCC=ccache gcc\nCPP=gcc -E\nCXXCPP=gcc -e\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=gfortran\nFC=gfortran" >> /root/.R/Makevars' && \ - bash -c 'echo -e "options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(CRAN = \"https://cran.devxy.io/$ARCH/rhel${OS_VERSION}/latest\", INLA = \"https://inla.r-inla-download.org/R/stable\")))" > /root/.Rprofile' && \ - bash -c 'echo -e "PKG_SYSREQS = TRUE\nPKG_SYSREQS_VERBOSE = TRUE\nPKG_SYSREQS_PLATFORM=\"redhat-${OS_VERSION}\"" > /root/.Renviron' - -### Makevars -# dbarts: CFLAGS += -flax-vector-conversions (https://github.com/vdorie/dbarts/issues/66) -# can't set ccache for gfortran because of https://gitlab.com/roigrp/solver/highs/-/issues/4 - -# these dirs will later be used by the CI with existing mounts -RUN mkdir -p /mnt/cache/packages /mnt/cache/pkgcache /mnt/cache/R-pkgs - -RUN R -q -e 'getOption("repos"); pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git", dependencies = TRUE)' - -CMD ["/bin/bash"] diff --git a/docker/Containerfile-redhat-jags b/docker/Containerfile-redhat-jags deleted file mode 100644 index e1e06a5..0000000 --- a/docker/Containerfile-redhat-jags +++ /dev/null @@ -1,13 +0,0 @@ -ARG VERSION -FROM redhat/ubi${VERSION} AS jags -ARG RED_HAT_DEV_PW - -ARG VERSION -RUN subscription-manager register --username mail@devxy.io --password "$RED_HAT_DEV_PW" && subscription-manager repos --enable codeready-builder-for-rhel-${VERSION}-$(arch)-rpms -RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-${VERSION}.noarch.rpm && dnf install -y epel-release -RUN dnf install -y -q gcc-c++ lapack-devel make git ca-certificates - -# These usually are not packaged in OS repositories and need to be installed from source -# - JAGS (needed for runjags which itself is again a dependency of many packages) -RUN curl -s -L -O https://sourceforge.net/projects/mcmc-jags/files/JAGS/4.x/Source/JAGS-4.3.2.tar.gz && \ - tar -xzf JAGS-4.3.2.tar.gz && cd JAGS-4.3.2 && ./configure --enable-silent-rules && make 2>&1 diff --git a/docker/Containerfile-ubuntu b/docker/Containerfile-ubuntu deleted file mode 100644 index 70beca1..0000000 --- a/docker/Containerfile-ubuntu +++ /dev/null @@ -1,68 +0,0 @@ -ARG OS_VERSION -FROM devxygmbh/ubuntu-jags:${OS_VERSION} AS jags - -ARG OS_VERSION -FROM ubuntu:${OS_VERSION} AS final - -ARG R_VERSION -ENV PATH=/opt/R/${R_VERSION}/bin:$PATH -ENV TZ="Europe/Berlin" -ENV DEBIAN_FRONTEND=noninteractive - -ENV NCPUS=2 -ENV R_LIBS_USER=/mnt/cache/R-pkgs -ENV R_PKG_CACHE_DIR=/mnt/cache/pkgcache - -ARG GITHUB_PAT - -COPY --link --from=jags /JAGS-4.3.2 /JAGS-4.3.2 - -# xvfb: for graphical pkgs requiring tcl, like gWidgets2tcltk -# tcl tk tk-dev tk-table: must be available before running xvfb-run (i.e. starting the graphical session) -# libmecab-dev: RcppMeCab -> no autodetection yet -# libfftw3-dev: ravetools -> actually auto-detect but somehow doesn't work in container? -RUN apt update && apt install -y --no-install-recommends gdebi-core g++ gfortran libbz2-dev libblas-dev libicu-dev liblapack-dev liblzma-dev libpaper-utils libpcre2-dev libtcl8.6 libtk8.6 libxt6 unzip zip zlib1g-dev libpcre3-dev libopenblas-dev curl ca-certificates make ccache htop libgit2-dev git libpq-dev libxml2-dev libgit2-dev cmake xvfb libcurl4-openssl-dev xz-utils tcl tk tk-dev tk-table pkg-config libmecab-dev libfftw3-dev automake - -### CUSTOM DEPENDENCIES -# - Chromium should be auto-installed once pkgdepends merges the latest system-requirements rule changes -RUN apt-get install -y software-properties-common && \ - add-apt-repository -y ppa:xtradeb/apps && \ - apt update && \ - apt install -y chromium - -ARG OS_VERSION -ARG R_VERSION -RUN if [ "$OS_VERSION" = "jammy" ]; then \ - OS_VERSION_TRIM=2204; \ - CXX_STD=14; \ - elif [ "$OS_VERSION" = "noble" ]; then \ - OS_VERSION_TRIM=2404; \ - CXX_STD=17; \ - fi; \ - if [ "$(arch)" = "x86_64" ]; then \ - export ARCH=amd64; \ - elif [ "$(arch)" = "aarch64" ]; then \ - export ARCH=arm64; \ - fi; \ - curl -v -O "https://devxy-r-builds.fsn1.your-objectstorage.com/${OS_VERSION_TRIM}/r-${R_VERSION}_1_$(dpkg --print-architecture).deb" && \ - gdebi -n -qq "r-${R_VERSION}_1_$(dpkg --print-architecture).deb" && \ - rm "r-${R_VERSION}_1_$(dpkg --print-architecture).deb" && \ - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' && \ - mkdir -p /root/.R && \ - bash -c 'echo -e "CXX_STD = $CXX_STD\nCFLAGS += -flax-vector-conversions\nCC=ccache gcc\nCPP=gcc -E\nCXXCPP=gcc -e\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=gfortran\nFC=gfortran" >> /root/.R/Makevars' && \ - bash -c 'echo -e "options(pkg.sysreqs_verbose = TRUE, crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(CRAN = \"https://cran.devxy.io/$ARCH/${OS_VERSION}/latest\", INLA = \"https://inla.r-inla-download.org/R/stable\")))" > ~/.Rprofile' - -### Makevars -# dbarts: CFLAGS += -flax-vector-conversions (https://github.com/vdorie/dbarts/issues/66) -# can't set ccache for gfortran because of https://gitlab.com/roigrp/solver/highs/-/issues/4 - -# these dirs will later be used by the CI with existing mounts -RUN mkdir -p /mnt/cache/packages /mnt/cache/pkgcache /mnt/cache/R-pkgs - -RUN apt install -y locales && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \ - && locale-gen en_US.utf8 \ - && /usr/sbin/update-locale LANG=en_US.UTF-8 - -RUN R -q -e 'getOption("repos"); pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git", dependencies = TRUE)' - -CMD ["/bin/bash"] diff --git a/docker/Containerfile-ubuntu-jags b/docker/Containerfile-ubuntu-jags deleted file mode 100644 index 6842dcb..0000000 --- a/docker/Containerfile-ubuntu-jags +++ /dev/null @@ -1,7 +0,0 @@ -ARG VERSION -FROM ubuntu:${VERSION} AS jags - -RUN apt update && apt install -y --no-install-recommends curl make ca-certificates gcc g++ gfortran liblapack-dev -# These usually are not packaged in OS repositories and need to be installed from source -# - JAGS (needed for runjags which itself is again a dependency of many packages) -RUN curl -q -L -O https://sourceforge.net/projects/mcmc-jags/files/JAGS/4.x/Source/JAGS-4.3.2.tar.gz && tar -xzf JAGS-4.3.2.tar.gz && cd JAGS-4.3.2 && ./configure --enable-silent-rules && make 2>&1 diff --git a/docs/.gitignore b/docs/.gitignore deleted file mode 100644 index 075b254..0000000 --- a/docs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/.quarto/ diff --git a/docs/_brand.yaml b/docs/_brand.yaml deleted file mode 100644 index b2b110f..0000000 --- a/docs/_brand.yaml +++ /dev/null @@ -1,48 +0,0 @@ -color: - palette: - dark: "#1d262f" - orange: "#FF9B00" - links: "#15428c" - # background: blue - # foreground: dark-grey - primary: '#1d262f' # hyperlinks, active states, primary action buttons, etc. - tertiary: "#FF9B00" # hover states, accents, and wells. - -logo: - medium: - light: logo.png - dark: logo.png - -defaults: - bootstrap: - defaults: - navbar-fg: "#1d262f" - navbar-bg: "#ffffff" - -typography: - fonts: - - family: Jura - source: bunny - - family: Inter - source: bunny - - family: Montserrat - source: bunny - base: Inter - headings: Montserrat - link: - decoration: underline - color: links - monospace: - color: primary - background-color: "#f0f1f3" - size: 0.8rem - # decoration: - -meta: - link: - home: https://www.devxy.io - gitlab: https://gitlab.com/devxy - bluesky: https://bsky.app/profile/devxy.bsky.social - linkedin: https://www.linkedin.com/company/105100347 - description: Where Data Science and DevOps meet 🤝️ - founded: 2024 \ No newline at end of file diff --git a/docs/_quarto.yml b/docs/_quarto.yml deleted file mode 100644 index 09b5082..0000000 --- a/docs/_quarto.yml +++ /dev/null @@ -1,70 +0,0 @@ -project: - type: website - -website: - title: "CRAN R Package Binaries" - reader-mode: false - favicon: favicon.ico - back-to-top-navigation: true - navbar: - right: - - icon: gitlab - href: https://gitlab.com/devxy - - icon: browser-safari - href: https://www.devxy.io - - icon: bluesky - href: https://www.devxy.io - sidebar: - logo: "" - style: "docked" - collapse-level: 4 - search: true - border: true - pinned: true - contents: - - text: 'Introduction' - file: index.qmd - - text: "---" - - text: 'Configuration' - file: configuration/index.qmd - - section: "**Use Cases**" - contents: - - text: "CI/CD" - file: cicd/index.qmd - - text: "R Development Environments" - file: r-dev-environments/index.qmd - - text: "Containers" - file: containers/index.qmd - - section: "**Technical Background**" - contents: - - text: "Infrastructure" - file: infrastructure/index.qmd - # - text: "Continuous Updates" - # file: 032-continuous-updates.qmd - # - section: "**Miscellaneous**" - # contents: - # - text: "Private Mirror" - # file: 040-private-mirror.qmd - # - text: "Fair Usage" - # file: 041-fair-usage.qmd - - text: "---" - - text: 'Contributing & Help' - file: "contributing/index.qmd" - - search: - location: sidebar - type: "textbox" - limit: 5 - copy-button: true - collapse-after: 2 -format: - html: - theme: cosmo - css: styles.css - fontsize: 1rem - toc: true - grid: - sidebar-width: 300px - include-in-header: - text: | - diff --git a/docs/cicd/index.qmd b/docs/cicd/index.qmd deleted file mode 100644 index 115b097..0000000 --- a/docs/cicd/index.qmd +++ /dev/null @@ -1,82 +0,0 @@ ---- -title: CI/CD ---- - -Continuous Integration / continous delivery (CI/CD) is a basic tool in todays Data Science toolstack to automate processes and execute repetite tasks on schedule. - -Running tasks in CI/CD should be fast and reliable, no matter the language, architecture or computing environment at hand. -As CI/CD requires installing many packages over and over again, having binaries and optimized distributions helps to reduce runtime. - -This page presents snippets for well and lesser known CI/CD engines to help getting started with the R package binaries of this project. - -::: {.callout-note} -The presented snippets rely on external projects and libraries which get updated regurarly. -There is no guarantee to work out-of-the box at all times. -If you found an error, please open a pull request in the linked repo. -::: - -## GitHub Actions - -### VM - -GitHub Actions can be run directly on a VM or in a containered context. -The available VM images can be found [here](https://github.com/actions/runner-images#available-images). -The only Linux distribution available is Ubuntu. - -For VM workflows, [r-lib/actions](https://github.com/r-lib/actions/blob/v2/examples) provides many examples for different use cases. - -To use package binaries from this project, use the following config for `r-lib/actions/setup-r@v2`: - -```yaml - - uses: r-lib/actions/setup-r@v2 - with: - r-version: ${{ matrix.config.r }} - http-user-agent: ${{ matrix.config.http-user-agent }} - Ncpus: 2 - cran: 'https://cran.devxy.io/amd64/noble/latest' -``` - -(`Ncpus: 2` has been set to allow for parallel installations.) - -### Container - -An alternative is to run Actions in a containerized context. -This spins up a container of the selected image in the VM and allows running on other distributions than Ubuntu, e.g. on Alpine. -To avoid having to install R first and configure a custom repository every time, [specialized images](https://hub.docker.com/r/devxygmbh/r-alpine) are provided which already have everything in place: - -```yaml -jobs: - container: - runs-on: ubuntu-latest - container: devxygmbh:r-alpine - steps: - - run: | - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - R -q -e 'getOption("repos")' - R -q -e 'options(Ncpus = 2); pak::local_install_dev_deps()' -``` - -::: {.callout-note} -When running in a containerized context, the predefined actions from `r-lib/actions` cannot be used. -Shell/R commands must be used directly. -::: - -::: {.callout-note} -Running containerized in Alpine should be substantially faster than running in a VM context. -::: - -::: {.callout-tip} -(91 packages) - -- Containerized: **1m 21s** -- VM: 2m 11s - -Both runs are based on the assumption that caching is not used. With caching enabled, the performance of both approaches should be roughly comparable. - -Source: [pat-s/workflow-compare](https://github.com/pat-s/workflow-compare) -::: - - -## GitLab Runner - -WIP \ No newline at end of file diff --git a/docs/configuration/index.qmd b/docs/configuration/index.qmd deleted file mode 100644 index e3b06a4..0000000 --- a/docs/configuration/index.qmd +++ /dev/null @@ -1,66 +0,0 @@ ---- -title: Configuration ---- - -In R on Linux, packages are downloaded from CRAN as source packages. -The default CRAN mirror is `https://cloud.r-project.org`. -The default repository URL needs to be **actively changed** to one providing binaries for the respective Linux distribution. -This can be done either for all users of the system, on a project level or for specific users only. -All options will be shown in the following. - -## URL Scheme - -The URL scheme for this project is the same for all operating systems and only differs in the OS and architecture identifiers. - -::: {.callout-note icon=false} - -## URL scheme - -`https://cran.devxy.io///latest` -::: - -For example, the URL for Alpine Linux 3.21 for `arm64` would be `https://cran.devxy.io/arm64/alpine321/latest`. - -The architecture identifiers are: - -- `arm64` -- `amd64` - -The OS identifiers are: - -- `jammy` (Ubuntu 22.04) -- `noble` (Ubuntu 24.04) -- `rhel8` (RedHat Enterprise Linux 8) -- `rhel9` (RedHat Enterprise Linux 9) -- `alpine320` (Alpine Linux 3.20) -- `alpine321` (Alpine Linux 3.21) - -## Repository Configuration - -### System-wide - -To configure the default R repository for all users of a system, create/edit `$R_HOME/etc/Rprofile.site`. -The location of this file depends on how R has been installed. -To find its location, execute `R -q -e "paste0(Sys.getenv('R_HOME'),'/etc/Rprofile.site')"`. - -**At the bottom** of the file, add the following: - -```r -options(repos = structure(c(CRAN = ""))) -``` - -The name "CRAN" is optional. -It is the canonical default and in some cases also used for special actions (e.g. in Posit Workbench and [{pak}](https://github.com/r-lib/pak)). -Unless you have a good reason, it is advisable to stick with that name. - -### User-wide - -To change the default repository option for a specific *user*, add the line from above to `~/.Rprofile`. - -### Project-wide - -To change the default repository option for a specific *project*, add the line from above to `.Rprofile` in the project root directory in which R is started. - -## Verification - -When starting R, execute `getOption("repos")` to verify the repository is correctly configured. \ No newline at end of file diff --git a/docs/containers/index.qmd b/docs/containers/index.qmd deleted file mode 100644 index ad4cb98..0000000 --- a/docs/containers/index.qmd +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Containers ---- - -The [rocker](https://hub.docker.com/u/rocker) project provides container images for R since several years. -Since recently, the images are multi-arch, i.e. `arm64` is supported alongside `amd64`. - -Yet, other distributions than Ubuntu are missing and don't use the devXY binaries out of the box. -To simplify running containers that make use of the devXY R binaries implicitly, the following images are provided: - -- [devxygmbh/r-alpine](https://hub.docker.com/r/devxygmbh/r-alpine) -- [devxygmbh/r-ubuntu](https://hub.docker.com/r/devxygmbh/r-ubuntu) - -These have the devXY repositories configured by default and can be used efficiently without further configuration changes. - -In contrast to the minimal `rocker/r-ver` images, the `r-ubuntu` images contain a few additional system libraries which are essential for troublefree R usage. - -The following screencast shows how quickly the `r-alpine` image can be used to install the full {tidyverse} package universe in just under 30s: - -
- -::: {.callout-note} -The performance of this screencast (and your local experience) depends on a few factors: local download speed, cached binaries in the CDN, proximity to a local edge node of the CDN and your CPU performance and finally the `Ncpus` option in R, defining how many installations are performed in parallel (by default 1). -::: \ No newline at end of file diff --git a/docs/continuous-updates/index.qmd b/docs/continuous-updates/index.qmd deleted file mode 100644 index 6d3f665..0000000 --- a/docs/continuous-updates/index.qmd +++ /dev/null @@ -1 +0,0 @@ -WIP \ No newline at end of file diff --git a/docs/contributing/index.qmd b/docs/contributing/index.qmd deleted file mode 100644 index 3bf7d20..0000000 --- a/docs/contributing/index.qmd +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Contributing & Help ---- - -Contributions are welcome. -The project is open-source and GPL licensed. - -The most helpful tasks are to investigate in detail why specific packages failed to build a binary. -It usually comes down to a missing system dependency (which is either not declared in the package's DESCRIPTION file or exotic and must be installed from source) or a C code compile issue. - -Please open an issue in the [`r-package-binaries/support`](https://gitlab.com/devxy/r-package-binaries/support) repo. - -::: {.callout-note} -The actual repo executing the builds ([devxy/build-binaries](https://git.devxy.io/devxy/build-binaries)) is hosted in a private git instance to be able to use specific private CI runners and is not easily usable for creating public issues, hence the dedicated repo on GitLab. -::: diff --git a/docs/fair-usage/index.qmd b/docs/fair-usage/index.qmd deleted file mode 100644 index e69de29..0000000 diff --git a/docs/favicon.ico b/docs/favicon.ico deleted file mode 100644 index 29cda76..0000000 Binary files a/docs/favicon.ico and /dev/null differ diff --git a/docs/index.qmd b/docs/index.qmd deleted file mode 100644 index 4aa7744..0000000 --- a/docs/index.qmd +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: "CRAN R Package Binaries" ---- - -This project provides R package binaries for various Linux distributions and architectures. -While it provides binaries for [CRAN packages](https://cran.r-project.org/), it is not affiliated with CRAN in any way. - -Binaries are distributed to the community for free with a monthly download cap of **50 Terabyte**. - -::: {.callout-note} -If you represent a for-profit company and plan to make extensive use of these binaries, please get in touch. We offer the option to obtain a private mirror with unlimited access. -::: - -While we strive to build binaries for all CRAN packages, there are various reasons why building binaries for a package might fail. -(The most common reason is a missing (exotic) external system dependency which the project links against.) -The [R package binary dashboard](https://app.devxy.io/app/r-package-binaries-dashboard) provides global coverage information for each for each operating system and which packages are currently missing. - -## Motivation - -The project was established to offer extensive R CRAN package support for common Linux distributions. -Since CRAN does not build binaries for Linux, Posit began providing binaries in 2020. -However, their efforts were limited to the amd64 architecture and excluded Alpine Linux, which is among the most popular distributions for container-based workflows. - -This project aims to make R a first-class citizien for CI/CD and containers using a community-driven open-source approach. \ No newline at end of file diff --git a/docs/infrastructure/index.qmd b/docs/infrastructure/index.qmd deleted file mode 100644 index 6b4c2f4..0000000 --- a/docs/infrastructure/index.qmd +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: Infrastructure ---- - -The infrastructure is composed out of the following main components: - -- A k3s cluster (on Hetzner) -- S3 object storage (on Hetzner) -- A CDN (Bunny CDN) - -As the resource need for this project is quite variable, running on Kubernetes allows the efficient use of (shared) resources. - -## k3s - -Efficiently building on different architectures requires access to native servers for those architectures. -While most cloud providers make this possible, Kubernetes simplifies the process by enabling orchestration of specific builds to designated nodes. -This flexibility ensures that builds are executed on the appropriate hardware, optimizing performance and compatibility across diverse environments. - -### Memory Requirements - -Because the memory requirements for each package are vastly different and range between a few hundred MBs to a ~ 12 GB, these must also be reflected in the resource requirements of each pod. -Using `requests.memory` of 5Gi and `limits.memory` of 14Gi has been proven to work reliable for scheduling and individual resource needs. - -### Matrix builds - -While daily package updates can be handled with a single process per OS/version, building binaries for all CRAN packages requires a different orchestration strategy. -On average, each CRAN package has six versions (calculated by dividing the total number of binaries by the OS/versions built and the number of unique packages), making some level of parallelization necessary. - -Initially, parallelizing at the level of package versions was implemented. -However, this approach led to occasional conflicts when dependencies were installed into a shared package cache. -It also introduced unpredictable memory requirements within workflows: some packages caused memory usage to spike beyond 30 GB, depending on the number of parallel workers. -These spikes not only caused individual processes to crash but also demanded significantly higher overall resource limits. - -A more robust solution was found by processing individual packages sequentially within each matrix job. - -To build all versions of all packages, CRAN packages are divided into subsets, each comprising 1/10 or fewer of the total packages, and these subsets are processed in parallel. -The total time required depends on factors such as the distribution (e.g., distributions with newer C compilers like Alpine tend to be faster) and the number of parallel workers. -This approach typically takes anywhere from a few days to two weeks. - -## Storage: S3 - -Binaries need to be stored somewhere, and what better option than S3? S3 is significantly more cost-effective than traditional cloud disk storage and offers the added benefit of being accessible via a public API. -Beyond AWS, the original provider and inventor of S3, there are numerous alternatives with better price-to-storage ratios and lower transfer costs. - -The timing was perfect when Hetzner introduced their own S3-compatible object storage, coinciding with the start of this project’s build processing. -This solution brings multiple advantages: lower overall storage costs, free internal traffic between Hetzner servers and their S3 storage, and the proximity of storage to the build servers, which minimizes upload latency. - -::: {.callout-note} -While uploading packages to S3 is not complicated, there hasn't been any way to create the required PACKAGES index files for binaries stored there. -This is why forked versions of {cranlike} and {desc} have been created. -::: - -## Content Delivery Network (CDN) - -Storing binaries in S3 works well for distribution, but it’s not inherently very fast. Adding a CDN in front of S3 enables caching and allows assets to be distributed via servers located in various regions worldwide. This significantly reduces download latency, making downloads feel much faster. - -All packages are delivered through a CDN, which includes three dedicated static caches strategically placed in Germany, the USA, and Asia. - -With a CDN in place, downloads are optimized to feel “fast” from virtually anywhere, with only minor variations depending on the user’s location. - -The CDN determines when an asset is added to its permanent cache and how often it is revalidated against the S3 source. Since package binaries are one-time builds that typically remain unchanged unless a forced rebuild occurs, relying heavily on a permanent cache is highly efficient in this context. diff --git a/docs/logo-icon.png b/docs/logo-icon.png deleted file mode 100644 index c1e2090..0000000 Binary files a/docs/logo-icon.png and /dev/null differ diff --git a/docs/logo.png b/docs/logo.png deleted file mode 100644 index 7dafa55..0000000 Binary files a/docs/logo.png and /dev/null differ diff --git a/docs/private-mirror/index.qmd b/docs/private-mirror/index.qmd deleted file mode 100644 index e69de29..0000000 diff --git a/docs/r-dev-environments/index.qmd b/docs/r-dev-environments/index.qmd deleted file mode 100644 index 57858db..0000000 --- a/docs/r-dev-environments/index.qmd +++ /dev/null @@ -1,32 +0,0 @@ -An R development environment is a shared Linux VM or Kubernetes Cluster that runs an Integrated Development Environment (IDE), e.g. Posit Workbench or RStudio Server, and other related applications, such as Posit Connect or ShinyProxy. -In such an environment, multiple users can share the available resources and configurations can be set centrally for all users[^1]. - -Using package binaries in shared development environments is a crucial component for the overall success and efficiency of each individual user. - -## Fast Package Downloads - -Besides using R package binaries in the first place, a fast download speed further improves the quality of such an environment. -The binaries of this project are 5-10 times faster in download compared to other available binaries*. - -::: {.callout-note} -The exact timing depends on the location from where the download is initiated. -The closer it is to static cache node of the distributing CDN, the faster. -::: - -## Reduced Costs Through arm64 Servers - -Besides accelerating downloads, this project enabled using `arm64` machines for such environments. -These server types are substantially cheaper than their `amd64` counterparts, allowing for significant cost savings without any negative drawbacks. - -The following example shows the on-demand prices of a 2 CPU 16 GB VM on Azure (2025/01): - -- amd64: 0.1640/h (Standard_E2ds_v6, eastus) -- arm64: 0.1150/h (Standard_E2ps_v6, eastus) - --> 29.88% savings - -::: {.callout-note} -Servers with the same specifications but from different generations can have significant price variations. For this comparison, an effort was made to use the latest configurations of both architectures to ensure a reasonably fair evaluation. -::: - -[^1]: For more information about managed R environments, see ["Posit Software Installation for Enterprises" offer from devXY](https://www.devxy.io/posit-enterprise/). diff --git a/docs/styles.css b/docs/styles.css deleted file mode 100644 index ce447fc..0000000 --- a/docs/styles.css +++ /dev/null @@ -1,31 +0,0 @@ -/* css styles */ - -div.sourceCode { - border-radius: 6px; -} - -p code:not(.sourceCode), -li code:not(.sourceCode), -kbd, -pre:not(.sourceCode), -samp { - background-color: #f6f6f6 !important; - padding: 0.25em; - border-radius: 6px !important; -} - -/* sidebar */ -.sidebar-navigation li a { - line-height: 1.2rem; - font-size: 0.8rem; -} - -.navbar { - border-bottom: 1px solid #e5e5e6; - max-height: 50px; - -} - -.navbar-logo { - max-height: 30px; -} \ No newline at end of file