chore: move docs/ and dockerfiles to own repo
This commit is contained in:
parent
daa105ff6b
commit
e2d6240d36
GPG key ID:
3C6318841EF78925
24 changed files with 0 additions and 721 deletions
35
Justfile
35
Justfile
|
|
@ -1,36 +1 @@
|
|||
# just image alpine 3.20
|
||||
# just image alpine 3.21 4.4.3
|
||||
# just image alpine 3.22 4.5.0
|
||||
# just image ubuntu jammy 4.4.3
|
||||
# just image ubuntu noble 4.4.3
|
||||
# just image redhat 8 4.4.3
|
||||
# just image redhat 9 4.4.3
|
||||
# just image redhat 10
|
||||
image OS OS_VERSION R_VERSION :
|
||||
if echo "{{OS}}" | grep -qi "redhat"; then \
|
||||
docker buildx build --progress plain --build-arg R_VERSION={{R_VERSION}} --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg OS_VERSION={{OS_VERSION}} --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" --platform linux/amd64,linux/arm64 -f docker/Containerfile-{{OS}} -t devxygmbh/rpkgs-build-env-{{OS}}:{{OS_VERSION}}-{{R_VERSION}} --push .; \
|
||||
else \
|
||||
docker buildx build --progress plain --build-arg R_VERSION={{R_VERSION}} --build-arg OS_VERSION={{OS_VERSION}} --build-arg GITHUB_PAT="$GITHUB_PAT" -f docker/Containerfile-{{OS}} --platform linux/amd64,linux/arm64 -t devxygmbh/rpkgs-build-env-{{OS}}:{{OS_VERSION}}-{{R_VERSION}} --push .; \
|
||||
fi
|
||||
|
||||
### JAGS takes forever to build in CI through buildx. Building the binary once and injecting it into the final image
|
||||
# just image-jags alpine 3.20
|
||||
# just image-jags alpine 3.21
|
||||
# just image-jags alpine 3.22
|
||||
# just image-jags ubuntu jammy
|
||||
# just image-jags ubuntu noble
|
||||
# just image-jags redhat 8
|
||||
# just image-jags redhat 9
|
||||
# just image-jags redhat 10
|
||||
image-jags OS OS_VERSION:
|
||||
if echo "{{OS}}" | grep -qi "redhat"; then \
|
||||
docker buildx build --progress plain --platform linux/amd64,linux/arm64 --build-arg OS_VERSION={{OS_VERSION}} --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" -f docker/Containerfile-{{OS}}-jags --push -t devxygmbh/{{OS}}-jags:{{OS_VERSION}} .; \
|
||||
else \
|
||||
docker buildx build --progress plain --platform linux/amd64,linux/arm64 --build-arg OS_VERSION={{OS_VERSION}} -f docker/Containerfile-{{OS}}-jags --push -t devxygmbh/{{OS}}-jags:{{OS_VERSION}} .;\
|
||||
fi
|
||||
|
||||
preview-docs:
|
||||
cd docs && quarto preview
|
||||
|
||||
publish-docs:
|
||||
cd docs && quarto render && rclone sync ./_site hetzner-s3://devxy-r-package-binaries-docs
|
||||
|
|
|
|||
|
|
@ -1,64 +0,0 @@
|
|||
ARG OS_VERSION
|
||||
FROM devxygmbh/alpine-jags:${OS_VERSION} AS jags
|
||||
|
||||
ARG OS_VERSION
|
||||
FROM alpine:${OS_VERSION} AS final
|
||||
|
||||
ARG R_VERSION=4.4.3
|
||||
ENV PATH=/opt/R/${R_VERSION}/bin:$PATH
|
||||
ENV TZ="Europe/Berlin"
|
||||
ENV R_LIBS_USER=/mnt/cache/R-pkgs
|
||||
ENV R_PKG_CACHE_DIR=/mnt/cache/pkgcache
|
||||
|
||||
ENV NCPUS=2
|
||||
|
||||
ARG GITHUB_PAT
|
||||
|
||||
COPY --link --from=jags /JAGS-4.3.2 /JAGS-4.3.2
|
||||
|
||||
# Background: JAGS needs lapack & lapack-dev but these conflict with openblas-dev which is required by R-dev
|
||||
# Solution: first compile JAGS, then remove lapack & lapack-dev again
|
||||
# # xvfb: for graphical pkgs requiring tcl, like gWidgets2tcltk
|
||||
RUN apk add -q --no-cache curl tar make blas blas-dev gcc gfortran g++ htop git cmake tk-dev xvfb xvfb-run xorg-server xauth xinit xz libdeflate libdeflate-dev
|
||||
|
||||
### CUSTOM DEPENDENCIES
|
||||
|
||||
# linux-headers: ps
|
||||
# libxml2-dev: xml2
|
||||
# libgit2-dev: gert
|
||||
# libpq-dev: RPostgres
|
||||
# libfftw3-dev: ravetools -> actually auto-detect but somehow doesn't work in container?
|
||||
RUN apk add -q --no-cache R-dev curl ca-certificates bash g++ tzdata openjdk17 tar sed patch openblas-dev pkgconfig linux-headers ccache libxml2-dev libgit2-dev libpq-dev pkgconfig fftw-dev automake autoconf libtool elfutils-dev
|
||||
# - Chromium should be auto-installed once pkgdepends merges the latest system-requirements rule changes
|
||||
RUN apk add -q --no-cache chromium
|
||||
|
||||
# https://github.com/RcppCore/Rcpp/issues/448#issuecomment-220148774
|
||||
# LDFLAGS+=-fPIC -> arrow (https://github.com/r-hub/r-minimal/blob/main/examples/arrow/Dockerfile)
|
||||
# dbarts: CFLAGS += -flax-vector-conversions (https://github.com/vdorie/dbarts/issues/66)
|
||||
# CXXFLAGS: -> Rcpp
|
||||
# -DDMLC_USE_FOPEN64=0 -> xgboost (https://github.com/dmlc/xgboost/issues/8595)
|
||||
# can't set ccache for gfortran because of https://gitlab.com/roigrp/solver/highs/-/issues/4
|
||||
RUN mkdir -p /root/.R && bash -c 'echo -e "CXX_STD = CXX14\nCXXFLAGS=-D__MUSL__ -DDMLC_USE_FOPEN64=0\nCPPFLAGS=-D__MUSL__ -DDMLC_USE_FOPEN64=0\nCFLAGS += -flax-vector-conversions\nLDFLAGS+=-fPIC\nCC=ccache gcc\nCPP=gcc -E\nCXXCPP=gcc -e\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=gfortran\nFC=gfortran" > /root/.R/Makevars'
|
||||
|
||||
# R repos
|
||||
RUN bash -c 'echo -e "options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(CRAN = \"https://cloud.r-project.org\", INLA = \"https://inla.r-inla-download.org/R/stable\")))" > /root/.Rprofile'
|
||||
RUN bash -c 'echo -e "PKG_SYSREQS=TRUE\nPKG_SYSREQS_VERBOSE=TRUE" > ~/.Renviron'
|
||||
|
||||
### INSTALL R
|
||||
|
||||
ARG OS_VERSION
|
||||
RUN export OS_VERSION_TRIM=$(echo $OS_VERSION | tr -d '.') && \
|
||||
curl -v -O "https://devxy-r-builds.fsn1.your-objectstorage.com/alpine${OS_VERSION_TRIM}/r-${R_VERSION}_1_$(arch).apk"
|
||||
RUN apk add --allow-untrusted "r-${R_VERSION}_1_$(arch).apk" && rm "r-${R_VERSION}_1_$(arch).apk"
|
||||
|
||||
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
||||
|
||||
# these dirs will later be used by the CI with existing mounts
|
||||
RUN mkdir -p /mnt/cache/packages /mnt/cache/pkgcache /mnt/cache/R-pkgs
|
||||
|
||||
# encoding -> https://github.com/r-hub/r-minimal/blob/fa9c117313e05860a0d20b113e5f794c51be8b8c/Dockerfile#L84
|
||||
RUN sed -i 's/,//g' /opt/R/${R_VERSION}/lib/R/library/utils/iconvlist
|
||||
|
||||
RUN R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git", dependencies = TRUE)'
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
ARG VERSION
|
||||
FROM alpine:${VERSION} AS jags
|
||||
|
||||
ARG JAGS_VERSION=4.3.2
|
||||
|
||||
RUN apk add -q --no-cache curl tar make lapack lapack-dev blas blas-dev gcc gfortran g++ git cmake tk-dev xz
|
||||
|
||||
# These usually are not packaged in OS repositories and need to be installed from source
|
||||
# - JAGS (needed for runjags which itself is again a dependency of many packages)
|
||||
RUN curl -s -L -O https://sourceforge.net/projects/mcmc-jags/files/JAGS/4.x/Source/JAGS-${JAGS_VERSION}.tar.gz && \
|
||||
tar -xzf JAGS-${JAGS_VERSION}.tar.gz && cd JAGS-${JAGS_VERSION} && ./configure --enable-silent-rules && make 2>&1
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
ARG OS_VERSION
|
||||
FROM devxygmbh/redhat-jags:${OS_VERSION} AS jags
|
||||
|
||||
ARG OS_VERSION
|
||||
FROM redhat/ubi${OS_VERSION} AS final
|
||||
|
||||
ARG R_VERSION
|
||||
ARG RED_HAT_DEV_PW
|
||||
ENV PATH=/opt/R/${R_VERSION}/bin:$PATH
|
||||
ENV TZ="Europe/Berlin"
|
||||
ENV R_LIBS_USER=/mnt/cache/R-pkgs
|
||||
ENV R_PKG_CACHE_DIR=/mnt/cache/pkgcache
|
||||
|
||||
ENV NCPUS=2
|
||||
|
||||
ARG GITHUB_PAT
|
||||
|
||||
COPY --link --from=jags /JAGS-4.3.2 /JAGS-4.3.2
|
||||
|
||||
ARG OS_VERSION
|
||||
RUN subscription-manager register --username mail@devxy.io --password "$RED_HAT_DEV_PW" && subscription-manager repos --enable codeready-builder-for-rhel-${OS_VERSION}-$(arch)-rpms
|
||||
RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-${OS_VERSION}.noarch.rpm && dnf install -y epel-release
|
||||
# xvfb: for graphical pkgs requiring tcl, like gWidgets2tcltk
|
||||
# ImageMagick-c++-devel ImageMagick-devel: boths are not auto-installed via pak sysreqs
|
||||
# mecab-devel: RcppMeCab -> no autodetection yet
|
||||
# libfftw3-dev: ravetools -> actually auto-detect but somehow doesn't work in container?
|
||||
ARG OS_VERSION
|
||||
RUN if [ "$OS_VERSION" = "8" ]; then \
|
||||
dnf install -y -q make lapack-devel git gcc gcc-c++ fontconfig-devel fribidi-devel libjpeg-turbo-devel libtiff-devel postgresql-devel htop libgit2_1.7-devel libxml2-devel hiredis ccache cmake xorg-x11-server-Xvfb ImageMagick-c++-devel ImageMagick-devel xz pkg-config mecab-devel fftw-devel tk-devel automake readline-devel; \
|
||||
elif [ "$OS_VERSION" = "9" ] || [ "$OS_VERSION" = "10" ]; then \
|
||||
dnf install -y -q make lapack-devel git gcc gcc-c++ fontconfig-devel fribidi-devel libjpeg-turbo-devel libtiff-devel postgresql-devel htop libgit2-devel libxml2-devel hiredis ccache cmake xorg-x11-server-Xvfb ImageMagick-c++-devel ImageMagick-devel xz pkg-config mecab-devel fftw-devel tk-devel automake readline-devel; \
|
||||
fi;
|
||||
|
||||
### CUSTOM DEPENDENCIES
|
||||
# - Chromium should be auto-installed once pkgdepends merges the latest system-requirements rule changes
|
||||
RUN dnf install -y chromium
|
||||
|
||||
### INSTALL R
|
||||
RUN curl -O "https://devxy-r-builds.fsn1.your-objectstorage.com/el${OS_VERSION}/R-${R_VERSION}-1-1.$(arch).rpm"
|
||||
RUN dnf install -q -y "R-${R_VERSION}-1-1.$(arch).rpm" && rm "R-${R_VERSION}-1-1.$(arch).rpm"
|
||||
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
||||
|
||||
RUN if [ "$OS_VERSION" = "8" ]; then \
|
||||
export CXX_STD=14; \
|
||||
elif [ "$OS_VERSION" = "9" ]; then \
|
||||
export CXX_STD=17; \
|
||||
elif [ "$OS_VERSION" = "10" ]; then \
|
||||
export CXX_STD=20; \
|
||||
fi; \
|
||||
if [ "$(arch)" = "x86_64" ]; then \
|
||||
export ARCH=amd64; \
|
||||
elif [ "$(arch)" = "aarch64" ]; then \
|
||||
export ARCH=arm64; \
|
||||
fi; \
|
||||
mkdir -p /root/.R && bash -c 'echo -e "CXX_STD = ${CXX_STD}\nCFLAGS += -flax-vector-conversions\nCC=ccache gcc\nCPP=gcc -E\nCXXCPP=gcc -e\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=gfortran\nFC=gfortran" >> /root/.R/Makevars' && \
|
||||
bash -c 'echo -e "options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(CRAN = \"https://cran.devxy.io/$ARCH/rhel${OS_VERSION}/latest\", INLA = \"https://inla.r-inla-download.org/R/stable\")))" > /root/.Rprofile' && \
|
||||
bash -c 'echo -e "PKG_SYSREQS = TRUE\nPKG_SYSREQS_VERBOSE = TRUE\nPKG_SYSREQS_PLATFORM=\"redhat-${OS_VERSION}\"" > /root/.Renviron'
|
||||
|
||||
### Makevars
|
||||
# dbarts: CFLAGS += -flax-vector-conversions (https://github.com/vdorie/dbarts/issues/66)
|
||||
# can't set ccache for gfortran because of https://gitlab.com/roigrp/solver/highs/-/issues/4
|
||||
|
||||
# these dirs will later be used by the CI with existing mounts
|
||||
RUN mkdir -p /mnt/cache/packages /mnt/cache/pkgcache /mnt/cache/R-pkgs
|
||||
|
||||
RUN R -q -e 'getOption("repos"); pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git", dependencies = TRUE)'
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
ARG VERSION
|
||||
FROM redhat/ubi${VERSION} AS jags
|
||||
ARG RED_HAT_DEV_PW
|
||||
|
||||
ARG VERSION
|
||||
RUN subscription-manager register --username mail@devxy.io --password "$RED_HAT_DEV_PW" && subscription-manager repos --enable codeready-builder-for-rhel-${VERSION}-$(arch)-rpms
|
||||
RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-${VERSION}.noarch.rpm && dnf install -y epel-release
|
||||
RUN dnf install -y -q gcc-c++ lapack-devel make git ca-certificates
|
||||
|
||||
# These usually are not packaged in OS repositories and need to be installed from source
|
||||
# - JAGS (needed for runjags which itself is again a dependency of many packages)
|
||||
RUN curl -s -L -O https://sourceforge.net/projects/mcmc-jags/files/JAGS/4.x/Source/JAGS-4.3.2.tar.gz && \
|
||||
tar -xzf JAGS-4.3.2.tar.gz && cd JAGS-4.3.2 && ./configure --enable-silent-rules && make 2>&1
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
ARG OS_VERSION
|
||||
FROM devxygmbh/ubuntu-jags:${OS_VERSION} AS jags
|
||||
|
||||
ARG OS_VERSION
|
||||
FROM ubuntu:${OS_VERSION} AS final
|
||||
|
||||
ARG R_VERSION
|
||||
ENV PATH=/opt/R/${R_VERSION}/bin:$PATH
|
||||
ENV TZ="Europe/Berlin"
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
ENV NCPUS=2
|
||||
ENV R_LIBS_USER=/mnt/cache/R-pkgs
|
||||
ENV R_PKG_CACHE_DIR=/mnt/cache/pkgcache
|
||||
|
||||
ARG GITHUB_PAT
|
||||
|
||||
COPY --link --from=jags /JAGS-4.3.2 /JAGS-4.3.2
|
||||
|
||||
# xvfb: for graphical pkgs requiring tcl, like gWidgets2tcltk
|
||||
# tcl tk tk-dev tk-table: must be available before running xvfb-run (i.e. starting the graphical session)
|
||||
# libmecab-dev: RcppMeCab -> no autodetection yet
|
||||
# libfftw3-dev: ravetools -> actually auto-detect but somehow doesn't work in container?
|
||||
RUN apt update && apt install -y --no-install-recommends gdebi-core g++ gfortran libbz2-dev libblas-dev libicu-dev liblapack-dev liblzma-dev libpaper-utils libpcre2-dev libtcl8.6 libtk8.6 libxt6 unzip zip zlib1g-dev libpcre3-dev libopenblas-dev curl ca-certificates make ccache htop libgit2-dev git libpq-dev libxml2-dev libgit2-dev cmake xvfb libcurl4-openssl-dev xz-utils tcl tk tk-dev tk-table pkg-config libmecab-dev libfftw3-dev automake
|
||||
|
||||
### CUSTOM DEPENDENCIES
|
||||
# - Chromium should be auto-installed once pkgdepends merges the latest system-requirements rule changes
|
||||
RUN apt-get install -y software-properties-common && \
|
||||
add-apt-repository -y ppa:xtradeb/apps && \
|
||||
apt update && \
|
||||
apt install -y chromium
|
||||
|
||||
ARG OS_VERSION
|
||||
ARG R_VERSION
|
||||
RUN if [ "$OS_VERSION" = "jammy" ]; then \
|
||||
OS_VERSION_TRIM=2204; \
|
||||
CXX_STD=14; \
|
||||
elif [ "$OS_VERSION" = "noble" ]; then \
|
||||
OS_VERSION_TRIM=2404; \
|
||||
CXX_STD=17; \
|
||||
fi; \
|
||||
if [ "$(arch)" = "x86_64" ]; then \
|
||||
export ARCH=amd64; \
|
||||
elif [ "$(arch)" = "aarch64" ]; then \
|
||||
export ARCH=arm64; \
|
||||
fi; \
|
||||
curl -v -O "https://devxy-r-builds.fsn1.your-objectstorage.com/${OS_VERSION_TRIM}/r-${R_VERSION}_1_$(dpkg --print-architecture).deb" && \
|
||||
gdebi -n -qq "r-${R_VERSION}_1_$(dpkg --print-architecture).deb" && \
|
||||
rm "r-${R_VERSION}_1_$(dpkg --print-architecture).deb" && \
|
||||
R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' && \
|
||||
mkdir -p /root/.R && \
|
||||
bash -c 'echo -e "CXX_STD = $CXX_STD\nCFLAGS += -flax-vector-conversions\nCC=ccache gcc\nCPP=gcc -E\nCXXCPP=gcc -e\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=gfortran\nFC=gfortran" >> /root/.R/Makevars' && \
|
||||
bash -c 'echo -e "options(pkg.sysreqs_verbose = TRUE, crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(CRAN = \"https://cran.devxy.io/$ARCH/${OS_VERSION}/latest\", INLA = \"https://inla.r-inla-download.org/R/stable\")))" > ~/.Rprofile'
|
||||
|
||||
### Makevars
|
||||
# dbarts: CFLAGS += -flax-vector-conversions (https://github.com/vdorie/dbarts/issues/66)
|
||||
# can't set ccache for gfortran because of https://gitlab.com/roigrp/solver/highs/-/issues/4
|
||||
|
||||
# these dirs will later be used by the CI with existing mounts
|
||||
RUN mkdir -p /mnt/cache/packages /mnt/cache/pkgcache /mnt/cache/R-pkgs
|
||||
|
||||
RUN apt install -y locales && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
|
||||
&& locale-gen en_US.utf8 \
|
||||
&& /usr/sbin/update-locale LANG=en_US.UTF-8
|
||||
|
||||
RUN R -q -e 'getOption("repos"); pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git", dependencies = TRUE)'
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
ARG VERSION
|
||||
FROM ubuntu:${VERSION} AS jags
|
||||
|
||||
RUN apt update && apt install -y --no-install-recommends curl make ca-certificates gcc g++ gfortran liblapack-dev
|
||||
# These usually are not packaged in OS repositories and need to be installed from source
|
||||
# - JAGS (needed for runjags which itself is again a dependency of many packages)
|
||||
RUN curl -q -L -O https://sourceforge.net/projects/mcmc-jags/files/JAGS/4.x/Source/JAGS-4.3.2.tar.gz && tar -xzf JAGS-4.3.2.tar.gz && cd JAGS-4.3.2 && ./configure --enable-silent-rules && make 2>&1
|
||||
1
docs/.gitignore
vendored
1
docs/.gitignore
vendored
|
|
@ -1 +0,0 @@
|
|||
/.quarto/
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
color:
|
||||
palette:
|
||||
dark: "#1d262f"
|
||||
orange: "#FF9B00"
|
||||
links: "#15428c"
|
||||
# background: blue
|
||||
# foreground: dark-grey
|
||||
primary: '#1d262f' # hyperlinks, active states, primary action buttons, etc.
|
||||
tertiary: "#FF9B00" # hover states, accents, and wells.
|
||||
|
||||
logo:
|
||||
medium:
|
||||
light: logo.png
|
||||
dark: logo.png
|
||||
|
||||
defaults:
|
||||
bootstrap:
|
||||
defaults:
|
||||
navbar-fg: "#1d262f"
|
||||
navbar-bg: "#ffffff"
|
||||
|
||||
typography:
|
||||
fonts:
|
||||
- family: Jura
|
||||
source: bunny
|
||||
- family: Inter
|
||||
source: bunny
|
||||
- family: Montserrat
|
||||
source: bunny
|
||||
base: Inter
|
||||
headings: Montserrat
|
||||
link:
|
||||
decoration: underline
|
||||
color: links
|
||||
monospace:
|
||||
color: primary
|
||||
background-color: "#f0f1f3"
|
||||
size: 0.8rem
|
||||
# decoration:
|
||||
|
||||
meta:
|
||||
link:
|
||||
home: https://www.devxy.io
|
||||
gitlab: https://gitlab.com/devxy
|
||||
bluesky: https://bsky.app/profile/devxy.bsky.social
|
||||
linkedin: https://www.linkedin.com/company/105100347
|
||||
description: Where Data Science and DevOps meet 🤝️
|
||||
founded: 2024
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
project:
|
||||
type: website
|
||||
|
||||
website:
|
||||
title: "CRAN R Package Binaries"
|
||||
reader-mode: false
|
||||
favicon: favicon.ico
|
||||
back-to-top-navigation: true
|
||||
navbar:
|
||||
right:
|
||||
- icon: gitlab
|
||||
href: https://gitlab.com/devxy
|
||||
- icon: browser-safari
|
||||
href: https://www.devxy.io
|
||||
- icon: bluesky
|
||||
href: https://www.devxy.io
|
||||
sidebar:
|
||||
logo: ""
|
||||
style: "docked"
|
||||
collapse-level: 4
|
||||
search: true
|
||||
border: true
|
||||
pinned: true
|
||||
contents:
|
||||
- text: 'Introduction'
|
||||
file: index.qmd
|
||||
- text: "---"
|
||||
- text: 'Configuration'
|
||||
file: configuration/index.qmd
|
||||
- section: "**Use Cases**"
|
||||
contents:
|
||||
- text: "CI/CD"
|
||||
file: cicd/index.qmd
|
||||
- text: "R Development Environments"
|
||||
file: r-dev-environments/index.qmd
|
||||
- text: "Containers"
|
||||
file: containers/index.qmd
|
||||
- section: "**Technical Background**"
|
||||
contents:
|
||||
- text: "Infrastructure"
|
||||
file: infrastructure/index.qmd
|
||||
# - text: "Continuous Updates"
|
||||
# file: 032-continuous-updates.qmd
|
||||
# - section: "**Miscellaneous**"
|
||||
# contents:
|
||||
# - text: "Private Mirror"
|
||||
# file: 040-private-mirror.qmd
|
||||
# - text: "Fair Usage"
|
||||
# file: 041-fair-usage.qmd
|
||||
- text: "---"
|
||||
- text: 'Contributing & Help'
|
||||
file: "contributing/index.qmd"
|
||||
|
||||
search:
|
||||
location: sidebar
|
||||
type: "textbox"
|
||||
limit: 5
|
||||
copy-button: true
|
||||
collapse-after: 2
|
||||
format:
|
||||
html:
|
||||
theme: cosmo
|
||||
css: styles.css
|
||||
fontsize: 1rem
|
||||
toc: true
|
||||
grid:
|
||||
sidebar-width: 300px
|
||||
include-in-header:
|
||||
text: |
|
||||
<script defer src="https://umami.devxy.io/web-info" data-website-id="5e495624-ac45-4f17-ab6a-6b30032f46bb"></script>
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
---
|
||||
title: CI/CD
|
||||
---
|
||||
|
||||
Continuous Integration / continous delivery (CI/CD) is a basic tool in todays Data Science toolstack to automate processes and execute repetite tasks on schedule.
|
||||
|
||||
Running tasks in CI/CD should be fast and reliable, no matter the language, architecture or computing environment at hand.
|
||||
As CI/CD requires installing many packages over and over again, having binaries and optimized distributions helps to reduce runtime.
|
||||
|
||||
This page presents snippets for well and lesser known CI/CD engines to help getting started with the R package binaries of this project.
|
||||
|
||||
::: {.callout-note}
|
||||
The presented snippets rely on external projects and libraries which get updated regurarly.
|
||||
There is no guarantee to work out-of-the box at all times.
|
||||
If you found an error, please open a pull request in the linked repo.
|
||||
:::
|
||||
|
||||
## GitHub Actions
|
||||
|
||||
### VM
|
||||
|
||||
GitHub Actions can be run directly on a VM or in a containered context.
|
||||
The available VM images can be found [here](https://github.com/actions/runner-images#available-images).
|
||||
The only Linux distribution available is Ubuntu.
|
||||
|
||||
For VM workflows, [r-lib/actions](https://github.com/r-lib/actions/blob/v2/examples) provides many examples for different use cases.
|
||||
|
||||
To use package binaries from this project, use the following config for `r-lib/actions/setup-r@v2`:
|
||||
|
||||
```yaml
|
||||
- uses: r-lib/actions/setup-r@v2
|
||||
with:
|
||||
r-version: ${{ matrix.config.r }}
|
||||
http-user-agent: ${{ matrix.config.http-user-agent }}
|
||||
Ncpus: 2
|
||||
cran: 'https://cran.devxy.io/amd64/noble/latest'
|
||||
```
|
||||
|
||||
(`Ncpus: 2` has been set to allow for parallel installations.)
|
||||
|
||||
### Container
|
||||
|
||||
An alternative is to run Actions in a containerized context.
|
||||
This spins up a container of the selected image in the VM and allows running on other distributions than Ubuntu, e.g. on Alpine.
|
||||
To avoid having to install R first and configure a custom repository every time, [specialized images](https://hub.docker.com/r/devxygmbh/r-alpine) are provided which already have everything in place:
|
||||
|
||||
```yaml
|
||||
jobs:
|
||||
container:
|
||||
runs-on: ubuntu-latest
|
||||
container: devxygmbh:r-alpine
|
||||
steps:
|
||||
- run: |
|
||||
R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
||||
R -q -e 'getOption("repos")'
|
||||
R -q -e 'options(Ncpus = 2); pak::local_install_dev_deps()'
|
||||
```
|
||||
|
||||
::: {.callout-note}
|
||||
When running in a containerized context, the predefined actions from `r-lib/actions` cannot be used.
|
||||
Shell/R commands must be used directly.
|
||||
:::
|
||||
|
||||
::: {.callout-note}
|
||||
Running containerized in Alpine should be substantially faster than running in a VM context.
|
||||
:::
|
||||
|
||||
::: {.callout-tip}
|
||||
(91 packages)
|
||||
|
||||
- Containerized: **1m 21s**
|
||||
- VM: 2m 11s
|
||||
|
||||
Both runs are based on the assumption that caching is not used. With caching enabled, the performance of both approaches should be roughly comparable.
|
||||
|
||||
Source: [pat-s/workflow-compare](https://github.com/pat-s/workflow-compare)
|
||||
:::
|
||||
|
||||
|
||||
## GitLab Runner
|
||||
|
||||
WIP
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
---
|
||||
title: Configuration
|
||||
---
|
||||
|
||||
In R on Linux, packages are downloaded from CRAN as source packages.
|
||||
The default CRAN mirror is `https://cloud.r-project.org`.
|
||||
The default repository URL needs to be **actively changed** to one providing binaries for the respective Linux distribution.
|
||||
This can be done either for all users of the system, on a project level or for specific users only.
|
||||
All options will be shown in the following.
|
||||
|
||||
## URL Scheme
|
||||
|
||||
The URL scheme for this project is the same for all operating systems and only differs in the OS and architecture identifiers.
|
||||
|
||||
::: {.callout-note icon=false}
|
||||
|
||||
## URL scheme
|
||||
|
||||
`https://cran.devxy.io/<architecture>/<os>/latest`
|
||||
:::
|
||||
|
||||
For example, the URL for Alpine Linux 3.21 for `arm64` would be `https://cran.devxy.io/arm64/alpine321/latest`.
|
||||
|
||||
The architecture identifiers are:
|
||||
|
||||
- `arm64`
|
||||
- `amd64`
|
||||
|
||||
The OS identifiers are:
|
||||
|
||||
- `jammy` (Ubuntu 22.04)
|
||||
- `noble` (Ubuntu 24.04)
|
||||
- `rhel8` (RedHat Enterprise Linux 8)
|
||||
- `rhel9` (RedHat Enterprise Linux 9)
|
||||
- `alpine320` (Alpine Linux 3.20)
|
||||
- `alpine321` (Alpine Linux 3.21)
|
||||
|
||||
## Repository Configuration
|
||||
|
||||
### System-wide
|
||||
|
||||
To configure the default R repository for all users of a system, create/edit `$R_HOME/etc/Rprofile.site`.
|
||||
The location of this file depends on how R has been installed.
|
||||
To find its location, execute `R -q -e "paste0(Sys.getenv('R_HOME'),'/etc/Rprofile.site')"`.
|
||||
|
||||
**At the bottom** of the file, add the following:
|
||||
|
||||
```r
|
||||
options(repos = structure(c(CRAN = "<URL>")))
|
||||
```
|
||||
|
||||
The name "CRAN" is optional.
|
||||
It is the canonical default and in some cases also used for special actions (e.g. in Posit Workbench and [{pak}](https://github.com/r-lib/pak)).
|
||||
Unless you have a good reason, it is advisable to stick with that name.
|
||||
|
||||
### User-wide
|
||||
|
||||
To change the default repository option for a specific *user*, add the line from above to `~/.Rprofile`.
|
||||
|
||||
### Project-wide
|
||||
|
||||
To change the default repository option for a specific *project*, add the line from above to `.Rprofile` in the project root directory in which R is started.
|
||||
|
||||
## Verification
|
||||
|
||||
When starting R, execute `getOption("repos")` to verify the repository is correctly configured.
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
---
|
||||
title: Containers
|
||||
---
|
||||
|
||||
The [rocker](https://hub.docker.com/u/rocker) project provides container images for R since several years.
|
||||
Since recently, the images are multi-arch, i.e. `arm64` is supported alongside `amd64`.
|
||||
|
||||
Yet, other distributions than Ubuntu are missing and don't use the devXY binaries out of the box.
|
||||
To simplify running containers that make use of the devXY R binaries implicitly, the following images are provided:
|
||||
|
||||
- [devxygmbh/r-alpine](https://hub.docker.com/r/devxygmbh/r-alpine)
|
||||
- [devxygmbh/r-ubuntu](https://hub.docker.com/r/devxygmbh/r-ubuntu)
|
||||
|
||||
These have the devXY repositories configured by default and can be used efficiently without further configuration changes.
|
||||
|
||||
In contrast to the minimal `rocker/r-ver` images, the `r-ubuntu` images contain a few additional system libraries which are essential for troublefree R usage.
|
||||
|
||||
The following screencast shows how quickly the `r-alpine` image can be used to install the full {tidyverse} package universe in just under 30s:
|
||||
|
||||
<div style="padding:59.12% 0 0 0;position:relative;"><iframe src="https://player.vimeo.com/video/1035389676?badge=0&autopause=0&player_id=0&app_id=58479" frameborder="0" allow="autoplay; fullscreen; picture-in-picture; clipboard-write" style="position:absolute;top:0;left:0;width:100%;height:100%;" title="Alpine package binaries: Install tidyverse"></iframe></div><script src="https://player.vimeo.com/api/player.js"></script>
|
||||
|
||||
::: {.callout-note}
|
||||
The performance of this screencast (and your local experience) depends on a few factors: local download speed, cached binaries in the CDN, proximity to a local edge node of the CDN and your CPU performance and finally the `Ncpus` option in R, defining how many installations are performed in parallel (by default 1).
|
||||
:::
|
||||
|
|
@ -1 +0,0 @@
|
|||
WIP
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
title: Contributing & Help
|
||||
---
|
||||
|
||||
Contributions are welcome.
|
||||
The project is open-source and GPL licensed.
|
||||
|
||||
The most helpful tasks are to investigate in detail why specific packages failed to build a binary.
|
||||
It usually comes down to a missing system dependency (which is either not declared in the package's DESCRIPTION file or exotic and must be installed from source) or a C code compile issue.
|
||||
|
||||
Please open an issue in the [`r-package-binaries/support`](https://gitlab.com/devxy/r-package-binaries/support) repo.
|
||||
|
||||
::: {.callout-note}
|
||||
The actual repo executing the builds ([devxy/build-binaries](https://git.devxy.io/devxy/build-binaries)) is hosted in a private git instance to be able to use specific private CI runners and is not easily usable for creating public issues, hence the dedicated repo on GitLab.
|
||||
:::
|
||||
BIN
docs/favicon.ico
BIN
docs/favicon.ico
Binary file not shown.
|
Before Width: | Height: | Size: 11 KiB |
|
|
@ -1,24 +0,0 @@
|
|||
---
|
||||
title: "CRAN R Package Binaries"
|
||||
---
|
||||
|
||||
This project provides R package binaries for various Linux distributions and architectures.
|
||||
While it provides binaries for [CRAN packages](https://cran.r-project.org/), it is not affiliated with CRAN in any way.
|
||||
|
||||
Binaries are distributed to the community for free with a monthly download cap of **50 Terabyte**.
|
||||
|
||||
::: {.callout-note}
|
||||
If you represent a for-profit company and plan to make extensive use of these binaries, please get in touch. We offer the option to obtain a private mirror with unlimited access.
|
||||
:::
|
||||
|
||||
While we strive to build binaries for all CRAN packages, there are various reasons why building binaries for a package might fail.
|
||||
(The most common reason is a missing (exotic) external system dependency which the project links against.)
|
||||
The [R package binary dashboard](https://app.devxy.io/app/r-package-binaries-dashboard) provides global coverage information for each for each operating system and which packages are currently missing.
|
||||
|
||||
## Motivation
|
||||
|
||||
The project was established to offer extensive R CRAN package support for common Linux distributions.
|
||||
Since CRAN does not build binaries for Linux, Posit began providing binaries in 2020.
|
||||
However, their efforts were limited to the amd64 architecture and excluded Alpine Linux, which is among the most popular distributions for container-based workflows.
|
||||
|
||||
This project aims to make R a first-class citizien for CI/CD and containers using a community-driven open-source approach.
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
---
|
||||
title: Infrastructure
|
||||
---
|
||||
|
||||
The infrastructure is composed out of the following main components:
|
||||
|
||||
- A k3s cluster (on Hetzner)
|
||||
- S3 object storage (on Hetzner)
|
||||
- A CDN (Bunny CDN)
|
||||
|
||||
As the resource need for this project is quite variable, running on Kubernetes allows the efficient use of (shared) resources.
|
||||
|
||||
## k3s
|
||||
|
||||
Efficiently building on different architectures requires access to native servers for those architectures.
|
||||
While most cloud providers make this possible, Kubernetes simplifies the process by enabling orchestration of specific builds to designated nodes.
|
||||
This flexibility ensures that builds are executed on the appropriate hardware, optimizing performance and compatibility across diverse environments.
|
||||
|
||||
### Memory Requirements
|
||||
|
||||
Because the memory requirements for each package are vastly different and range between a few hundred MBs to a ~ 12 GB, these must also be reflected in the resource requirements of each pod.
|
||||
Using `requests.memory` of 5Gi and `limits.memory` of 14Gi has been proven to work reliable for scheduling and individual resource needs.
|
||||
|
||||
### Matrix builds
|
||||
|
||||
While daily package updates can be handled with a single process per OS/version, building binaries for all CRAN packages requires a different orchestration strategy.
|
||||
On average, each CRAN package has six versions (calculated by dividing the total number of binaries by the OS/versions built and the number of unique packages), making some level of parallelization necessary.
|
||||
|
||||
Initially, parallelizing at the level of package versions was implemented.
|
||||
However, this approach led to occasional conflicts when dependencies were installed into a shared package cache.
|
||||
It also introduced unpredictable memory requirements within workflows: some packages caused memory usage to spike beyond 30 GB, depending on the number of parallel workers.
|
||||
These spikes not only caused individual processes to crash but also demanded significantly higher overall resource limits.
|
||||
|
||||
A more robust solution was found by processing individual packages sequentially within each matrix job.
|
||||
|
||||
To build all versions of all packages, CRAN packages are divided into subsets, each comprising 1/10 or fewer of the total packages, and these subsets are processed in parallel.
|
||||
The total time required depends on factors such as the distribution (e.g., distributions with newer C compilers like Alpine tend to be faster) and the number of parallel workers.
|
||||
This approach typically takes anywhere from a few days to two weeks.
|
||||
|
||||
## Storage: S3
|
||||
|
||||
Binaries need to be stored somewhere, and what better option than S3? S3 is significantly more cost-effective than traditional cloud disk storage and offers the added benefit of being accessible via a public API.
|
||||
Beyond AWS, the original provider and inventor of S3, there are numerous alternatives with better price-to-storage ratios and lower transfer costs.
|
||||
|
||||
The timing was perfect when Hetzner introduced their own S3-compatible object storage, coinciding with the start of this project’s build processing.
|
||||
This solution brings multiple advantages: lower overall storage costs, free internal traffic between Hetzner servers and their S3 storage, and the proximity of storage to the build servers, which minimizes upload latency.
|
||||
|
||||
::: {.callout-note}
|
||||
While uploading packages to S3 is not complicated, there hasn't been any way to create the required PACKAGES index files for binaries stored there.
|
||||
This is why forked versions of {cranlike} and {desc} have been created.
|
||||
:::
|
||||
|
||||
## Content Delivery Network (CDN)
|
||||
|
||||
Storing binaries in S3 works well for distribution, but it’s not inherently very fast. Adding a CDN in front of S3 enables caching and allows assets to be distributed via servers located in various regions worldwide. This significantly reduces download latency, making downloads feel much faster.
|
||||
|
||||
All packages are delivered through a CDN, which includes three dedicated static caches strategically placed in Germany, the USA, and Asia.
|
||||
|
||||
With a CDN in place, downloads are optimized to feel “fast” from virtually anywhere, with only minor variations depending on the user’s location.
|
||||
|
||||
The CDN determines when an asset is added to its permanent cache and how often it is revalidated against the S3 source. Since package binaries are one-time builds that typically remain unchanged unless a forced rebuild occurs, relying heavily on a permanent cache is highly efficient in this context.
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 49 KiB |
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue