devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 10 Pull requests Releases Activity

save state

Browse source
This commit is contained in:
Patrick Schratz 2024-05-29 19:52:07 +02:00
commit 23142a03f6
Signed by: pat-s
GPG key ID: 3C6318841EF78925
5 changed files with 279 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

12
00-run.sh Normal file
View file

@ -0,0 +1,12 @@
#!/bin/bash
docker run --name r-pkg-binaries-rhel9 -e RED_HAT_DEV_PW -d redhat/ubi9 tail -f /dev/null
docker exec -it r-pkg-binaries-rhel9 bash
export R_VERSION=4.3.3
export PACKAGES="sf"
bash -c ./01-prepare.sh
bash -c ./02-build-binaries.sh
bash -c ./03-upload.sh

38
01-prepare.sh Normal file
View file

@ -0,0 +1,38 @@
#!/bin/bash
### install R
PLATFORM_ID=$(cat /etc/os-release | grep 'PLATFORM_ID=' | cut -d'=' -f2 | tr -d '"')
if [ $PLATFORM_ID = "platform:el9" ]; then
subscription-manager register --username pat-s --password $RED_HAT_DEV_PW
subscription-manager repos --enable codeready-builder-for-rhel-9-$(arch)-rpms
dnf -y install git rsync
# dnf update -y
# FIXME: protect PW
echo "R Version: ${R_VERSION}"
echo "Downloading R-${R_VERSION}.rpm from S3..."
curl -o /tmp/R-${R_VERSION}.rpm https://cynkra-r-builds-arm64.s3.eu-central-1.amazonaws.com/el9/R-"${R_VERSION}"-1-1."$(arch)".rpm
dnf install -y /tmp/R-${R_VERSION}.rpm
fi
export PATH=/opt/R/${R_VERSION}/bin:$PATH
### install R packages needed for building
R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pak::pak(c("pkgbuild", "cranlike", "drat"))'
### install awscli
curl -s "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip"
unzip -qq awscliv2.zip
./aws/install
rm -rf awscliv2.zip
mkdir /root/.aws
echo -e "[default]\n
aws_access_key_id=AKIATHTKSC6FRVE5LNE7\n
aws_secret_access_key=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\n" >/root/.aws/credentials

154
02-build-binaries.sh Normal file
View file

@ -0,0 +1,154 @@
#! /bin/bash
R_VERSION=4.0.5
PACKAGES="mlr3filters stringi"
# install sysdeps for all packages
for w in $PACKAGES; do
### dynamic start
export PACKAGE=$w
export R_LIBS_USER=/root/user_lib
mkdir -p /root/user_lib
export R_VERSION=$(R --version | grep -Eo '[0-9.]+' | head -n 1)
export R_VERSION_MINOR=$(echo "$R_VERSION" | cut -d. -f1,2)
export ARCH=$(arch)
DIST_FAM=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2 | tr -d '"')
if [[ $DIST_FAM == "debian" ]]; then
export CODENAME=$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d'=' -f2 | tr -d '"')
elif [[ $DIST_FAM =~ ^(rhel) ]]; then
if [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el9" ]]; then
export CODENAME=rhel9
elif [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el8" ]]; then
export CODENAME=rhel8
fi
fi
git clone https://github.com/cran/"${PACKAGE}".git
cd "${PACKAGE}"
# also installs system dependencies
# FIXME: auto-detection does not work on arm-based systems
export PKG_SYSREQS_PLATFORM=redhat-9
export PKG_SYSREQS=TRUE
export PKG_SYSREQS_VERBOSE=TRUE
R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pak::local_install_dev_deps()'
cd ..
done
# build binaries from tags
for w in $PACKAGES; do
# this runs in parallel
# run() {
export PACKAGE=$w
export R_LIBS_USER=/root/user_lib
export R_VERSION=$(R --version | grep -Eo '[0-9.]+' | head -n 1)
export R_VERSION_MINOR=$(echo "$R_VERSION" | cut -d. -f1,2)
export ARCH=$(arch)
DIST_FAM=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2 | tr -d '"')
if [[ $DIST_FAM == "debian" ]]; then
export CODENAME=$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d'=' -f2 | tr -d '"')
elif [[ $DIST_FAM =~ ^(fedora) ]]; then
if [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el9" ]]; then
export CODENAME=rhel9
elif [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el8" ]]; then
export CODENAME=rhel8
fi
fi
# example CRAN dir structure: https://cran.r-project.org/src/contrib
mkdir -p /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/Archive
mkdir -p /root/src/contrib/Archive
cd "${PACKAGE}"
# NB: we start with the latest tag as we make the greedy assumption that the latest release has all deps of the previous ones
export TAGS=$(git tag -l --sort=-v:refname)
for i in $TAGS; do
git checkout -q -f $i
# the tag might differ from the actual version
TAG=$(grep "Version" DESCRIPTION | grep -Po "([0-9]+(\.[0-9]+)+)")
# don't install syslibs again
export PKG_SYSREQS=FALSE
export PKG_SYSREQS_VERBOSE=FALSE
# build source
EXISTS=$(find /root/src/contrib/ | grep -P "${PACKAGE}_${TAG}")
if [[ -n $EXISTS ]]; then
printf "[Skip] ${PACKAGE} '${TAG}' (source), already exists.\n"
# continue
else
printf "[Build] ${PACKAGE} '${TAG}' (source)\n"
fi
OUTPUT=$(R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pkgbuild::build(binary = FALSE, vignettes = FALSE, dest_path = "/root/src/contrib")')
if [[ $(printf "$OUTPUT" | grep "Error") != "" ]]; then
printf "[Error] ${PACKAGE} '${TAG}' (binary)\n"
printf $OUTPUT >/root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"_"${TAG}"_error
continue
else
# build binary
EXISTS=$(find /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/ | grep -P "${PACKAGE}_${TAG}")
if [[ -n $EXISTS ]]; then
printf "[Skip] ${PACKAGE} '${TAG}' (binary), already exists.\n"
continue
else
printf "[Build] ${PACKAGE} '${TAG}' (binary)\n"
fi
R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pak::local_install_dev_deps()'
R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pkgbuild::build(binary = TRUE, vignettes = FALSE, dest_path = sprintf("/root/__linux__/%s/%s/latest/src/contrib", Sys.getenv("CODENAME"), Sys.getenv("R_VERSION_MINOR")))'
# remove _aarch64-unknown-linux-gnu part in filename
rsync -a /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"_"${TAG}"*.tar.gz /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"_"${TAG}".tar.gz
fi
# create archive dir for package
mkdir -p /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/Archive/"${PACKAGE}"
# list all files in dir
all_versions=$(ls /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"*.tar.gz)
# Find the latest version
latest_version=$(ls /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"*.tar.gz | sort -V | tail -n 1)
# Move all other versions to the Archived directory
for file in $all_versions; do
if [ "$file" != "$latest_version" ]; then
rsync -a "$file" /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/Archive/"${PACKAGE}"/
fi
done
# list all files in dir
all_versions=$(ls /root/src/contrib/"${PACKAGE}"*.tar.gz)
# Find the latest version
latest_version=$(ls /root/src/contrib/"${PACKAGE}"*.tar.gz | sort -V | tail -n 1)
# create archive dir for package
mkdir -p /root/src/contrib/Archive/"${PACKAGE}"
# Move all other versions to the Archived directory
for file in $all_versions; do
if [ "$file" != "$latest_version" ]; then
rsync -a "$file" /root/src/contrib/Archive/"${PACKAGE}"/
fi
done
done
cd ..
rm -rf /root/"${PACKAGE}"
# }
# run &
done
# src
R -q -e "cranlike::add_PACKAGES(list.files('/root/src/contrib/', pattern='tar.gz'), '/root/src/contrib/')"
R -q -e "cranlike::package_versions('/root/src/contrib/')"
# bin
R -q -e "cranlike::add_PACKAGES(list.files('/root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/', pattern='tar.gz'), '/root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/')"
R -q -e "cranlike::package_versions('/root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/')"

44
03-upload.sh Normal file
View file

@ -0,0 +1,44 @@
#!/bin/bash
PACKAGES="mlr3filters stringi"
export R_VERSION=$(R --version | grep -Eo '[0-9.]+' | head -n 1)
export R_VERSION_MINOR=$(echo "$R_VERSION" | cut -d. -f1,2)
export ARCH=$(arch)
DIST_FAM=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2 | tr -d '"')
if [[ $DIST_FAM == "debian" ]]; then
export CODENAME=$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d'=' -f2 | tr -d '"')
elif [[ $DIST_FAM =~ ^(fedora) ]]; then
if [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el9" ]]; then
export CODENAME=rhel9
elif [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el8" ]]; then
export CODENAME=rhel8
fi
fi
for w in $PACKAGES; do
# source
aws s3 sync /root/src/contrib/ s3://cynkra-r-binaries-arm64/src/contrib/ --exclude '*error' --acl public-read
# binaries
aws s3 sync /root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/ s3://cynkra-r-binaries-arm64/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/ --exclude '*error' --acl public-read
done
### WORKING
# R -q -e 'install.packages("mlr3filters", repos="https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest/")'
# R -q -e 'remotes::install_version("mlr3filters", "0.7.0", repos="https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest")'
# install.packages("Rcpp", repos="https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/43/latest/")
# R -q -e 'options(repos = structure(c(CRAN = "https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest"))); remove.packages("mlr3filters"); pak::cache_delete(); pak::meta_clean(force=T); pak::pak("mlr3filters"); packageVersion("mlr3filters")'
# R -q -e 'options(repos = structure(c(CRAN = "https://cloud.r-project.org"))); remove.packages("mlr3filters"); pak::cache_delete(); pak::meta_clean(force=T); pak::pak("mlr3filters"); packageVersion("mlr3filters")'
# con <- gzcon(url(sprintf("%s/src/contrib/Meta/archive.rds",
# c("CRAN" = "https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest")), "rb"))
# readRDS(con)
# remotes:::package_find_archives("mlr3filters", "https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest", verbose = T)

31
04-create-archive-rds.sh Normal file
View file

@ -0,0 +1,31 @@
#!/bin/bash
# source: https://stackoverflow.com/a/55337537/4185785
echo '
library(plyr)
generateArchive <- function(archiveDir) {
archive <- file.info(list.files(archiveDir, recursive = TRUE, full.names = TRUE, pattern = "*.tar.gz"))
archive$packageName <- basename(dirname(rownames(archive)))
archive$packageFile <- basename(rownames(archive))
archive <- dlply(archive, "packageName", function(x) {
rownames(x) <- paste0(x$packageName, "/", x$packageFile)
x$packageName <- NULL
x$packageFile <- NULL
x
})
}
archiveDir <- "/root/__linux__/rhel9/4.3/latest/src/contrib/Archive"
metaPath <- "/root/__linux__/rhel9/4.3/latest/src/contrib//Meta"
dir.create(metaPath, showWarnings = FALSE)
archive <- generateArchive(archiveDir)
attr(archive, "split_type") <- NULL
attr(archive, "split_labels") <- NULL
saveRDS(archive, file.path(metaPath, "archive.rds"))
' >/root/create-archive.R
Rscript /root/create-archive.R
# sync Meta/archive.rds
aws s3 cp /root/__linux__/rhel9/4.3/latest/src/contrib/Meta/archive.rds s3://cynkra-r-binaries-arm64/__linux__/rhel9/4.3/latest/src/contrib/Meta/archive.rds --acl public-read