From 23142a03f696ae049dbc466e2d89895531bb44c4 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 29 May 2024 19:52:07 +0200 Subject: [PATCH] save state --- 00-run.sh | 12 +++ 01-prepare.sh | 38 ++++++++++ 02-build-binaries.sh | 154 +++++++++++++++++++++++++++++++++++++++ 03-upload.sh | 44 +++++++++++ 04-create-archive-rds.sh | 31 ++++++++ 5 files changed, 279 insertions(+) create mode 100644 00-run.sh create mode 100644 01-prepare.sh create mode 100644 02-build-binaries.sh create mode 100644 03-upload.sh create mode 100644 04-create-archive-rds.sh diff --git a/00-run.sh b/00-run.sh new file mode 100644 index 0000000..e2c5a48 --- /dev/null +++ b/00-run.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +docker run --name r-pkg-binaries-rhel9 -e RED_HAT_DEV_PW -d redhat/ubi9 tail -f /dev/null +docker exec -it r-pkg-binaries-rhel9 bash + +export R_VERSION=4.3.3 +export PACKAGES="sf" + +bash -c ./01-prepare.sh +bash -c ./02-build-binaries.sh +bash -c ./03-upload.sh + diff --git a/01-prepare.sh b/01-prepare.sh new file mode 100644 index 0000000..b7d8935 --- /dev/null +++ b/01-prepare.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +### install R + +PLATFORM_ID=$(cat /etc/os-release | grep 'PLATFORM_ID=' | cut -d'=' -f2 | tr -d '"') + +if [ $PLATFORM_ID = "platform:el9" ]; then + + subscription-manager register --username pat-s --password $RED_HAT_DEV_PW + + subscription-manager repos --enable codeready-builder-for-rhel-9-$(arch)-rpms + dnf -y install git rsync + # dnf update -y + # FIXME: protect PW + + echo "R Version: ${R_VERSION}" + echo "Downloading R-${R_VERSION}.rpm from S3..." + curl -o /tmp/R-${R_VERSION}.rpm https://cynkra-r-builds-arm64.s3.eu-central-1.amazonaws.com/el9/R-"${R_VERSION}"-1-1."$(arch)".rpm + dnf install -y /tmp/R-${R_VERSION}.rpm +fi + +export PATH=/opt/R/${R_VERSION}/bin:$PATH + +### install R packages needed for building + +R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pak::pak(c("pkgbuild", "cranlike", "drat"))' + +### install awscli +curl -s "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" +unzip -qq awscliv2.zip +./aws/install +rm -rf awscliv2.zip + +mkdir /root/.aws +echo -e "[default]\n +aws_access_key_id=AKIATHTKSC6FRVE5LNE7\n +aws_secret_access_key=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\n" >/root/.aws/credentials diff --git a/02-build-binaries.sh b/02-build-binaries.sh new file mode 100644 index 0000000..363a528 --- /dev/null +++ b/02-build-binaries.sh @@ -0,0 +1,154 @@ +#! /bin/bash + +R_VERSION=4.0.5 +PACKAGES="mlr3filters stringi" + +# install sysdeps for all packages +for w in $PACKAGES; do + + ### dynamic start + + export PACKAGE=$w + export R_LIBS_USER=/root/user_lib + mkdir -p /root/user_lib + + export R_VERSION=$(R --version | grep -Eo '[0-9.]+' | head -n 1) + export R_VERSION_MINOR=$(echo "$R_VERSION" | cut -d. -f1,2) + export ARCH=$(arch) + + DIST_FAM=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2 | tr -d '"') + if [[ $DIST_FAM == "debian" ]]; then + export CODENAME=$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d'=' -f2 | tr -d '"') + elif [[ $DIST_FAM =~ ^(rhel) ]]; then + if [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el9" ]]; then + export CODENAME=rhel9 + elif [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el8" ]]; then + export CODENAME=rhel8 + fi + fi + + git clone https://github.com/cran/"${PACKAGE}".git + cd "${PACKAGE}" + + # also installs system dependencies + # FIXME: auto-detection does not work on arm-based systems + export PKG_SYSREQS_PLATFORM=redhat-9 + export PKG_SYSREQS=TRUE + export PKG_SYSREQS_VERBOSE=TRUE + R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pak::local_install_dev_deps()' + + cd .. +done + +# build binaries from tags +for w in $PACKAGES; do + + # this runs in parallel + # run() { + export PACKAGE=$w + export R_LIBS_USER=/root/user_lib + + export R_VERSION=$(R --version | grep -Eo '[0-9.]+' | head -n 1) + export R_VERSION_MINOR=$(echo "$R_VERSION" | cut -d. -f1,2) + export ARCH=$(arch) + DIST_FAM=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2 | tr -d '"') + if [[ $DIST_FAM == "debian" ]]; then + export CODENAME=$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d'=' -f2 | tr -d '"') + elif [[ $DIST_FAM =~ ^(fedora) ]]; then + if [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el9" ]]; then + export CODENAME=rhel9 + elif [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el8" ]]; then + export CODENAME=rhel8 + fi + fi + + # example CRAN dir structure: https://cran.r-project.org/src/contrib + mkdir -p /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/Archive + mkdir -p /root/src/contrib/Archive + + cd "${PACKAGE}" + # NB: we start with the latest tag as we make the greedy assumption that the latest release has all deps of the previous ones + export TAGS=$(git tag -l --sort=-v:refname) + + for i in $TAGS; do + git checkout -q -f $i + + # the tag might differ from the actual version + TAG=$(grep "Version" DESCRIPTION | grep -Po "([0-9]+(\.[0-9]+)+)") + + # don't install syslibs again + export PKG_SYSREQS=FALSE + export PKG_SYSREQS_VERBOSE=FALSE + + # build source + EXISTS=$(find /root/src/contrib/ | grep -P "${PACKAGE}_${TAG}") + if [[ -n $EXISTS ]]; then + printf "[Skip] ${PACKAGE} '${TAG}' (source), already exists.\n" + # continue + else + printf "[Build] ${PACKAGE} '${TAG}' (source)\n" + fi + OUTPUT=$(R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pkgbuild::build(binary = FALSE, vignettes = FALSE, dest_path = "/root/src/contrib")') + + if [[ $(printf "$OUTPUT" | grep "Error") != "" ]]; then + printf "[Error] ${PACKAGE} '${TAG}' (binary)\n" + printf $OUTPUT >/root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"_"${TAG}"_error + continue + else + # build binary + EXISTS=$(find /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/ | grep -P "${PACKAGE}_${TAG}") + if [[ -n $EXISTS ]]; then + printf "[Skip] ${PACKAGE} '${TAG}' (binary), already exists.\n" + continue + else + printf "[Build] ${PACKAGE} '${TAG}' (binary)\n" + fi + R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pak::local_install_dev_deps()' + R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pkgbuild::build(binary = TRUE, vignettes = FALSE, dest_path = sprintf("/root/__linux__/%s/%s/latest/src/contrib", Sys.getenv("CODENAME"), Sys.getenv("R_VERSION_MINOR")))' + # remove _aarch64-unknown-linux-gnu part in filename + rsync -a /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"_"${TAG}"*.tar.gz /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"_"${TAG}".tar.gz + fi + + # create archive dir for package + mkdir -p /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/Archive/"${PACKAGE}" + # list all files in dir + all_versions=$(ls /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"*.tar.gz) + # Find the latest version + latest_version=$(ls /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"*.tar.gz | sort -V | tail -n 1) + + # Move all other versions to the Archived directory + for file in $all_versions; do + if [ "$file" != "$latest_version" ]; then + rsync -a "$file" /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/Archive/"${PACKAGE}"/ + fi + done + + # list all files in dir + all_versions=$(ls /root/src/contrib/"${PACKAGE}"*.tar.gz) + # Find the latest version + latest_version=$(ls /root/src/contrib/"${PACKAGE}"*.tar.gz | sort -V | tail -n 1) + + # create archive dir for package + mkdir -p /root/src/contrib/Archive/"${PACKAGE}" + # Move all other versions to the Archived directory + for file in $all_versions; do + if [ "$file" != "$latest_version" ]; then + rsync -a "$file" /root/src/contrib/Archive/"${PACKAGE}"/ + fi + done + + done + + cd .. + rm -rf /root/"${PACKAGE}" + # } + # run & +done + +# src +R -q -e "cranlike::add_PACKAGES(list.files('/root/src/contrib/', pattern='tar.gz'), '/root/src/contrib/')" +R -q -e "cranlike::package_versions('/root/src/contrib/')" + +# bin +R -q -e "cranlike::add_PACKAGES(list.files('/root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/', pattern='tar.gz'), '/root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/')" +R -q -e "cranlike::package_versions('/root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/')" diff --git a/03-upload.sh b/03-upload.sh new file mode 100644 index 0000000..51753c3 --- /dev/null +++ b/03-upload.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +PACKAGES="mlr3filters stringi" + +export R_VERSION=$(R --version | grep -Eo '[0-9.]+' | head -n 1) +export R_VERSION_MINOR=$(echo "$R_VERSION" | cut -d. -f1,2) +export ARCH=$(arch) + +DIST_FAM=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2 | tr -d '"') +if [[ $DIST_FAM == "debian" ]]; then + export CODENAME=$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d'=' -f2 | tr -d '"') +elif [[ $DIST_FAM =~ ^(fedora) ]]; then + if [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el9" ]]; then + export CODENAME=rhel9 + elif [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el8" ]]; then + export CODENAME=rhel8 + fi +fi + +for w in $PACKAGES; do + # source + aws s3 sync /root/src/contrib/ s3://cynkra-r-binaries-arm64/src/contrib/ --exclude '*error' --acl public-read + + # binaries + aws s3 sync /root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/ s3://cynkra-r-binaries-arm64/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/ --exclude '*error' --acl public-read +done + +### WORKING +# R -q -e 'install.packages("mlr3filters", repos="https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest/")' +# R -q -e 'remotes::install_version("mlr3filters", "0.7.0", repos="https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest")' +# install.packages("Rcpp", repos="https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/43/latest/") + + +# R -q -e 'options(repos = structure(c(CRAN = "https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest"))); remove.packages("mlr3filters"); pak::cache_delete(); pak::meta_clean(force=T); pak::pak("mlr3filters"); packageVersion("mlr3filters")' + +# R -q -e 'options(repos = structure(c(CRAN = "https://cloud.r-project.org"))); remove.packages("mlr3filters"); pak::cache_delete(); pak::meta_clean(force=T); pak::pak("mlr3filters"); packageVersion("mlr3filters")' + + + +# con <- gzcon(url(sprintf("%s/src/contrib/Meta/archive.rds", +# c("CRAN" = "https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest")), "rb")) +# readRDS(con) + +# remotes:::package_find_archives("mlr3filters", "https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest", verbose = T) diff --git a/04-create-archive-rds.sh b/04-create-archive-rds.sh new file mode 100644 index 0000000..969acd5 --- /dev/null +++ b/04-create-archive-rds.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# source: https://stackoverflow.com/a/55337537/4185785 + +echo ' +library(plyr) +generateArchive <- function(archiveDir) { + archive <- file.info(list.files(archiveDir, recursive = TRUE, full.names = TRUE, pattern = "*.tar.gz")) + archive$packageName <- basename(dirname(rownames(archive))) + archive$packageFile <- basename(rownames(archive)) + archive <- dlply(archive, "packageName", function(x) { + rownames(x) <- paste0(x$packageName, "/", x$packageFile) + x$packageName <- NULL + x$packageFile <- NULL + x + }) +} + +archiveDir <- "/root/__linux__/rhel9/4.3/latest/src/contrib/Archive" +metaPath <- "/root/__linux__/rhel9/4.3/latest/src/contrib//Meta" +dir.create(metaPath, showWarnings = FALSE) +archive <- generateArchive(archiveDir) +attr(archive, "split_type") <- NULL +attr(archive, "split_labels") <- NULL +saveRDS(archive, file.path(metaPath, "archive.rds")) +' >/root/create-archive.R + +Rscript /root/create-archive.R + +# sync Meta/archive.rds +aws s3 cp /root/__linux__/rhel9/4.3/latest/src/contrib/Meta/archive.rds s3://cynkra-r-binaries-arm64/__linux__/rhel9/4.3/latest/src/contrib/Meta/archive.rds --acl public-read