diff --git a/00-run.sh b/00-run.sh deleted file mode 100644 index e90f7e9..0000000 --- a/00-run.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [[ $1 == "redhat-9" ]]; then - docker run --name r-pkg-binaries-rhel9 -e RED_HAT_DEV_PW -d redhat/ubi9 tail -f /dev/null - docker exec -it r-pkg-binaries-rhel9 bash -elif [[ $1 == "ubuntu-24.04" ]]; then - docker run --name r-pkg-binaries-noble -d ubuntu/noble tail -f /dev/null - docker exec -it r-pkg-binaries-noble bash -elif [[ $1 == "redhat-8" ]]; then - docker run --name r-pkg-binaries-rhel8 -e RED_HAT_DEV_PW -d redhat/ubi8 tail -f /dev/null - docker exec -it r-pkg-binaries-rhel8 bash -fi - -# export R_VERSION=4.0.5 -# export PACKAGES="sf" - -bash -c ./01-prepare.sh -# bash -c ./02-build-binaries.sh -# bash -c ./03-upload.sh diff --git a/01-prepare.sh b/01-prepare.sh deleted file mode 100644 index bcd4250..0000000 --- a/01-prepare.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -### install R - -PLATFORM_ID=$(cat /etc/os-release | grep 'PLATFORM_ID=' | cut -d'=' -f2 | tr -d '"') - -if [ $PLATFORM_ID = "platform:el9" ]; then - - subscription-manager register --username pat-s --password $RED_HAT_DEV_PW - - subscription-manager repos --enable codeready-builder-for-rhel-9-$(arch)-rpms - dnf -y install git rsync - - echo "R Version: ${R_VERSION}" - echo "Downloading R-${R_VERSION}.rpm from S3..." - curl -o /tmp/R-${R_VERSION}.rpm https://cynkra-r-builds-arm64.s3.eu-central-1.amazonaws.com/el9/R-"${R_VERSION}"-1-1."$(arch)".rpm - dnf install -y /tmp/R-${R_VERSION}.rpm - -elif [ $PLATFORM_ID = "platform:el8" ]; then - - subscription-manager register --username pat-s --password $RED_HAT_DEV_PW - - subscription-manager repos --enable codeready-builder-for-rhel-8-$(arch)-rpms - - dnf -y install git rsync - echo "R Version: ${R_VERSION}" - echo "Downloading R-${R_VERSION}.rpm from S3..." - curl -o /tmp/R-${R_VERSION}.rpm https://cynkra-r-builds-arm64.s3.eu-central-1.amazonaws.com/el8/R-"${R_VERSION}"-1-1."$(arch)".rpm - dnf install -y /tmp/R-${R_VERSION}.rpm -fi - -export PATH=/opt/R/${R_VERSION}/bin:$PATH - -### install R packages needed for building - -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' -R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pak::pak(c("pkgbuild", "cranlike"))' - -### install awscli -# curl -s "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" -# unzip -qq awscliv2.zip -# ./aws/install -# rm -rf awscliv2.zip - -# mkdir /root/.aws -# echo -e "[default]\n -# aws_access_key_id=AKIATHTKSC6FRVE5LNE7\n -# aws_secret_access_key=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\n" >/root/.aws/credentials diff --git a/02-build-binaries.sh b/02-build-binaries.sh deleted file mode 100644 index 6304447..0000000 --- a/02-build-binaries.sh +++ /dev/null @@ -1,158 +0,0 @@ -#! /bin/bash - -R_VERSION=4.0.5 -PACKAGES="mlr3filters stringi" - -# install sysdeps for all packages -for w in $PACKAGES; do - - ### dynamic start - - export PACKAGE=$w - export R_LIBS_USER=/root/user_lib - mkdir -p /root/user_lib - - export R_VERSION=$(R --version | grep -Eo '[0-9.]+' | head -n 1) - export R_VERSION_MINOR=$(echo "$R_VERSION" | cut -d. -f1,2) - export ARCH=$(arch) - - DIST_FAM=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2 | tr -d '"') - if [[ $DIST_FAM == "debian" ]]; then - export CODENAME=$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d'=' -f2 | tr -d '"') - elif [[ $DIST_FAM =~ ^(rhel) ]]; then - if [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el9" ]]; then - export CODENAME=rhel9 - elif [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el8" ]]; then - export CODENAME=rhel8 - fi - fi - - git clone https://github.com/cran/"${PACKAGE}".git - cd "${PACKAGE}" - - # also installs system dependencies - # FIXME: auto-detection does not work on arm-based systems - export PKG_SYSREQS_PLATFORM=redhat-9 - export PKG_SYSREQS=TRUE - export PKG_SYSREQS_VERBOSE=TRUE - R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pak::local_install_dev_deps()' - - cd .. -done - -# build binaries from tags -for w in $PACKAGES; do - - # this runs in parallel - # run() { - export PACKAGE=$w - export R_LIBS_USER=/root/user_lib - - export R_VERSION=$(R --version | grep -Eo '[0-9.]+' | head -n 1) - export R_VERSION_MINOR=$(echo "$R_VERSION" | cut -d. -f1,2) - export ARCH=$(arch) - DIST_FAM=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2 | tr -d '"') - if [[ $DIST_FAM == "debian" ]]; then - export CODENAME=$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d'=' -f2 | tr -d '"') - elif [[ $DIST_FAM =~ ^(fedora) ]]; then - if [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el9" ]]; then - export CODENAME=rhel9 - elif [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el8" ]]; then - export CODENAME=rhel8 - fi - fi - - # example CRAN dir structure: https://cran.r-project.org/src/contrib - mkdir -p /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/Archive - mkdir -p /root/src/contrib/Archive - - cd "${PACKAGE}" - # NB: we start with the latest tag as we make the greedy assumption that the latest release has all deps of the previous ones - export TAGS=$(git tag -l --sort=-v:refname) - - for i in $TAGS; do - git checkout -q -f $i - - # the tag might differ from the actual version - TAG=$(grep "Version" DESCRIPTION | grep -Po "([0-9]+(\.[0-9]+)+)") - - # don't install syslibs again - export PKG_SYSREQS=FALSE - export PKG_SYSREQS_VERBOSE=FALSE - - # build source - EXISTS=$(find /root/src/contrib/ | grep -P "${PACKAGE}_${TAG}") - if [[ -n $EXISTS ]]; then - printf "[Skip] ${PACKAGE} '${TAG}' (source), already exists.\n" - # continue - else - printf "[Build] ${PACKAGE} '${TAG}' (source)\n" - fi - OUTPUT=$(R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pkgbuild::build(binary = FALSE, vignettes = FALSE, dest_path = "/root/src/contrib")') - - if [[ $(printf "$OUTPUT" | grep "Error") != "" ]]; then - printf "[Error] ${PACKAGE} '${TAG}' (binary)\n" - printf $OUTPUT >/root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"_"${TAG}"_error - continue - else - # build binary - EXISTS=$(find /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/ | grep -P "${PACKAGE}_${TAG}") - if [[ -n $EXISTS ]]; then - printf "[Skip] ${PACKAGE} '${TAG}' (binary), already exists.\n" - continue - else - printf "[Build] ${PACKAGE} '${TAG}' (binary)\n" - fi - R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pak::local_install_dev_deps()' - R -q -e 'options(repos = c(CRAN="https://stat.ethz.ch/CRAN/")); pkgbuild::build(binary = TRUE, vignettes = FALSE, dest_path = sprintf("/root/__linux__/%s/%s/latest/src/contrib", Sys.getenv("CODENAME"), Sys.getenv("R_VERSION_MINOR")))' - # remove _aarch64-unknown-linux-gnu part in filename - rsync -a /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"_"${TAG}"*.tar.gz /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"_"${TAG}".tar.gz - fi - - # create archive dir for package - mkdir -p /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/Archive/"${PACKAGE}" - # list all files in dir - all_versions=$(ls /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"*.tar.gz) - # Find the latest version - latest_version=$(ls /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/"${PACKAGE}"*.tar.gz | sort -V | tail -n 1) - - # Move all other versions to the Archived directory - for file in $all_versions; do - if [ "$file" != "$latest_version" ]; then - rsync -a "$file" /root/__linux__/"${CODENAME}"/"${R_VERSION_MINOR}"/latest/src/contrib/Archive/"${PACKAGE}"/ - fi - done - - # list all files in dir - all_versions=$(ls /root/src/contrib/"${PACKAGE}"*.tar.gz) - # Find the latest version - latest_version=$(ls /root/src/contrib/"${PACKAGE}"*.tar.gz | sort -V | tail -n 1) - - # create archive dir for package - mkdir -p /root/src/contrib/Archive/"${PACKAGE}" - # Move all other versions to the Archived directory - for file in $all_versions; do - if [ "$file" != "$latest_version" ]; then - rsync -a "$file" /root/src/contrib/Archive/"${PACKAGE}"/ - fi - done - - done - - cd .. - rm -rf /root/"${PACKAGE}" - # } - # run & -done - -# clean lefover files with wroning naming -find /root/__linux__/rhel9/4.0/latest/src/contrib -type f -name '*aarch64-unknown-linux*' -delete -find /root/src/contrib/ -type f -name '*aarch64-unknown-linux*' -delete - -# src -R -q -e "cranlike::add_PACKAGES(list.files('/root/src/contrib/', pattern='tar.gz'), '/root/src/contrib/')" -R -q -e "cranlike::package_versions('/root/src/contrib/')" - -# bin -R -q -e "cranlike::add_PACKAGES(list.files('/root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/', pattern='tar.gz'), '/root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/')" -R -q -e "cranlike::package_versions('/root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/')" diff --git a/03-upload.sh b/03-upload.sh deleted file mode 100644 index 484bf45..0000000 --- a/03-upload.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -PACKAGES="mlr3filters stringi" - -export R_VERSION=$(R --version | grep -Eo '[0-9.]+' | head -n 1) -export R_VERSION_MINOR=$(echo "$R_VERSION" | cut -d. -f1,2) -export ARCH=$(arch) - -DIST_FAM=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2 | tr -d '"') -if [[ $DIST_FAM == "debian" ]]; then - export CODENAME=$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d'=' -f2 | tr -d '"') -elif [[ $DIST_FAM =~ ^(fedora) ]]; then - if [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el9" ]]; then - export CODENAME=rhel9 - elif [[ $(grep 'PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') == "platform:el8" ]]; then - export CODENAME=rhel8 - fi -fi - -for w in $PACKAGES; do - # source - # aws s3 sync /root/src/contrib/ s3://cynkra-r-binaries-arm64/src/contrib/ --exclude '*error' --acl public-read - - # binaries - aws s3 sync /root/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/ s3://cynkra-r-binaries-arm64/__linux__/${CODENAME}/${R_VERSION_MINOR}/latest/src/contrib/ --exclude '*error' --acl public-read -done - -### WORKING -# R -q -e 'install.packages("mlr3filters", repos="https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest/")' -# R -q -e 'remotes::install_version("mlr3filters", "0.7.0", repos="https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest")' -# install.packages("Rcpp", repos="https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/43/latest/") - - -# R -q -e 'options(repos = structure(c(CRAN = "https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest"))); remove.packages("mlr3filters"); pak::cache_delete(); pak::meta_clean(force=T); pak::pak("mlr3filters"); packageVersion("mlr3filters")' - -# R -q -e 'options(repos = structure(c(CRAN = "https://cloud.r-project.org"))); remove.packages("mlr3filters"); pak::cache_delete(); pak::meta_clean(force=T); pak::pak("mlr3filters"); packageVersion("mlr3filters")' - - - -# con <- gzcon(url(sprintf("%s/src/contrib/Meta/archive.rds", -# c("CRAN" = "https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest")), "rb")) -# readRDS(con) - -# remotes:::package_find_archives("mlr3filters", "https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.3/latest", verbose = T) - - - -# /opt/R/4.3.3/bin/R -q -e 'options(repos = structure(c(CRAN = "https://cynkra-r-binaries-arm64.s3.eu-central-1.amazonaws.com/__linux__/rhel9/4.0/latest"))); pak::cache_delete(); pak::meta_clean(force=T); pak::pak("mlr3filters"); packageVersion("mlr3filters"); remove.packages("mlr3filters");'