From de2d7878e310d1303e45aae54369b5fc7926ddb4 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 2 Jun 2024 15:17:33 +0200 Subject: [PATCH] add parallel execution --- DESCRIPTION | 1 + NAMESPACE | 4 ++ R/build_binaries.R | 124 +++++++++++++++++++++++++++++++-------------- exec.sh | 27 ++++++++++ 4 files changed, 117 insertions(+), 39 deletions(-) create mode 100755 exec.sh diff --git a/DESCRIPTION b/DESCRIPTION index 9b3ee5c..7a0cbd5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,5 +18,6 @@ Imports: pak, paws.storage, pkgbuild, + progressr, purrr, xml2 diff --git a/NAMESPACE b/NAMESPACE index c2f3679..fee03a3 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,5 +1,9 @@ # Generated by roxygen2: do not edit by hand +import(progressr) +importFrom(cli,cli_alert_info) +importFrom(future,plan) +importFrom(future.apply,future_mapply) importFrom(gert,git_config_set) importFrom(lubridate,as_date) importFrom(lubridate,dmy_hms) diff --git a/R/build_binaries.R b/R/build_binaries.R index 3fe9f33..54d59b0 100644 --- a/R/build_binaries.R +++ b/R/build_binaries.R @@ -1,3 +1,6 @@ +#' @import progressr +#' @importFrom future plan +#' @importFrom future.apply future_mapply #' @importFrom gert git_config_set #' @importFrom pak local_install_dev_deps #' @importFrom pkgbuild build @@ -22,7 +25,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, recursive = TRUE ) - local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag) + cli::cli_h2("Installing system dependencies") ### Install system dependencies if (install_system_dependencies) { @@ -31,49 +34,51 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, gert::git_config_global_set("advice.detachedHead", "false") - # if tag = NULL, build all tags if (is.null(tag)) { - # when building all tags, we do a full clone to checkout all subsequent tags - # Using system git here as {gert} does not provide this functionality - # FIXME: - # system2("git", args = c("config", "advice.detachedHead", "false")) - # system2("git", args = c( - # "clone", sprintf("https://github.com/cran/%s", package_name), sprintf("%s/%s", local_clone_dir, package_name) - # )) - } else { - cli::cli_alert_info("{.fun build_binary_package}: 1. Cloning package {.env {package_name}} with tag {.env {tag}}.") - - # Using system git here as {gert} does not provide this functionality - system2("git", args = c( - "clone", "-q", sprintf("--branch=%s", tag), - sprintf("https://github.com/cran/%s", package_name), local_clone_dir_single - )) - - cli::cli_alert_info("{.fun build_binary_package}: 2. Building package {.env {package_name}} with tag {.env {tag}}.") - - pkgbuild::build( - path = sprintf("%s", local_clone_dir_single), - binary = TRUE, vignettes = FALSE, - dest_path = dir_out_bin + # get all tags of package + gert::git_clone(sprintf("https://github.com/cran/%s", package_name), + path = sprintf("%s/%s", tempdir(), "tmp1"), + verbose = FALSE ) + # Retrieve all tags + all_tags <- gert::git_tag_list(repo = sprintf("%s/%s", tempdir(), "tmp1")) + unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE) + tag <- all_tags$name + package_name <- rep(package_name, length(tag)) - if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) { - cli::cli_alert_info('{.fun build_binary_package}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}') - # remove _aarch64-unknown-linux-gnu part in filename - system2("rsync", args = c( - sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag), - sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag) - )) - } else { - cli::cli_alert_warning('{.fun build_binary_package}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.') - } - unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag)) - - cli::cli_alert_info("{.fun build_binary_package}: 4. Removing {.path {local_clone_dir_single}}.") - unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) + cli::cli_alert_info("Building binaries for {.pkg {package_name[[1]]}} with tags {.env {tag}}.") } -} + cli::cli_h2("Building") + + # devtools::install(quiet = TRUE) + + # Set up the progress handler + progressr::handlers(global = TRUE) + progressr::handlers("cli") + + # cli::cli_alert("{package_name}") + # cli::cli_alert("{tag}") + + out <- progressr::with_progress({ + p <- progressr::progressor(along = tag) + future_mapply(function(x, y) { + p() + build_single_tag(x, y, dir_out_bin, local_clone_dir) + }, package_name, tag, future.seed = TRUE) + }) + + # future.apply::future_mapply(build_single_tag, + # package_name = package_name, tag = tag, + # MoreArgs = list( + # dir_out_bin = dir_out_bin, + # local_clone_dir = local_clone_dir + # ), + # future.seed = TRUE + # ) + + return(invisible(TRUE)) +} install_package_system_dependencies <- function(package_name, pak_pkg_syrsreqs_platform = "redhat-9", @@ -99,3 +104,44 @@ install_package_system_dependencies <- function(package_name, unlink(sprintf("%s", local_clone_dir_single), recursive = TRUE, force = TRUE) } } + + +#' @importFrom cli cli_alert_info +#' @importFrom pkgbuild build +build_single_tag <- function(package_name, tag = NULL, + dir_out_bin, + local_clone_dir) { + + local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag) + + cli::cli_alert_info("{.fun build_single_tag}: 1. Cloning package {.pkg {package_name}} with tag {.env {tag}}.") + + # Using system git here as {gert} does not provide this functionality + system2("git", args = c( + "clone", "-q", sprintf("--branch=%s", tag), + sprintf("https://github.com/cran/%s", package_name), local_clone_dir_single + )) + + cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.env {tag}}.") + + pkgbuild::build( + path = sprintf("%s", local_clone_dir_single), + binary = TRUE, vignettes = FALSE, + dest_path = dir_out_bin + ) + + if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) { + cli::cli_alert_info('{.fun build_single_tag}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}') + # remove _aarch64-unknown-linux-gnu part in filename + system2("rsync", args = c( + sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag), + sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag) + )) + } else { + cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.') + } + unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag)) + + cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.") + unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) +} diff --git a/exec.sh b/exec.sh new file mode 100755 index 0000000..b99ab3c --- /dev/null +++ b/exec.sh @@ -0,0 +1,27 @@ +# bootstrap +# NB: ranger requires CXX14, R 4.0.5 still uses CXX11 +export R_VERSION=4.0.5 +export R_PROGRESSR_ENABLE=TRUE +/opt/R/$R_VERSION/bin/R +mkdir $HOME/.R && echo -e 'CXX_STD = CXX14\n\nVER=\nCCACHE=ccache\nCC=$(CCACHE) gcc$(VER) -std=gnu99\nCXX=$(CCACHE) g++$(VER)\nC11=$(CCACHE) g++$(VER)\nC14=$(CCACHE) g++$(VER)\nFC=$(CCACHE) gfortran$(VER)\nF77=$(CCACHE) gfortran$(VER)' > $HOME/.R/Makevars +mkdir $HOME/.ccache && echo -e 'max_size = 5.0G\nsloppiness = include_file_ctime\nhash_dir=false' > $HOME/.ccache/ccache.conf + +/opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +/opt/R/$R_VERSION/bin/R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE); Sys.setenv(PKG_SYSREQS_VERBOSE = TRUE); pak::pak("devtools")' +/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); Sys.setenv(PKG_SYSREQS = TRUE); Sys.setenv(PKG_SYSREQS_VERBOSE = TRUE); pak::local_install_dev_deps()' + +/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); pak::pak(c("future", "future.apply"))' + +# single tag +/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); devtools::load_all(); build_binary_package("mlr3filters", tag = "0.5.0", install_system_dependencies=FALSE)' +# all tags (sequential) +/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); devtools::load_all(); build_binary_package("mlr3filters", install_system_dependencies=FALSE)' +# all tags (parallel) +/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); devtools::load_all(); future::plan("multicore", workers = 2); build_binary_package("mlr3filters", install_system_dependencies=FALSE);' + + + + +# execute +rsync -a ~/git/homelab/rBinaries homelab-de:/home/pjs/ +ssh homelab-de "cd /home/pjs/rBinaries; /opt/R/$R_VERSION/bin/R -q -e \"devtools::load_all(); build_binary_package('mlr3filters', tag = '0.8.0')\""