add parallel execution

DESCRIPTION

@@ -18,5 +18,6 @@ Imports:
     pak,
     paws.storage,
     pkgbuild,
+    progressr,
     purrr,
     xml2

NAMESPACE

@@ -1,5 +1,9 @@
 # Generated by roxygen2: do not edit by hand
 
+import(progressr)
+importFrom(cli,cli_alert_info)
+importFrom(future,plan)
+importFrom(future.apply,future_mapply)
 importFrom(gert,git_config_set)
 importFrom(lubridate,as_date)
 importFrom(lubridate,dmy_hms)

R/build_binaries.R

@@ -1,3 +1,6 @@
+#' @import progressr
+#' @importFrom future plan
+#' @importFrom future.apply future_mapply
 #' @importFrom gert git_config_set
 #' @importFrom pak local_install_dev_deps
 #' @importFrom pkgbuild build
@@ -22,7 +25,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
     recursive = TRUE
   )
 
-  local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag)
+  cli::cli_h2("Installing system dependencies")
 
   ### Install system dependencies
   if (install_system_dependencies) {
@@ -31,49 +34,51 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
 
   gert::git_config_global_set("advice.detachedHead", "false")
 
-  # if tag = NULL, build all tags
   if (is.null(tag)) {
-    # when building all tags, we do a full clone to checkout all subsequent tags
+    # get all tags of package
-    # Using system git here as {gert} does not provide this functionality
+    gert::git_clone(sprintf("https://github.com/cran/%s", package_name),
-    # FIXME:
+      path = sprintf("%s/%s", tempdir(), "tmp1"),
-    # system2("git", args = c("config", "advice.detachedHead", "false"))
+      verbose = FALSE
-    # system2("git", args = c(
-    #   "clone", sprintf("https://github.com/cran/%s", package_name), sprintf("%s/%s", local_clone_dir, package_name)
-    # ))
-  } else {
-    cli::cli_alert_info("{.fun build_binary_package}: 1. Cloning package {.env {package_name}} with tag {.env {tag}}.")
-
-    # Using system git here as {gert} does not provide this functionality
-    system2("git", args = c(
-      "clone", "-q", sprintf("--branch=%s", tag),
-      sprintf("https://github.com/cran/%s", package_name), local_clone_dir_single
-    ))
-
-    cli::cli_alert_info("{.fun build_binary_package}: 2. Building package {.env {package_name}} with tag {.env {tag}}.")
-
-    pkgbuild::build(
-      path = sprintf("%s", local_clone_dir_single),
-      binary = TRUE, vignettes = FALSE,
-      dest_path = dir_out_bin
     )
+    # Retrieve all tags
+    all_tags <- gert::git_tag_list(repo = sprintf("%s/%s", tempdir(), "tmp1"))
+    unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE)
+    tag <- all_tags$name
+    package_name <- rep(package_name, length(tag))
 
-    if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
+    cli::cli_alert_info("Building binaries for {.pkg {package_name[[1]]}} with tags {.env {tag}}.")
-      cli::cli_alert_info('{.fun build_binary_package}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
-      # remove _aarch64-unknown-linux-gnu part in filename
-      system2("rsync", args = c(
-        sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag),
-        sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)
-      ))
-    } else {
-      cli::cli_alert_warning('{.fun build_binary_package}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.')
-    }
-    unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag))
-
-    cli::cli_alert_info("{.fun build_binary_package}: 4. Removing {.path {local_clone_dir_single}}.")
-    unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
-  }
   }
 
+  cli::cli_h2("Building")
+
+  # devtools::install(quiet = TRUE)
+
+  # Set up the progress handler
+  progressr::handlers(global = TRUE)
+  progressr::handlers("cli")
+
+  # cli::cli_alert("{package_name}")
+  # cli::cli_alert("{tag}")
+
+  out <- progressr::with_progress({
+    p <- progressr::progressor(along = tag)
+    future_mapply(function(x, y) {
+      p()
+      build_single_tag(x, y, dir_out_bin, local_clone_dir)
+    }, package_name, tag, future.seed = TRUE)
+  })
+
+  #   future.apply::future_mapply(build_single_tag,
+  #   package_name = package_name, tag = tag,
+  #   MoreArgs = list(
+  #     dir_out_bin = dir_out_bin,
+  #     local_clone_dir = local_clone_dir
+  #   ),
+  #   future.seed = TRUE
+  # )
+
+  return(invisible(TRUE))
+}
 
 install_package_system_dependencies <- function(package_name,
                                                 pak_pkg_syrsreqs_platform = "redhat-9",
@@ -99,3 +104,44 @@ install_package_system_dependencies <- function(package_name,
     unlink(sprintf("%s", local_clone_dir_single), recursive = TRUE, force = TRUE)
   }
 }
+
+
+#' @importFrom cli cli_alert_info
+#' @importFrom pkgbuild build
+build_single_tag <- function(package_name, tag = NULL,
+                             dir_out_bin,
+                             local_clone_dir) {
+
+  local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag)
+
+  cli::cli_alert_info("{.fun build_single_tag}: 1. Cloning package {.pkg {package_name}} with tag {.env {tag}}.")
+
+  # Using system git here as {gert} does not provide this functionality
+  system2("git", args = c(
+    "clone", "-q", sprintf("--branch=%s", tag),
+    sprintf("https://github.com/cran/%s", package_name), local_clone_dir_single
+  ))
+
+  cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.env {tag}}.")
+
+  pkgbuild::build(
+    path = sprintf("%s", local_clone_dir_single),
+    binary = TRUE, vignettes = FALSE,
+    dest_path = dir_out_bin
+  )
+
+  if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
+    cli::cli_alert_info('{.fun build_single_tag}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
+    # remove _aarch64-unknown-linux-gnu part in filename
+    system2("rsync", args = c(
+      sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag),
+      sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)
+    ))
+  } else {
+    cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.')
+  }
+  unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag))
+
+  cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.")
+  unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
+}

exec.sh

@@ -0,0 +1,27 @@
+# bootstrap
+# NB: ranger requires CXX14, R 4.0.5 still uses CXX11
+export R_VERSION=4.0.5
+export R_PROGRESSR_ENABLE=TRUE
+/opt/R/$R_VERSION/bin/R
+mkdir $HOME/.R && echo -e 'CXX_STD = CXX14\n\nVER=\nCCACHE=ccache\nCC=$(CCACHE) gcc$(VER) -std=gnu99\nCXX=$(CCACHE) g++$(VER)\nC11=$(CCACHE) g++$(VER)\nC14=$(CCACHE) g++$(VER)\nFC=$(CCACHE) gfortran$(VER)\nF77=$(CCACHE) gfortran$(VER)' > $HOME/.R/Makevars
+mkdir $HOME/.ccache && echo -e 'max_size = 5.0G\nsloppiness = include_file_ctime\nhash_dir=false' > $HOME/.ccache/ccache.conf
+
+/opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
+/opt/R/$R_VERSION/bin/R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE); Sys.setenv(PKG_SYSREQS_VERBOSE = TRUE); pak::pak("devtools")'
+/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); Sys.setenv(PKG_SYSREQS = TRUE); Sys.setenv(PKG_SYSREQS_VERBOSE = TRUE); pak::local_install_dev_deps()'
+
+/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); pak::pak(c("future", "future.apply"))'
+
+# single tag
+/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); devtools::load_all(); build_binary_package("mlr3filters", tag = "0.5.0", install_system_dependencies=FALSE)'
+# all tags (sequential)
+/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); devtools::load_all(); build_binary_package("mlr3filters", install_system_dependencies=FALSE)'
+# all tags (parallel)
+/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); devtools::load_all(); future::plan("multicore", workers = 2); build_binary_package("mlr3filters", install_system_dependencies=FALSE);'
+
+
+
+
+# execute
+rsync -a ~/git/homelab/rBinaries homelab-de:/home/pjs/
+ssh homelab-de "cd /home/pjs/rBinaries; /opt/R/$R_VERSION/bin/R -q -e \"devtools::load_all(); build_binary_package('mlr3filters', tag = '0.8.0')\""