add parallel execution
This commit is contained in:
parent
5e32452d09
commit
de2d7878e3
GPG key ID:
3C6318841EF78925
4 changed files with 116 additions and 38 deletions
|
|
@ -18,5 +18,6 @@ Imports:
|
|||
pak,
|
||||
paws.storage,
|
||||
pkgbuild,
|
||||
progressr,
|
||||
purrr,
|
||||
xml2
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
# Generated by roxygen2: do not edit by hand
|
||||
|
||||
import(progressr)
|
||||
importFrom(cli,cli_alert_info)
|
||||
importFrom(future,plan)
|
||||
importFrom(future.apply,future_mapply)
|
||||
importFrom(gert,git_config_set)
|
||||
importFrom(lubridate,as_date)
|
||||
importFrom(lubridate,dmy_hms)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
#' @import progressr
|
||||
#' @importFrom future plan
|
||||
#' @importFrom future.apply future_mapply
|
||||
#' @importFrom gert git_config_set
|
||||
#' @importFrom pak local_install_dev_deps
|
||||
#' @importFrom pkgbuild build
|
||||
|
|
@ -22,7 +25,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
|||
recursive = TRUE
|
||||
)
|
||||
|
||||
local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag)
|
||||
cli::cli_h2("Installing system dependencies")
|
||||
|
||||
### Install system dependencies
|
||||
if (install_system_dependencies) {
|
||||
|
|
@ -31,49 +34,51 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
|||
|
||||
gert::git_config_global_set("advice.detachedHead", "false")
|
||||
|
||||
# if tag = NULL, build all tags
|
||||
if (is.null(tag)) {
|
||||
# when building all tags, we do a full clone to checkout all subsequent tags
|
||||
# Using system git here as {gert} does not provide this functionality
|
||||
# FIXME:
|
||||
# system2("git", args = c("config", "advice.detachedHead", "false"))
|
||||
# system2("git", args = c(
|
||||
# "clone", sprintf("https://github.com/cran/%s", package_name), sprintf("%s/%s", local_clone_dir, package_name)
|
||||
# ))
|
||||
} else {
|
||||
cli::cli_alert_info("{.fun build_binary_package}: 1. Cloning package {.env {package_name}} with tag {.env {tag}}.")
|
||||
|
||||
# Using system git here as {gert} does not provide this functionality
|
||||
system2("git", args = c(
|
||||
"clone", "-q", sprintf("--branch=%s", tag),
|
||||
sprintf("https://github.com/cran/%s", package_name), local_clone_dir_single
|
||||
))
|
||||
|
||||
cli::cli_alert_info("{.fun build_binary_package}: 2. Building package {.env {package_name}} with tag {.env {tag}}.")
|
||||
|
||||
pkgbuild::build(
|
||||
path = sprintf("%s", local_clone_dir_single),
|
||||
binary = TRUE, vignettes = FALSE,
|
||||
dest_path = dir_out_bin
|
||||
# get all tags of package
|
||||
gert::git_clone(sprintf("https://github.com/cran/%s", package_name),
|
||||
path = sprintf("%s/%s", tempdir(), "tmp1"),
|
||||
verbose = FALSE
|
||||
)
|
||||
# Retrieve all tags
|
||||
all_tags <- gert::git_tag_list(repo = sprintf("%s/%s", tempdir(), "tmp1"))
|
||||
unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE)
|
||||
tag <- all_tags$name
|
||||
package_name <- rep(package_name, length(tag))
|
||||
|
||||
if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
|
||||
cli::cli_alert_info('{.fun build_binary_package}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
|
||||
# remove _aarch64-unknown-linux-gnu part in filename
|
||||
system2("rsync", args = c(
|
||||
sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag),
|
||||
sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)
|
||||
))
|
||||
} else {
|
||||
cli::cli_alert_warning('{.fun build_binary_package}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.')
|
||||
}
|
||||
unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag))
|
||||
|
||||
cli::cli_alert_info("{.fun build_binary_package}: 4. Removing {.path {local_clone_dir_single}}.")
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
cli::cli_alert_info("Building binaries for {.pkg {package_name[[1]]}} with tags {.env {tag}}.")
|
||||
}
|
||||
}
|
||||
|
||||
cli::cli_h2("Building")
|
||||
|
||||
# devtools::install(quiet = TRUE)
|
||||
|
||||
# Set up the progress handler
|
||||
progressr::handlers(global = TRUE)
|
||||
progressr::handlers("cli")
|
||||
|
||||
# cli::cli_alert("{package_name}")
|
||||
# cli::cli_alert("{tag}")
|
||||
|
||||
out <- progressr::with_progress({
|
||||
p <- progressr::progressor(along = tag)
|
||||
future_mapply(function(x, y) {
|
||||
p()
|
||||
build_single_tag(x, y, dir_out_bin, local_clone_dir)
|
||||
}, package_name, tag, future.seed = TRUE)
|
||||
})
|
||||
|
||||
# future.apply::future_mapply(build_single_tag,
|
||||
# package_name = package_name, tag = tag,
|
||||
# MoreArgs = list(
|
||||
# dir_out_bin = dir_out_bin,
|
||||
# local_clone_dir = local_clone_dir
|
||||
# ),
|
||||
# future.seed = TRUE
|
||||
# )
|
||||
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
||||
install_package_system_dependencies <- function(package_name,
|
||||
pak_pkg_syrsreqs_platform = "redhat-9",
|
||||
|
|
@ -99,3 +104,44 @@ install_package_system_dependencies <- function(package_name,
|
|||
unlink(sprintf("%s", local_clone_dir_single), recursive = TRUE, force = TRUE)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#' @importFrom cli cli_alert_info
|
||||
#' @importFrom pkgbuild build
|
||||
build_single_tag <- function(package_name, tag = NULL,
|
||||
dir_out_bin,
|
||||
local_clone_dir) {
|
||||
|
||||
local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag)
|
||||
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 1. Cloning package {.pkg {package_name}} with tag {.env {tag}}.")
|
||||
|
||||
# Using system git here as {gert} does not provide this functionality
|
||||
system2("git", args = c(
|
||||
"clone", "-q", sprintf("--branch=%s", tag),
|
||||
sprintf("https://github.com/cran/%s", package_name), local_clone_dir_single
|
||||
))
|
||||
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.env {tag}}.")
|
||||
|
||||
pkgbuild::build(
|
||||
path = sprintf("%s", local_clone_dir_single),
|
||||
binary = TRUE, vignettes = FALSE,
|
||||
dest_path = dir_out_bin
|
||||
)
|
||||
|
||||
if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
|
||||
cli::cli_alert_info('{.fun build_single_tag}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
|
||||
# remove _aarch64-unknown-linux-gnu part in filename
|
||||
system2("rsync", args = c(
|
||||
sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag),
|
||||
sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)
|
||||
))
|
||||
} else {
|
||||
cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.')
|
||||
}
|
||||
unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag))
|
||||
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.")
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
}
|
||||
|
|
|
|||
27
exec.sh
Executable file
27
exec.sh
Executable file
|
|
@ -0,0 +1,27 @@
|
|||
# bootstrap
|
||||
# NB: ranger requires CXX14, R 4.0.5 still uses CXX11
|
||||
export R_VERSION=4.0.5
|
||||
export R_PROGRESSR_ENABLE=TRUE
|
||||
/opt/R/$R_VERSION/bin/R
|
||||
mkdir $HOME/.R && echo -e 'CXX_STD = CXX14\n\nVER=\nCCACHE=ccache\nCC=$(CCACHE) gcc$(VER) -std=gnu99\nCXX=$(CCACHE) g++$(VER)\nC11=$(CCACHE) g++$(VER)\nC14=$(CCACHE) g++$(VER)\nFC=$(CCACHE) gfortran$(VER)\nF77=$(CCACHE) gfortran$(VER)' > $HOME/.R/Makevars
|
||||
mkdir $HOME/.ccache && echo -e 'max_size = 5.0G\nsloppiness = include_file_ctime\nhash_dir=false' > $HOME/.ccache/ccache.conf
|
||||
|
||||
/opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
||||
/opt/R/$R_VERSION/bin/R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE); Sys.setenv(PKG_SYSREQS_VERBOSE = TRUE); pak::pak("devtools")'
|
||||
/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); Sys.setenv(PKG_SYSREQS = TRUE); Sys.setenv(PKG_SYSREQS_VERBOSE = TRUE); pak::local_install_dev_deps()'
|
||||
|
||||
/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); pak::pak(c("future", "future.apply"))'
|
||||
|
||||
# single tag
|
||||
/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); devtools::load_all(); build_binary_package("mlr3filters", tag = "0.5.0", install_system_dependencies=FALSE)'
|
||||
# all tags (sequential)
|
||||
/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); devtools::load_all(); build_binary_package("mlr3filters", install_system_dependencies=FALSE)'
|
||||
# all tags (parallel)
|
||||
/opt/R/$R_VERSION/bin/R -q -e 'setwd("/home/pjs/rBinaries"); devtools::load_all(); future::plan("multicore", workers = 2); build_binary_package("mlr3filters", install_system_dependencies=FALSE);'
|
||||
|
||||
|
||||
|
||||
|
||||
# execute
|
||||
rsync -a ~/git/homelab/rBinaries homelab-de:/home/pjs/
|
||||
ssh homelab-de "cd /home/pjs/rBinaries; /opt/R/$R_VERSION/bin/R -q -e \"devtools::load_all(); build_binary_package('mlr3filters', tag = '0.8.0')\""
|
||||
Loading…
Reference in a new issue