devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 11 Pull requests Releases Activity

finally add progressr support

Browse source
This commit is contained in:
Patrick Schratz 2024-09-11 00:15:18 +02:00
commit b1898f357c
Signed by: pat-s
GPG key ID: 3C6318841EF78925
6 changed files with 20 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.woodpecker/build.yaml
View file

File diff suppressed because one or more lines are too long

2
.woodpecker/process_cran_updates.yaml
View file

@ -55,7 +55,7 @@ steps:
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
# set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
- xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error", repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]][${BLOCK}]; pkgs = setdiff(pkgs, c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA" , "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM")); library(bincraftR); future::plan("sequential"); options(future.globals.onReference = NULL); process_cran_updates(interval = lubridate::interval(lubridate::today() - 2, lubridate::today() - 2), platform = "${PLATFORM}", process_updated = TRUE, process_new = TRUE)' - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error", repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]][${BLOCK}]; pkgs = setdiff(pkgs, c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA" , "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM")); library(bincraftR); future::plan("sequential"); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = lubridate::interval(lubridate::today() - 2, lubridate::today() - 2), platform = "${PLATFORM}", process_updated = TRUE, process_new = TRUE)'
backend_options: backend_options:
kubernetes: kubernetes:
### PROD ### PROD

2
.woodpecker/warm_up_build.yaml
View file

@ -46,7 +46,7 @@ steps:
image: busybox:latest image: busybox:latest
failure: ignore failure: ignore
commands: commands:
- sleep 200 - sleep 10
- echo "Warmed Up" - echo "Warmed Up"
backend_options: backend_options:
kubernetes: kubernetes:

2
DESCRIPTION
View file

@ -17,6 +17,7 @@ Imports:
magrittr, magrittr,
pkgbuild, pkgbuild,
progressr, progressr,
progress,
stringr, stringr,
purrr, purrr,
RPostgres, RPostgres,
@ -30,3 +31,4 @@ Remotes:
Encoding: UTF-8 Encoding: UTF-8
Roxygen: list(markdown = TRUE) Roxygen: list(markdown = TRUE)
RoxygenNote: 7.3.2 RoxygenNote: 7.3.2
SystemRequirements: libgit2 (>= 1.0): libgit2-devel (rpm) or libgit2-dev (deb)

8
Justfile
View file

@ -1,6 +1,10 @@
image OS: image OS ARCH:
if echo "{{OS}}" | grep -qi "redhat"; then \ if echo "{{OS}}" | grep -qi "redhat"; then \
docker buildx build --progress plain --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm64-binaries-r-{{OS}}:latest --push .; \ docker buildx build --progress plain --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm64-binaries-r-{{OS}}:latest --push .; \
else \ else \
docker buildx build --progress plain --build-arg GITHUB_PAT="$GITHUB_PAT" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm64-binaries-r-{{OS}}:latest --push .; \ docker buildx build --progress plain --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg ARCH={{ARCH}} -f docker/Dockerfile-{{OS}} --platform linux/{{ARCH}} -t devxygmbh/{{ARCH}}-binaries-r-{{OS}}:latest --push .; \
fi fi
# e.g. just build jammy arm64 later 1
build OS ARCH PACKAGE NCPUS:
docker run --rm -it --platform linux/{{ARCH}} -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e PGPASS="$PGPASS" -e NCPUS={{NCPUS}} --pull=always devxygmbh/{{ARCH}}-binaries-r-{{OS}}:latest R -q -e "options(progressr.enable = TRUE); library(future); plan(multisession); progressr::handlers(global = TRUE); progressr::handlers('cli'); bincraftR::build_binary_package(\"{{PACKAGE}}\", build_for_minor=FALSE, debug=FALSE, force=TRUE, deps_verbose = TRUE)"

16
R/build_binaries.R
View file

@ -81,16 +81,12 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
cli::cli_alert("Building binaries for {.pkg {package_name[[1]]}} with tags {.field {tag}}.") cli::cli_alert("Building binaries for {.pkg {package_name[[1]]}} with tags {.field {tag}}.")
# Set up the progress handler p <- progressr::progressor(along = tag)
# progressr::handlers(global = TRUE)
# progressr::handlers("cli", "debug")
# out <- progressr::with_progress({ worker_fun <- function(x, y, p) {
# p <- progressr::progressor(along = tag) p()
future.apply::future_mapply(function(x, y) {
tryCatch( tryCatch(
{ {
# p()
dump <- build_single_tag(x, y, dir_out_bin, local_clone_dir, dump <- build_single_tag(x, y, dir_out_bin, local_clone_dir,
platform = platform, debug = debug, force = force, platform = platform, debug = debug, force = force,
install_system_dependencies = install_system_dependencies, install_system_dependencies = install_system_dependencies,
@ -114,9 +110,11 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE, error = e$stderr) store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE, error = e$stderr)
} }
) )
}, package_name, tag, future.seed = TRUE) }
# }) future.apply::future_mapply(worker_fun, package_name, tag,
future.seed = TRUE, MoreArgs = list(p)
)
total_build_time <- round(Sys.time() - t1, 2) total_build_time <- round(Sys.time() - t1, 2)
cli::cli_alert("Execution time ({.pkg {package_name[[1]]}}) ({length(tag)} tags): {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.") cli::cli_alert("Execution time ({.pkg {package_name[[1]]}}) ({length(tag)} tags): {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.")