devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 11 Pull requests 1 Releases Activity

only parallelise package builds

Browse source
This commit is contained in:
Patrick Schratz 2024-09-23 13:25:21 +02:00
commit 6db7443844
Signed by: pat-s
GPG key ID: 3C6318841EF78925
5 changed files with 93 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

76
.woodpecker/build.yaml
View file

@ -20,27 +20,27 @@ matrix:
# NCPUS: 2 # NCPUS: 2
# MEMORY: 29Gi # MEMORY: 29Gi
# STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2404 # - PLATFORM: ubuntu-2404
BLOCK: 1248:length(pkgs) # BLOCK: 1248:length(pkgs)
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2204 # - PLATFORM: ubuntu-2204
BLOCK: 1325:length(pkgs) # BLOCK: 1325:length(pkgs)
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: alpine-320 # - PLATFORM: alpine-320
BLOCK: 1199:length(pkgs) # BLOCK: 1199:length(pkgs)
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
### AMD ### AMD
- PLATFORM: redhat-9 - PLATFORM: redhat-9
BLOCK: 690:length(pkgs) BLOCK: 699:length(pkgs)
ARCH: amd64 ARCH: amd64
NCPUS: 2 NCPUS: 2
MEMORY: 29Gi MEMORY: 29Gi
@ -51,24 +51,24 @@ matrix:
NCPUS: 2 NCPUS: 2
MEMORY: 29Gi MEMORY: 29Gi
STRATEGY: multisession STRATEGY: multisession
- PLATFORM: ubuntu-2404 # - PLATFORM: ubuntu-2404
BLOCK: 1481:length(pkgs) # BLOCK: 1481:length(pkgs)
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2204 # - PLATFORM: ubuntu-2204
BLOCK: 1542:length(pkgs) # BLOCK: 1542:length(pkgs)
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: alpine-320 # - PLATFORM: alpine-320
BLOCK: 1161:length(pkgs) # BLOCK: 1161:length(pkgs)
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
steps: steps:
- name: 'Build binaries (${PLATFORM}: ${BLOCK})' - name: 'Build binaries (${PLATFORM}: ${BLOCK})'
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
@ -132,7 +132,7 @@ steps:
# tfrmtbuilder -> V8 (alpine) # tfrmtbuilder -> V8 (alpine)
# Rcmdr (alpine) # Rcmdr (alpine)
# readstata13 (alpine) # readstata13 (alpine)
# - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers(global = TRUE); progressr::handlers("cli"); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
backend_options: backend_options:
kubernetes: kubernetes:
### PROD-parallel ### PROD-parallel

74
.woodpecker/warm_up_build.yaml
View file

@ -17,24 +17,24 @@ matrix:
# NCPUS: 2 # NCPUS: 2
# MEMORY: 29Gi # MEMORY: 29Gi
# STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2404 # - PLATFORM: ubuntu-2404
BLOCK: 1248:length(pkgs) # BLOCK: 1248:length(pkgs)
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2204 # - PLATFORM: ubuntu-2204
BLOCK: 1325:length(pkgs) # BLOCK: 1325:length(pkgs)
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: alpine-320 # - PLATFORM: alpine-320
BLOCK: 1199:length(pkgs) # BLOCK: 1199:length(pkgs)
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
### AMD ### AMD
- PLATFORM: redhat-9 - PLATFORM: redhat-9
BLOCK: 690:length(pkgs) BLOCK: 690:length(pkgs)
@ -48,30 +48,30 @@ matrix:
NCPUS: 2 NCPUS: 2
MEMORY: 29Gi MEMORY: 29Gi
STRATEGY: multisession STRATEGY: multisession
- PLATFORM: ubuntu-2404 # - PLATFORM: ubuntu-2404
BLOCK: 1481:length(pkgs) # BLOCK: 1481:length(pkgs)
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2204 # - PLATFORM: ubuntu-2204
BLOCK: 1542:length(pkgs) # BLOCK: 1542:length(pkgs)
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: alpine-320 # - PLATFORM: alpine-320
BLOCK: 1161:length(pkgs) # BLOCK: 1161:length(pkgs)
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
steps: steps:
- name: 'Warming up' - name: 'Warming up'
image: busybox:latest image: busybox:latest
failure: ignore failure: ignore
commands: commands:
- sleep 200 - sleep 1
- echo "Warmed Up" - echo "Warmed Up"
backend_options: backend_options:
kubernetes: kubernetes:

4
R/archive_packages.R
View file

@ -65,6 +65,10 @@ archive_package <- function(
} }
} }
cli::cli_alert("{.fun archive_package}: Archiving {.field {basename(old_versions)}}, keeping {.field {basename(all_versions[index])}}.") cli::cli_alert("{.fun archive_package}: Archiving {.field {basename(old_versions)}}, keeping {.field {basename(all_versions[index])}}.")
# suppress progressr output here
progressr::handlers("void")
# don't parallelise
future::plan("sequential")
s3fs::s3_file_move(old_versions, sprintf("%s/Archive/%s/%s", remote_bin_dir, .x, basename(old_versions)), overwrite = TRUE) s3fs::s3_file_move(old_versions, sprintf("%s/Archive/%s/%s", remote_bin_dir, .x, basename(old_versions)), overwrite = TRUE)
cli::cli_alert_success("{.fun archive_package}: Successfully archived package {.pkg {.x}}.") cli::cli_alert_success("{.fun archive_package}: Successfully archived package {.pkg {.x}}.")
if (!is.null(pause)) { if (!is.null(pause)) {

10
R/build_binaries.R
View file

@ -16,7 +16,9 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
deps_verbose = FALSE, deps_verbose = FALSE,
debug = FALSE, debug = FALSE,
force = FALSE, force = FALSE,
archive = TRUE) { archive = TRUE,
future_strategy = "multisession",
future_workers = 2) {
cli::cli_h2("Preparations ({.pkg {package_name}})") cli::cli_h2("Preparations ({.pkg {package_name}})")
codename <- set_codename(codename) codename <- set_codename(codename)
@ -81,6 +83,12 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
cli::cli_alert("[{format(Sys.time(), format='%H:%M:%S')}] Building binaries for {.pkg {package_name[[1]]}} with tags {.field {tag}}.") cli::cli_alert("[{format(Sys.time(), format='%H:%M:%S')}] Building binaries for {.pkg {package_name[[1]]}} with tags {.field {tag}}.")
future::plan(future_strategy,
workers = future_workers,
rscript_startup = quote(options(crayon.enabled = TRUE))
)
progressr::handlers("cli")
p <- progressr::progressor(along = tag) p <- progressr::progressor(along = tag)
worker_fun <- function(x, y, p) { worker_fun <- function(x, y, p) {

5
R/upload.R
View file

@ -40,6 +40,11 @@ upload_single_binary_to_s3 <- function(
exists <- s3fs::s3_file_exists(sprintf("%s/%s", remote_bin_path, tarball_name)) exists <- s3fs::s3_file_exists(sprintf("%s/%s", remote_bin_path, tarball_name))
# suppress progressr output here
progressr::handlers("void")
# don't parallelise
future::plan("sequential")
if ((!exists && !force) || (!exists && force)) { if ((!exists && !force) || (!exists && force)) {
cli::cli_alert("{.fun upload_single_binary_to_s3}: Uploading {.pkg {package_name}} {.field {tag}} to {.path {sprintf('%s/%s', remote_bin_path, tarball_name)}}.") cli::cli_alert("{.fun upload_single_binary_to_s3}: Uploading {.pkg {package_name}} {.field {tag}} to {.path {sprintf('%s/%s', remote_bin_path, tarball_name)}}.")
s3fs::s3_file_upload( s3fs::s3_file_upload(