only parallelise package builds

2024-09-23 13:25:21 +02:00 · 2024-09-23 13:25:21 +02:00 · 6db7443844
commit 6db7443844
parent 25c493d5b9
5 changed files with 93 additions and 76 deletions
--- a/.woodpecker/build.yaml
+++ b/.woodpecker/build.yaml
@ -20,27 +20,27 @@ matrix:
    #   NCPUS: 2
    #   MEMORY: 29Gi
    #   STRATEGY: multisession
-    - PLATFORM: ubuntu-2404
-      BLOCK: 1248:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: ubuntu-2204
-      BLOCK: 1325:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: alpine-320
-      BLOCK: 1199:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
+    # - PLATFORM: ubuntu-2404
+    #   BLOCK: 1248:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: ubuntu-2204
+    #   BLOCK: 1325:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: alpine-320
+    #   BLOCK: 1199:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
    ### AMD
    - PLATFORM: redhat-9
-      BLOCK: 690:length(pkgs)
+      BLOCK: 699:length(pkgs)
      ARCH: amd64
      NCPUS: 2
      MEMORY: 29Gi
@ -51,24 +51,24 @@ matrix:
      NCPUS: 2
      MEMORY: 29Gi
      STRATEGY: multisession
-    - PLATFORM: ubuntu-2404
-      BLOCK: 1481:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: ubuntu-2204
-      BLOCK: 1542:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: alpine-320
-      BLOCK: 1161:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
+    # - PLATFORM: ubuntu-2404
+    #   BLOCK: 1481:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: ubuntu-2204
+    #   BLOCK: 1542:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: alpine-320
+    #   BLOCK: 1161:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
 steps:
  - name: 'Build binaries (${PLATFORM}: ${BLOCK})'
    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
@ -132,7 +132,7 @@ steps:
      # tfrmtbuilder -> V8 (alpine)
      # Rcmdr (alpine)
      # readstata13 (alpine)
-      # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers(global = TRUE); progressr::handlers("cli"); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
+      # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
    backend_options:
      kubernetes:
        ### PROD-parallel
--- a/.woodpecker/warm_up_build.yaml
+++ b/.woodpecker/warm_up_build.yaml
@ -17,24 +17,24 @@ matrix:
    #   NCPUS: 2
    #   MEMORY: 29Gi
    #   STRATEGY: multisession
-    - PLATFORM: ubuntu-2404
-      BLOCK: 1248:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: ubuntu-2204
-      BLOCK: 1325:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: alpine-320
-      BLOCK: 1199:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
+    # - PLATFORM: ubuntu-2404
+    #   BLOCK: 1248:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: ubuntu-2204
+    #   BLOCK: 1325:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: alpine-320
+    #   BLOCK: 1199:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
    ### AMD
    - PLATFORM: redhat-9
      BLOCK: 690:length(pkgs)
@ -48,30 +48,30 @@ matrix:
      NCPUS: 2
      MEMORY: 29Gi
      STRATEGY: multisession
-    - PLATFORM: ubuntu-2404
-      BLOCK: 1481:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: ubuntu-2204
-      BLOCK: 1542:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: alpine-320
-      BLOCK: 1161:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
+    # - PLATFORM: ubuntu-2404
+    #   BLOCK: 1481:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: ubuntu-2204
+    #   BLOCK: 1542:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: alpine-320
+    #   BLOCK: 1161:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
 steps:
  - name: 'Warming up'
    image: busybox:latest
    failure: ignore
    commands:
-      - sleep 200
+      - sleep 1
      - echo "Warmed Up"
    backend_options:
      kubernetes:
--- a/R/archive_packages.R
+++ b/R/archive_packages.R
@ -65,6 +65,10 @@ archive_package <- function(
        }
      }
      cli::cli_alert("{.fun archive_package}: Archiving {.field {basename(old_versions)}}, keeping {.field {basename(all_versions[index])}}.")
+      # suppress progressr output here
+      progressr::handlers("void")
+      # don't parallelise
+      future::plan("sequential")
      s3fs::s3_file_move(old_versions, sprintf("%s/Archive/%s/%s", remote_bin_dir, .x, basename(old_versions)), overwrite = TRUE)
      cli::cli_alert_success("{.fun archive_package}: Successfully archived package {.pkg {.x}}.")
      if (!is.null(pause)) {
--- a/R/build_binaries.R
+++ b/R/build_binaries.R
@ -16,7 +16,9 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
                                 deps_verbose = FALSE,
                                 debug = FALSE,
                                 force = FALSE,
-                                 archive = TRUE) {
+                                 archive = TRUE,
+                                 future_strategy = "multisession",
+                                 future_workers = 2) {
  cli::cli_h2("Preparations ({.pkg {package_name}})")
  codename <- set_codename(codename)

@ -81,6 +83,12 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,

  cli::cli_alert("[{format(Sys.time(),  format='%H:%M:%S')}] Building binaries for {.pkg {package_name[[1]]}} with tags {.field {tag}}.")

+  future::plan(future_strategy,
+    workers = future_workers,
+    rscript_startup = quote(options(crayon.enabled = TRUE))
+  )
+
+  progressr::handlers("cli")
  p <- progressr::progressor(along = tag)

  worker_fun <- function(x, y, p) {
--- a/R/upload.R
+++ b/R/upload.R
@ -40,6 +40,11 @@ upload_single_binary_to_s3 <- function(

  exists <- s3fs::s3_file_exists(sprintf("%s/%s", remote_bin_path, tarball_name))

+  # suppress progressr output here
+  progressr::handlers("void")
+  # don't parallelise
+  future::plan("sequential")
+
  if ((!exists && !force) || (!exists && force)) {
    cli::cli_alert("{.fun upload_single_binary_to_s3}: Uploading {.pkg {package_name}} {.field {tag}} to {.path {sprintf('%s/%s', remote_bin_path, tarball_name)}}.")
    s3fs::s3_file_upload(