only parallelise package builds

2024-09-23 13:25:21 +02:00 · 2024-09-23 13:25:21 +02:00 · 6db7443844
commit 6db7443844
parent 25c493d5b9
2 changed files with 93 additions and 76 deletions
--- a/.woodpecker/build.yaml
+++ b/.woodpecker/build.yaml
@ -20,27 +20,27 @@ matrix:
    #   NCPUS: 2
    #   MEMORY: 29Gi
    #   STRATEGY: multisession
-    - PLATFORM: ubuntu-2404
-      BLOCK: 1248:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: ubuntu-2204
-      BLOCK: 1325:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: alpine-320
-      BLOCK: 1199:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
+    # - PLATFORM: ubuntu-2404
+    #   BLOCK: 1248:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: ubuntu-2204
+    #   BLOCK: 1325:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: alpine-320
+    #   BLOCK: 1199:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
    ### AMD
    - PLATFORM: redhat-9
-      BLOCK: 690:length(pkgs)
+      BLOCK: 699:length(pkgs)
      ARCH: amd64
      NCPUS: 2
      MEMORY: 29Gi
@ -51,24 +51,24 @@ matrix:
      NCPUS: 2
      MEMORY: 29Gi
      STRATEGY: multisession
-    - PLATFORM: ubuntu-2404
-      BLOCK: 1481:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: ubuntu-2204
-      BLOCK: 1542:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: alpine-320
-      BLOCK: 1161:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
+    # - PLATFORM: ubuntu-2404
+    #   BLOCK: 1481:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: ubuntu-2204
+    #   BLOCK: 1542:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: alpine-320
+    #   BLOCK: 1161:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
 steps:
  - name: 'Build binaries (${PLATFORM}: ${BLOCK})'
    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
@ -132,7 +132,7 @@ steps:
      # tfrmtbuilder -> V8 (alpine)
      # Rcmdr (alpine)
      # readstata13 (alpine)
-      # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers(global = TRUE); progressr::handlers("cli"); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
+      # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
    backend_options:
      kubernetes:
        ### PROD-parallel
--- a/.woodpecker/warm_up_build.yaml
+++ b/.woodpecker/warm_up_build.yaml
@ -17,24 +17,24 @@ matrix:
    #   NCPUS: 2
    #   MEMORY: 29Gi
    #   STRATEGY: multisession
-    - PLATFORM: ubuntu-2404
-      BLOCK: 1248:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: ubuntu-2204
-      BLOCK: 1325:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: alpine-320
-      BLOCK: 1199:length(pkgs)
-      ARCH: arm64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
+    # - PLATFORM: ubuntu-2404
+    #   BLOCK: 1248:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: ubuntu-2204
+    #   BLOCK: 1325:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: alpine-320
+    #   BLOCK: 1199:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
    ### AMD
    - PLATFORM: redhat-9
      BLOCK: 690:length(pkgs)
@ -48,30 +48,30 @@ matrix:
      NCPUS: 2
      MEMORY: 29Gi
      STRATEGY: multisession
-    - PLATFORM: ubuntu-2404
-      BLOCK: 1481:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: ubuntu-2204
-      BLOCK: 1542:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
-    - PLATFORM: alpine-320
-      BLOCK: 1161:length(pkgs)
-      ARCH: amd64
-      NCPUS: 2
-      MEMORY: 29Gi
-      STRATEGY: multisession
+    # - PLATFORM: ubuntu-2404
+    #   BLOCK: 1481:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: ubuntu-2204
+    #   BLOCK: 1542:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
+    # - PLATFORM: alpine-320
+    #   BLOCK: 1161:length(pkgs)
+    #   ARCH: amd64
+    #   NCPUS: 2
+    #   MEMORY: 29Gi
+    #   STRATEGY: multisession
 steps:
  - name: 'Warming up'
    image: busybox:latest
    failure: ignore
    commands:
-      - sleep 200
+      - sleep 1
      - echo "Warmed Up"
    backend_options:
      kubernetes: