diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index c1f2b45..e8c50d1 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -20,27 +20,27 @@ matrix: # NCPUS: 2 # MEMORY: 29Gi # STRATEGY: multisession - - PLATFORM: ubuntu-2404 - BLOCK: 1248:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - BLOCK: 1325:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: alpine-320 - BLOCK: 1199:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession + # - PLATFORM: ubuntu-2404 + # BLOCK: 1248:length(pkgs) + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2204 + # BLOCK: 1325:length(pkgs) + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: alpine-320 + # BLOCK: 1199:length(pkgs) + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession ### AMD - PLATFORM: redhat-9 - BLOCK: 690:length(pkgs) + BLOCK: 699:length(pkgs) ARCH: amd64 NCPUS: 2 MEMORY: 29Gi @@ -51,24 +51,24 @@ matrix: NCPUS: 2 MEMORY: 29Gi STRATEGY: multisession - - PLATFORM: ubuntu-2404 - BLOCK: 1481:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - BLOCK: 1542:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: alpine-320 - BLOCK: 1161:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession + # - PLATFORM: ubuntu-2404 + # BLOCK: 1481:length(pkgs) + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2204 + # BLOCK: 1542:length(pkgs) + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: alpine-320 + # BLOCK: 1161:length(pkgs) + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession steps: - name: 'Build binaries (${PLATFORM}: ${BLOCK})' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] @@ -132,7 +132,7 @@ steps: # tfrmtbuilder -> V8 (alpine) # Rcmdr (alpine) # readstata13 (alpine) - # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers(global = TRUE); progressr::handlers("cli"); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' backend_options: kubernetes: ### PROD-parallel diff --git a/.woodpecker/warm_up_build.yaml b/.woodpecker/warm_up_build.yaml index 98ad159..4bd25ed 100644 --- a/.woodpecker/warm_up_build.yaml +++ b/.woodpecker/warm_up_build.yaml @@ -17,24 +17,24 @@ matrix: # NCPUS: 2 # MEMORY: 29Gi # STRATEGY: multisession - - PLATFORM: ubuntu-2404 - BLOCK: 1248:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - BLOCK: 1325:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: alpine-320 - BLOCK: 1199:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession + # - PLATFORM: ubuntu-2404 + # BLOCK: 1248:length(pkgs) + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2204 + # BLOCK: 1325:length(pkgs) + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: alpine-320 + # BLOCK: 1199:length(pkgs) + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession ### AMD - PLATFORM: redhat-9 BLOCK: 690:length(pkgs) @@ -48,30 +48,30 @@ matrix: NCPUS: 2 MEMORY: 29Gi STRATEGY: multisession - - PLATFORM: ubuntu-2404 - BLOCK: 1481:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - BLOCK: 1542:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: alpine-320 - BLOCK: 1161:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession + # - PLATFORM: ubuntu-2404 + # BLOCK: 1481:length(pkgs) + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2204 + # BLOCK: 1542:length(pkgs) + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: alpine-320 + # BLOCK: 1161:length(pkgs) + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession steps: - name: 'Warming up' image: busybox:latest failure: ignore commands: - - sleep 200 + - sleep 1 - echo "Warmed Up" backend_options: kubernetes: diff --git a/R/archive_packages.R b/R/archive_packages.R index 40c9f00..ce3d47f 100644 --- a/R/archive_packages.R +++ b/R/archive_packages.R @@ -65,6 +65,10 @@ archive_package <- function( } } cli::cli_alert("{.fun archive_package}: Archiving {.field {basename(old_versions)}}, keeping {.field {basename(all_versions[index])}}.") + # suppress progressr output here + progressr::handlers("void") + # don't parallelise + future::plan("sequential") s3fs::s3_file_move(old_versions, sprintf("%s/Archive/%s/%s", remote_bin_dir, .x, basename(old_versions)), overwrite = TRUE) cli::cli_alert_success("{.fun archive_package}: Successfully archived package {.pkg {.x}}.") if (!is.null(pause)) { diff --git a/R/build_binaries.R b/R/build_binaries.R index 1426665..3b8f8d1 100644 --- a/R/build_binaries.R +++ b/R/build_binaries.R @@ -16,7 +16,9 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, deps_verbose = FALSE, debug = FALSE, force = FALSE, - archive = TRUE) { + archive = TRUE, + future_strategy = "multisession", + future_workers = 2) { cli::cli_h2("Preparations ({.pkg {package_name}})") codename <- set_codename(codename) @@ -81,6 +83,12 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, cli::cli_alert("[{format(Sys.time(), format='%H:%M:%S')}] Building binaries for {.pkg {package_name[[1]]}} with tags {.field {tag}}.") + future::plan(future_strategy, + workers = future_workers, + rscript_startup = quote(options(crayon.enabled = TRUE)) + ) + + progressr::handlers("cli") p <- progressr::progressor(along = tag) worker_fun <- function(x, y, p) { diff --git a/R/upload.R b/R/upload.R index fc96f75..75c23f6 100644 --- a/R/upload.R +++ b/R/upload.R @@ -40,6 +40,11 @@ upload_single_binary_to_s3 <- function( exists <- s3fs::s3_file_exists(sprintf("%s/%s", remote_bin_path, tarball_name)) + # suppress progressr output here + progressr::handlers("void") + # don't parallelise + future::plan("sequential") + if ((!exists && !force) || (!exists && force)) { cli::cli_alert("{.fun upload_single_binary_to_s3}: Uploading {.pkg {package_name}} {.field {tag}} to {.path {sprintf('%s/%s', remote_bin_path, tarball_name)}}.") s3fs::s3_file_upload(