From adbd8b1ed54fe50323f818403f1b473be23f962a Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 13 Sep 2024 11:21:39 +0200 Subject: [PATCH] build all amd --- .woodpecker/build.yaml | 110 +++++++++++++++++++------------- .woodpecker/warm_up_build.yaml | 111 ++++++++++++++++++++++----------- 2 files changed, 144 insertions(+), 77 deletions(-) diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index 76a954c..f67221d 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -8,42 +8,67 @@ depends_on: matrix: include: - - PLATFORM: redhat-9 - BLOCK: 20099:21169 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: redhat-8 - BLOCK: 20099:21169 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2404 - BLOCK: 20099:21169 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - BLOCK: 20099:21169 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - # - PLATFORM: alpine-320 - # BLOCK: 7065:length(pkgs) + # - PLATFORM: redhat-9 + # BLOCK: 20099:21169 # ARCH: arm64 # NCPUS: 2 # MEMORY: 29Gi # STRATEGY: multisession - # - PLATFORM: alpine-320 - # BLOCK: 1090:length(pkgs) - # ARCH: amd64 + # - PLATFORM: redhat-8 + # BLOCK: 20099:21169 + # ARCH: arm64 # NCPUS: 2 # MEMORY: 29Gi # STRATEGY: multisession + # - PLATFORM: ubuntu-2404 + # BLOCK: 20099:21169 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2204 + # BLOCK: 20099:21169 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + ### AMD + - PLATFORM: redhat-9 + BLOCK: 1:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + - PLATFORM: redhat-8 + BLOCK: 1:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + - PLATFORM: ubuntu-2404 + BLOCK: 1:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + - PLATFORM: ubuntu-2204 + BLOCK: 1:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + - PLATFORM: alpine-320 + BLOCK: 7065:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + - PLATFORM: alpine-320 + BLOCK: 1090:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession steps: - name: 'Build binaries (${PLATFORM}: ${BLOCK})' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] @@ -71,27 +96,28 @@ steps: # more complicated system dependencies which pak cannot resolve - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point - # - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); pkgs = foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' + # - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno", "webshot2")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); pkgs = foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' ### Manual pkg builds - - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c("matrixStats", "MazamaLocationUtils", "MBA", "mboost", "MCARtest", "mcb", "mco", "MCPModGeneral", "mdatools", "mdsr", "Mega2R", "metajam", "MethEvolSIM", "microsimulation", "minic", "miniGUI", "minqa", "mirai", "mirt", "mirtCAT", "mispitools", "missRanger", "mobr", "moc.gapbk", "modsem", "MPV", "mrbayes", "MSCquartets", "msm", "mstDIF", "MultBiplotR", "multibias", "multibiplotGUI", "multIntTestFunc", "mvs", "mxfda", "myClim", "nanonext", "ncdf4", "ndtv", "neotoma2", "nestedcv", "NetFACS", "nflplotR", "NGBVS", "nlgm", "NlinTS", "nlme", "nomnoml", "oce", "octopucs", "od", "omopgenerics", "opencpu", "OpenMx", "openssl", "openxlsx2", "optbdmaeAT", "optrcdmaeAT", "ordinal", "ordinalpattern", "outlierensembles", "pairwiseCI", "paramlink2", "party", "partykit", "pastboon", "patchDVI", "patterncausality", "pbdZMQ", "pcaPP", "pdR", "pedtools", "PerFit", "perms", "PFLR", "phonfieldwork", "piecepackr", "PKPDsim", "plfMA", "plotBart", "PMCMRplus", "PNAR", "PointedSDMs", "polmineR", "poweRbal", "PRA", "PRECAST", "prettifyAddins", "preventr", "PriceIndices", "priceR", "prior3D", "priorCON", "PROsetta", "proteomicsCV", "PubChemR", "pubh", "pubmed.mineR", "qcpm", "QTE.RD", "qtkit", "qtl", "qualtRics", "quanteda.textplots", "Racmacs", "rapiclient", "rapidsplithalf", "rassta", "ravetools", "rayrender", "Rbeast", "RblDataLicense", "Rblpapi", "RclusTool", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos", "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "Rcplex", "RcppCWB", "RcppMeCab", "RcppParallel", "readrba", "referenceIntervals", "refineR", "refseqR", "relMix", "resilience", "ResultModelManager", "revdbayes", "rfviz", "RGraphSpace", "riAFTBART", "rio", "rjags", "rjson", "Rlabkey", "rmarchingcubes", "rmcfs", "RmecabKo", "RMySQL", "rnpn", "robotstxt", "robsurvey", "robustbase", "ROI.plugin.cplex", "ROI.plugin.highs", "ROI.plugin.qpoases", "rollup", "rolog", "ROracle", "rplum", "RPointCloud", "Rpoppler", "rrcov", "Rsagacmd", "RSP", "rstpm2", "rswipl", "rtide", "rTwig", "rugarch", "rytstat", "s3fs", "saeHB.panel.beta", "sampcompR", "SAMtool", "scDIFtest", "sched", "secr", "servr", "SGP", "shiny.benchmark", "shinyIRT", "ShinyItemAnalysis", "shinytest2", "SIAmodules", "Signac", "SigTree", "simCAT", "SimDesign", "SimNPH", "simodels", "simplePHENOTYPES", "SingleCaseES", "skedastic", "skpr", "sna", "snplinkage", "SNSequate", "soptdmaeA", "spatstat.explore", "speakeasyR", "Spectran", "spTimer", "SSDforR", "stan4bart", "startupmsg", "statnet", "stats4teaching", "stcpR6", "streamDAG", "strvalidator", "surface", "switchboard", "tabledown", "tables", "tcltk2", "TestAnaAPP", "TextMiningGUI", "tfrmtbuilder", "tiledb", "tkImgR", "tmsens", "TreeDist", "TTAinterfaceTrendAnalysis", "TVMM", "uHMM", "voi", "vvtableau", "wbacon", "webshot2", "ymd", "YTAnalytics"); pkgs = setdiff(pkgs, c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers(global = TRUE); progressr::handlers("cli"); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' + # webshot2: wait for https://github.com/rstudio/r-system-requirements/pull/178 + - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c("WH", "ymd"); pkgs = setdiff(pkgs, c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers(global = TRUE); progressr::handlers("cli"); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' backend_options: kubernetes: ### PROD-parallel - # resources: - # requests: - # memory: 25Gi - # cpu: 12000m - # limits: - # memory: 32Gi - # cpu: 16000m - ### PROD-sequential resources: requests: memory: ${MEMORY} - cpu: 2000m + cpu: 12000m limits: - memory: 29Gi - cpu: 4000m + memory: ${MEMORY} + cpu: 16000m + ### PROD-sequential + # resources: + # requests: + # memory: ${MEMORY} + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m # DEBUG # resources: # requests: diff --git a/.woodpecker/warm_up_build.yaml b/.woodpecker/warm_up_build.yaml index 2a6b876..15cc8a1 100644 --- a/.woodpecker/warm_up_build.yaml +++ b/.woodpecker/warm_up_build.yaml @@ -5,42 +5,67 @@ skip_clone: true matrix: include: - - PLATFORM: redhat-9 - BLOCK: 20099:21169 - ARCH: arm64 - NCPUS: 2 - MEMORY: 28Gi - STRATEGY: multisession - - PLATFORM: redhat-8 - BLOCK: 20099:21169 - ARCH: arm64 - NCPUS: 2 - MEMORY: 28Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2404 - BLOCK: 20099:21169 - ARCH: arm64 - NCPUS: 2 - MEMORY: 28Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - BLOCK: 20099:21169 - ARCH: arm64 - NCPUS: 2 - MEMORY: 28Gi - STRATEGY: multisession - # - PLATFORM: alpine-320 - # BLOCK: 7065:length(pkgs) + # - PLATFORM: redhat-9 + # BLOCK: 20099:21169 # ARCH: arm64 # NCPUS: 2 - # MEMORY: 28Gi + # MEMORY: 29Gi # STRATEGY: multisession - # - PLATFORM: alpine-320 - # BLOCK: 1090:length(pkgs) - # ARCH: amd64 + # - PLATFORM: redhat-8 + # BLOCK: 20099:21169 + # ARCH: arm64 # NCPUS: 2 - # MEMORY: 28Gi + # MEMORY: 29Gi # STRATEGY: multisession + # - PLATFORM: ubuntu-2404 + # BLOCK: 20099:21169 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2204 + # BLOCK: 20099:21169 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + ### AMD + - PLATFORM: redhat-9 + BLOCK: 1:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + - PLATFORM: redhat-8 + BLOCK: 1:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + - PLATFORM: ubuntu-2404 + BLOCK: 1:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + - PLATFORM: ubuntu-2204 + BLOCK: 1:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + - PLATFORM: alpine-320 + BLOCK: 7065:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + - PLATFORM: alpine-320 + BLOCK: 1090:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession steps: - name: 'Warming up' image: busybox:latest @@ -50,14 +75,30 @@ steps: - echo "Warmed Up" backend_options: kubernetes: - ### PROD-sequential + ### PROD-parallel resources: requests: memory: ${MEMORY} - cpu: 2000m + cpu: 12000m limits: - memory: 30Gi - cpu: 4000m + memory: ${MEMORY} + cpu: 16000m + ### PROD-sequential + # resources: + # requests: + # memory: ${MEMORY} + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m nodeSelector: kubernetes.io/arch: '${ARCH}' tolerations: