devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 10 Pull requests Releases Activity

build all amd

Browse source
This commit is contained in:
Patrick Schratz 2024-09-13 11:21:39 +02:00
commit adbd8b1ed5
Signed by: pat-s
GPG key ID: 3C6318841EF78925
2 changed files with 128 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

110
.woodpecker/build.yaml
View file

@ -8,42 +8,67 @@ depends_on:
matrix:
include:
- PLATFORM: redhat-9
BLOCK: 20099:21169
ARCH: arm64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: redhat-8
BLOCK: 20099:21169
ARCH: arm64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: ubuntu-2404
BLOCK: 20099:21169
ARCH: arm64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: ubuntu-2204
BLOCK: 20099:21169
ARCH: arm64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
# - PLATFORM: alpine-320
# BLOCK: 7065:length(pkgs)
# - PLATFORM: redhat-9
# BLOCK: 20099:21169
# ARCH: arm64
# NCPUS: 2
# MEMORY: 29Gi
# STRATEGY: multisession
# - PLATFORM: alpine-320
# BLOCK: 1090:length(pkgs)
# ARCH: amd64
# - PLATFORM: redhat-8
# BLOCK: 20099:21169
# ARCH: arm64
# NCPUS: 2
# MEMORY: 29Gi
# STRATEGY: multisession
# - PLATFORM: ubuntu-2404
# BLOCK: 20099:21169
# ARCH: arm64
# NCPUS: 2
# MEMORY: 29Gi
# STRATEGY: multisession
# - PLATFORM: ubuntu-2204
# BLOCK: 20099:21169
# ARCH: arm64
# NCPUS: 2
# MEMORY: 29Gi
# STRATEGY: multisession
### AMD
- PLATFORM: redhat-9
BLOCK: 1:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: redhat-8
BLOCK: 1:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: ubuntu-2404
BLOCK: 1:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: ubuntu-2204
BLOCK: 1:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: alpine-320
BLOCK: 7065:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: alpine-320
BLOCK: 1090:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
steps:
- name: 'Build binaries (${PLATFORM}: ${BLOCK})'
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
@ -71,27 +96,28 @@ steps:
# more complicated system dependencies which pak cannot resolve
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
# set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point
# - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); pkgs = foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
# - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno", "webshot2")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); pkgs = foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
### Manual pkg builds
- xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c("matrixStats", "MazamaLocationUtils", "MBA", "mboost", "MCARtest", "mcb", "mco", "MCPModGeneral", "mdatools", "mdsr", "Mega2R", "metajam", "MethEvolSIM", "microsimulation", "minic", "miniGUI", "minqa", "mirai", "mirt", "mirtCAT", "mispitools", "missRanger", "mobr", "moc.gapbk", "modsem", "MPV", "mrbayes", "MSCquartets", "msm", "mstDIF", "MultBiplotR", "multibias", "multibiplotGUI", "multIntTestFunc", "mvs", "mxfda", "myClim", "nanonext", "ncdf4", "ndtv", "neotoma2", "nestedcv", "NetFACS", "nflplotR", "NGBVS", "nlgm", "NlinTS", "nlme", "nomnoml", "oce", "octopucs", "od", "omopgenerics", "opencpu", "OpenMx", "openssl", "openxlsx2", "optbdmaeAT", "optrcdmaeAT", "ordinal", "ordinalpattern", "outlierensembles", "pairwiseCI", "paramlink2", "party", "partykit", "pastboon", "patchDVI", "patterncausality", "pbdZMQ", "pcaPP", "pdR", "pedtools", "PerFit", "perms", "PFLR", "phonfieldwork", "piecepackr", "PKPDsim", "plfMA", "plotBart", "PMCMRplus", "PNAR", "PointedSDMs", "polmineR", "poweRbal", "PRA", "PRECAST", "prettifyAddins", "preventr", "PriceIndices", "priceR", "prior3D", "priorCON", "PROsetta", "proteomicsCV", "PubChemR", "pubh", "pubmed.mineR", "qcpm", "QTE.RD", "qtkit", "qtl", "qualtRics", "quanteda.textplots", "Racmacs", "rapiclient", "rapidsplithalf", "rassta", "ravetools", "rayrender", "Rbeast", "RblDataLicense", "Rblpapi", "RclusTool", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos", "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "Rcplex", "RcppCWB", "RcppMeCab", "RcppParallel", "readrba", "referenceIntervals", "refineR", "refseqR", "relMix", "resilience", "ResultModelManager", "revdbayes", "rfviz", "RGraphSpace", "riAFTBART", "rio", "rjags", "rjson", "Rlabkey", "rmarchingcubes", "rmcfs", "RmecabKo", "RMySQL", "rnpn", "robotstxt", "robsurvey", "robustbase", "ROI.plugin.cplex", "ROI.plugin.highs", "ROI.plugin.qpoases", "rollup", "rolog", "ROracle", "rplum", "RPointCloud", "Rpoppler", "rrcov", "Rsagacmd", "RSP", "rstpm2", "rswipl", "rtide", "rTwig", "rugarch", "rytstat", "s3fs", "saeHB.panel.beta", "sampcompR", "SAMtool", "scDIFtest", "sched", "secr", "servr", "SGP", "shiny.benchmark", "shinyIRT", "ShinyItemAnalysis", "shinytest2", "SIAmodules", "Signac", "SigTree", "simCAT", "SimDesign", "SimNPH", "simodels", "simplePHENOTYPES", "SingleCaseES", "skedastic", "skpr", "sna", "snplinkage", "SNSequate", "soptdmaeA", "spatstat.explore", "speakeasyR", "Spectran", "spTimer", "SSDforR", "stan4bart", "startupmsg", "statnet", "stats4teaching", "stcpR6", "streamDAG", "strvalidator", "surface", "switchboard", "tabledown", "tables", "tcltk2", "TestAnaAPP", "TextMiningGUI", "tfrmtbuilder", "tiledb", "tkImgR", "tmsens", "TreeDist", "TTAinterfaceTrendAnalysis", "TVMM", "uHMM", "voi", "vvtableau", "wbacon", "webshot2", "ymd", "YTAnalytics"); pkgs = setdiff(pkgs, c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers(global = TRUE); progressr::handlers("cli"); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
# webshot2: wait for https://github.com/rstudio/r-system-requirements/pull/178
- xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c("WH", "ymd"); pkgs = setdiff(pkgs, c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers(global = TRUE); progressr::handlers("cli"); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
backend_options:
kubernetes:
### PROD-parallel
# resources:
# requests:
# memory: 25Gi
# cpu: 12000m
# limits:
# memory: 32Gi
# cpu: 16000m
### PROD-sequential
resources:
requests:
memory: ${MEMORY}
cpu: 2000m
cpu: 12000m
limits:
memory: 29Gi
cpu: 4000m
memory: ${MEMORY}
cpu: 16000m
### PROD-sequential
# resources:
# requests:
# memory: ${MEMORY}
# cpu: 2000m
# limits:
# memory: 29Gi
# cpu: 4000m
# DEBUG
# resources:
# requests:

111
.woodpecker/warm_up_build.yaml
View file

@ -5,42 +5,67 @@ skip_clone: true
matrix:
include:
- PLATFORM: redhat-9
BLOCK: 20099:21169
ARCH: arm64
NCPUS: 2
MEMORY: 28Gi
STRATEGY: multisession
- PLATFORM: redhat-8
BLOCK: 20099:21169
ARCH: arm64
NCPUS: 2
MEMORY: 28Gi
STRATEGY: multisession
- PLATFORM: ubuntu-2404
BLOCK: 20099:21169
ARCH: arm64
NCPUS: 2
MEMORY: 28Gi
STRATEGY: multisession
- PLATFORM: ubuntu-2204
BLOCK: 20099:21169
ARCH: arm64
NCPUS: 2
MEMORY: 28Gi
STRATEGY: multisession
# - PLATFORM: alpine-320
# BLOCK: 7065:length(pkgs)
# - PLATFORM: redhat-9
# BLOCK: 20099:21169
# ARCH: arm64
# NCPUS: 2
# MEMORY: 28Gi
# MEMORY: 29Gi
# STRATEGY: multisession
# - PLATFORM: alpine-320
# BLOCK: 1090:length(pkgs)
# ARCH: amd64
# - PLATFORM: redhat-8
# BLOCK: 20099:21169
# ARCH: arm64
# NCPUS: 2
# MEMORY: 28Gi
# MEMORY: 29Gi
# STRATEGY: multisession
# - PLATFORM: ubuntu-2404
# BLOCK: 20099:21169
# ARCH: arm64
# NCPUS: 2
# MEMORY: 29Gi
# STRATEGY: multisession
# - PLATFORM: ubuntu-2204
# BLOCK: 20099:21169
# ARCH: arm64
# NCPUS: 2
# MEMORY: 29Gi
# STRATEGY: multisession
### AMD
- PLATFORM: redhat-9
BLOCK: 1:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: redhat-8
BLOCK: 1:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: ubuntu-2404
BLOCK: 1:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: ubuntu-2204
BLOCK: 1:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: alpine-320
BLOCK: 7065:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
- PLATFORM: alpine-320
BLOCK: 1090:length(pkgs)
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
STRATEGY: multisession
steps:
- name: 'Warming up'
image: busybox:latest
@ -50,14 +75,30 @@ steps:
- echo "Warmed Up"
backend_options:
kubernetes:
### PROD-sequential
### PROD-parallel
resources:
requests:
memory: ${MEMORY}
cpu: 2000m
cpu: 12000m
limits:
memory: 30Gi
cpu: 4000m
memory: ${MEMORY}
cpu: 16000m
### PROD-sequential
# resources:
# requests:
# memory: ${MEMORY}
# cpu: 2000m
# limits:
# memory: 29Gi
# cpu: 4000m
# DEBUG
# resources:
# requests:
# memory: 4Gi
# cpu: 2000m
# limits:
# memory: 4Gi
# cpu: 2000m
nodeSelector:
kubernetes.io/arch: '${ARCH}'
tolerations: