108 lines
12 KiB
YAML
108 lines
12 KiB
YAML
when:
|
|
event: [manual]
|
|
|
|
skip_clone: true
|
|
|
|
depends_on:
|
|
- warm_up_build
|
|
|
|
matrix:
|
|
include:
|
|
- PLATFORM: redhat-9
|
|
BLOCK: 20099:21169
|
|
ARCH: arm64
|
|
NCPUS: 2
|
|
MEMORY: 28Gi
|
|
STRATEGY: multisession
|
|
- PLATFORM: redhat-8
|
|
BLOCK: 20099:21169
|
|
ARCH: arm64
|
|
NCPUS: 2
|
|
MEMORY: 28Gi
|
|
STRATEGY: multisession
|
|
- PLATFORM: ubuntu-2404
|
|
BLOCK: 20099:21169
|
|
ARCH: arm64
|
|
NCPUS: 2
|
|
MEMORY: 28Gi
|
|
STRATEGY: multisession
|
|
- PLATFORM: ubuntu-2204
|
|
BLOCK: 20099:21169
|
|
ARCH: arm64
|
|
NCPUS: 2
|
|
MEMORY: 28Gi
|
|
STRATEGY: multisession
|
|
# - PLATFORM: alpine-320
|
|
# BLOCK: 7065:length(pkgs)
|
|
# ARCH: arm64
|
|
# NCPUS: 2
|
|
# MEMORY: 28Gi
|
|
# STRATEGY: multisession
|
|
# - PLATFORM: alpine-320
|
|
# BLOCK: 1090:length(pkgs)
|
|
# ARCH: amd64
|
|
# NCPUS: 2
|
|
# MEMORY: 28Gi
|
|
# STRATEGY: multisession
|
|
steps:
|
|
- name: 'Build binaries (${PLATFORM}: ${BLOCK})'
|
|
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
|
|
image: docker.io/devxygmbh/${ARCH}-binaries-r-${PLATFORM}
|
|
pull: true
|
|
# to ensure the PACKAGES upload step is being executed
|
|
failure: ignore
|
|
environment:
|
|
GIT_USER: pat-s
|
|
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
|
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
|
CCACHE_DIR: /mnt/cache/ccache
|
|
volumes:
|
|
- ${ARCH}-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
|
|
commands:
|
|
# download latest package version (avoid having to rebuild image every time)
|
|
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
|
|
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
|
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR
|
|
# for some reason this one is often corrupt on redhat-9
|
|
- rm -rf /mnt/cache/R-pkgs/RPostgres /mnt/cache/pkgcache/R/pkgcache
|
|
- R CMD INSTALL .
|
|
# more complicated system dependencies which pak cannot resolve
|
|
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
|
# set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point
|
|
# - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); pkgs = foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
|
|
### Manual pkg builds
|
|
- R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c("adproclus", "airt", "alcyon", "AlphaSimR", "altmeta", "analogue", "anipaths", "anomaly", "APCalign", "apsimx", "arcgisgeocode", "archivist", "ARDECO", "arima2", "arrow", "arules", "arulesSequences", "asbio", "asciiSetupReader", "ashr", "asremlPlus", "atable", "ATQ", "autoFC", "av", "baorista", "bartCause", "bartMan", "BayesMallows", "bayesnec", "bdvis", "biblio", "bidask", "BifactorIndicesCalculator", "bigDM", "bigsnpr", "bimets", "BioM2", "biostat3", "biplotbootGUI", "bispdep", "biwavelet", "BKTR", "blme", "bnlearn", "boot", "boxcoxmix", "BranchGLM", "BRugs", "bscui", "bsvarSIGNs", "calendar", "CaPO4Sim", "caretEnsemble", "casebase", "cba", "celltrackR", "cffr", "circular", "ClassComparison", "clrng", "clustermq", "CLVTools", "cncaGUI", "collapse", "conrad", "contingencytables", "ConvertPar", "convevol", "CopernicusDEM", "cops", "copula", "corrplot", "CoTiMA", "countSTAR", "coursekata", "crandep", "CRE", "CrossCarry", "csquares", "CTD", "ctgdist", "ctsem", "cusp", "cv", "D3mirt", "DanielBiostatistics10th", "dartR", "dartR.base", "dartR.captive", "dartR.popgen", "dartR.sexlinked", "dartR.sim", "dartR.spatial", "datana", "datardis", "dbarts", "decp", "deepgp", "deeptime", "deeptrafo", "designmatch", "dfidx", "DFIT", "difR", "DIFshiny", "digest", "distr", "distrEllipse", "distrSim", "diveR", "dmm", "DPQmpfr", "DPTM", "drda", "dsmSearch", "duckdbfs", "DWDLargeR", "dyn.log", "dynaTree", "dynBiplotGUI", "dynr", "eaf", "EBcoBART", "ecocbo", "ecocomDP", "EcoEnsemble", "ecoreg", "eddington", "EFA.dimensions", "EMCluster", "emmeans", "EnrichIntersect", "envi", "envvar", "equateIRT", "equateMultiple", "EthSEQ", "evalITR", "EventDetectGUI", "exampletestr", "expm", "extremevalues", "faoutlier", "fastcpd", "fastDummies", "fastICA", "FastUtils", "fBasics", "fcl", "fdasrvf", "fdrtool", "filters", "finbif", "fio", "FjordLight", "flan", "flexsurv", "FLORAL", "flow", "flps", "fluidsynth", "fntl", "FoReco", "fragility", "FuzzyPovertyR", "galamm", "gamstransfer", "GAS", "gasfluxes", "gateR", "geofi", "geosimilarity", "geppe", "gert", "GET", "GetTDData", "GFDrmst", "ggpath", "ggrepel", "ghapps", "giacR", "GLCMTextures", "glmmPen", "glmmrBase", "googleAnalyticsR", "googletraffic", "GPCMlasso", "gpindex", "gpuR", "grates", "grattanInflators", "GRIDCOPULA", "gridGeometry", "groupTesting", "GRShiny", "gsw", "gwid", "gWidgets2tcltk", "GWmodel", "Haplin", "hce", "heteromixgm", "HierPortfolios", "highs", "HMC", "hmstimer", "HSAUR3", "hstats", "htsr", "httr2", "hunspell", "HydroPortailStats", "ibdsim2", "IceSat2R", "ichimoku", "iClusterVB", "icosa", "ilabelled", "ImputeRobust", "incubate", "indicspecies", "iNEXT.3D", "INLAspacetime", "IPEDSuploadables", "iplots", "ipolygrowth", "irtawsi", "irtGUI", "irtpwr", "irtQ", "irtreliability", "IsoriX", "iSTATS", "jagshelper", "jmvcore", "joyn", "jqr", "jrt", "jsmodule", "jstager", "KappaGUI", "kequate", "kernelshap", "kernlab", "kit", "klassR", "KoboconnectR", "KSgeneral", "L1pack", "LBI", "leaflet.extras", "lefko3", "lessR", "lgpr", "lingtypology", "lme4breeding", "loon", "loon.ggplot", "loon.shiny", "loon.tourr", "lordif", "LRTesteR", "MachineShop", "magickGUI", "makepipe", "malariaAtlas", "MALDIquant", "mathml", "matrixStats", "MazamaLocationUtils", "MBA", "mboost", "MCARtest", "mcb", "mco", "MCPModGeneral", "mdatools", "mdsr", "Mega2R", "metajam", "MethEvolSIM", "microsimulation", "minic", "miniGUI", "minqa", "mirai", "mirt", "mirtCAT", "mispitools", "missRanger", "mobr", "moc.gapbk", "modsem", "MPV", "mrbayes", "MSCquartets", "msm", "mstDIF", "MultBiplotR", "multibias", "multibiplotGUI", "multIntTestFunc", "mvs", "mxfda", "myClim", "nanonext", "ncdf4", "ndtv", "neotoma2", "nestedcv", "NetFACS", "nflplotR", "NGBVS", "nlgm", "NlinTS", "nlme", "nomnoml", "oce", "octopucs", "od", "omopgenerics", "opencpu", "OpenMx", "openssl", "openxlsx2", "optbdmaeAT", "optrcdmaeAT", "ordinal", "ordinalpattern", "outlierensembles", "pairwiseCI", "paramlink2", "party", "partykit", "pastboon", "patchDVI", "patterncausality", "pbdZMQ", "pcaPP", "pdR", "pedtools", "PerFit", "perms", "PFLR", "phonfieldwork", "piecepackr", "PKPDsim", "plfMA", "plotBart", "PMCMRplus", "PNAR", "PointedSDMs", "polmineR", "poweRbal", "PRA", "PRECAST", "prettifyAddins", "preventr", "PriceIndices", "priceR", "prior3D", "priorCON", "PROsetta", "proteomicsCV", "PubChemR", "pubh", "pubmed.mineR", "qcpm", "QTE.RD", "qtkit", "qtl", "qualtRics", "quanteda.textplots", "Racmacs", "rapiclient", "rapidsplithalf", "rassta", "ravetools", "rayrender", "Rbeast", "RblDataLicense", "Rblpapi", "RclusTool", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos", "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "Rcplex", "RcppCWB", "RcppMeCab", "RcppParallel", "readrba", "referenceIntervals", "refineR", "refseqR", "relMix", "resilience", "ResultModelManager", "revdbayes", "rfviz", "RGraphSpace", "riAFTBART", "rio", "rjags", "rjson", "Rlabkey", "rmarchingcubes", "rmcfs", "RmecabKo", "RMySQL", "rnpn", "robotstxt", "robsurvey", "robustbase", "ROI.plugin.cplex", "ROI.plugin.highs", "ROI.plugin.qpoases", "rollup", "rolog", "ROracle", "rplum", "RPointCloud", "Rpoppler", "rrcov", "Rsagacmd", "RSP", "rstpm2", "rswipl", "rtide", "rTwig", "rugarch", "rytstat", "s3fs", "saeHB.panel.beta", "sampcompR", "SAMtool", "scDIFtest", "sched", "secr", "servr", "SGP", "shiny.benchmark", "shinyIRT", "ShinyItemAnalysis", "shinytest2", "SIAmodules", "Signac", "SigTree", "simCAT", "SimDesign", "SimNPH", "simodels", "simplePHENOTYPES", "SingleCaseES", "skedastic", "skpr", "sna", "snplinkage", "SNSequate", "soptdmaeA", "spatstat.explore", "speakeasyR", "Spectran", "spTimer", "SSDforR", "stan4bart", "startupmsg", "statnet", "stats4teaching", "stcpR6", "streamDAG", "strvalidator", "surface", "switchboard", "tabledown", "tables", "tcltk2", "TestAnaAPP", "TextMiningGUI", "tfrmtbuilder", "tiledb", "tkImgR", "tmsens", "TreeDist", "TTAinterfaceTrendAnalysis", "TVMM", "uHMM", "voi", "vvtableau", "wbacon", "webshot2", "ymd", "YTAnalytics"); pkgs = setdiff(pkgs, c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); pkgs = foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
|
|
backend_options:
|
|
kubernetes:
|
|
### PROD-parallel
|
|
# resources:
|
|
# requests:
|
|
# memory: 25Gi
|
|
# cpu: 12000m
|
|
# limits:
|
|
# memory: 32Gi
|
|
# cpu: 16000m
|
|
### PROD-sequential
|
|
resources:
|
|
requests:
|
|
memory: ${MEMORY}
|
|
cpu: 2000m
|
|
limits:
|
|
memory: 30Gi
|
|
cpu: 4000m
|
|
# DEBUG
|
|
# resources:
|
|
# requests:
|
|
# memory: 4Gi
|
|
# cpu: 2000m
|
|
# limits:
|
|
# memory: 4Gi
|
|
# cpu: 2000m
|
|
nodeSelector:
|
|
kubernetes.io/arch: "${ARCH}"
|
|
tolerations:
|
|
- key: "CI"
|
|
operator: "Equal"
|
|
value: "true"
|
|
effect: "NoSchedule"
|