From d5a739b1d9ed78dbfab10741c0089052cc44ecfd Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 8 Sep 2024 18:02:29 +0200 Subject: [PATCH] rebuild some pkgs --- .woodpecker/build.yaml | 54 ++++++++++++++++++++-------------- .woodpecker/warm_up_build.yaml | 50 +++++++++++++++++++------------ 2 files changed, 63 insertions(+), 41 deletions(-) diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index 4008128..9d96be8 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -8,30 +8,42 @@ depends_on: matrix: include: - # - PLATFORM: redhat-9 - # BLOCK: 20099:21169 - # ARCH: arm64 - # - PLATFORM: redhat-8 - # BLOCK: 20099:21169 - # ARCH: arm64 - # - PLATFORM: ubuntu-2404 - # BLOCK: 20099:21169 - # ARCH: arm64 - # - PLATFORM: ubuntu-2204 - # BLOCK: 20099:21169 - # ARCH: arm64 - - PLATFORM: alpine-320 - BLOCK: 7065:length(pkgs) + - PLATFORM: redhat-9 + BLOCK: 20099:21169 ARCH: arm64 NCPUS: 2 MEMORY: 28Gi STRATEGY: multisession - - PLATFORM: alpine-320 - BLOCK: 1090:length(pkgs) - ARCH: amd64 + - PLATFORM: redhat-8 + BLOCK: 20099:21169 + ARCH: arm64 NCPUS: 2 MEMORY: 28Gi STRATEGY: multisession + - PLATFORM: ubuntu-2404 + BLOCK: 20099:21169 + ARCH: arm64 + NCPUS: 2 + MEMORY: 28Gi + STRATEGY: multisession + - PLATFORM: ubuntu-2204 + BLOCK: 20099:21169 + ARCH: arm64 + NCPUS: 2 + MEMORY: 28Gi + STRATEGY: multisession + # - PLATFORM: alpine-320 + # BLOCK: 7065:length(pkgs) + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 28Gi + # STRATEGY: multisession + # - PLATFORM: alpine-320 + # BLOCK: 1090:length(pkgs) + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 28Gi + # STRATEGY: multisession steps: - name: 'Build binaries (${PLATFORM}: ${BLOCK})' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] @@ -55,14 +67,12 @@ steps: # for some reason this one is often corrupt on redhat-9 - rm -rf /mnt/cache/R-pkgs/RPostgres /mnt/cache/pkgcache/R/pkgcache - R CMD INSTALL . - - mkdir -p /root/.R && /bin/bash -c 'echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" >> /root/.R/Makevars' - # manually install some packages into the cache so that some builds don't fail - # Hmisc -> ABCanalysis - # - R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))' # more complicated system dependencies which pak cannot resolve - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point - - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' + # - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); pkgs = foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' + ### Manual pkg builds + - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c("adproclus", "airt", "alcyon", "AlphaSimR", "altmeta", "analogue", "anipaths", "anomaly", "APCalign", "apsimx", "arcgisgeocode", "archivist", "ARDECO", "arima2", "arrow", "arules", "arulesSequences", "asbio", "asciiSetupReader", "ashr", "asremlPlus", "atable", "ATQ", "autoFC", "av", "baorista", "bartCause", "bartMan", "BayesMallows", "bayesnec", "bdvis", "biblio", "bidask", "BifactorIndicesCalculator", "bigDM", "bigsnpr", "bimets", "BioM2", "biostat3", "biplotbootGUI", "bispdep", "biwavelet", "BKTR", "blme", "bnlearn", "boot", "boxcoxmix", "BranchGLM", "BRugs", "bscui", "bsvarSIGNs", "calendar", "CaPO4Sim", "caretEnsemble", "casebase", "cba", "celltrackR", "cffr", "circular", "ClassComparison", "clrng", "clustermq", "CLVTools", "cncaGUI", "collapse", "conrad", "contingencytables", "ConvertPar", "convevol", "CopernicusDEM", "cops", "copula", "corrplot", "CoTiMA", "countSTAR", "coursekata", "crandep", "CRE", "CrossCarry", "csquares", "CTD", "ctgdist", "ctsem", "cusp", "cv", "D3mirt", "DanielBiostatistics10th", "dartR", "dartR.base", "dartR.captive", "dartR.popgen", "dartR.sexlinked", "dartR.sim", "dartR.spatial", "datana", "datardis", "dbarts", "decp", "deepgp", "deeptime", "deeptrafo", "designmatch", "dfidx", "DFIT", "difR", "DIFshiny", "digest", "distr", "distrEllipse", "distrSim", "diveR", "dmm", "DPQmpfr", "DPTM", "drda", "dsmSearch", "duckdbfs", "DWDLargeR", "dyn.log", "dynaTree", "dynBiplotGUI", "dynr", "eaf", "EBcoBART", "ecocbo", "ecocomDP", "EcoEnsemble", "ecoreg", "eddington", "EFA.dimensions", "EMCluster", "emmeans", "EnrichIntersect", "envi", "envvar", "equateIRT", "equateMultiple", "EthSEQ", "evalITR", "EventDetectGUI", "exampletestr", "expm", "extremevalues", "faoutlier", "fastcpd", "fastDummies", "fastICA", "FastUtils", "fBasics", "fcl", "fdasrvf", "fdrtool", "filters", "finbif", "fio", "FjordLight", "flan", "flexsurv", "FLORAL", "flow", "flps", "fluidsynth", "fntl", "FoReco", "fragility", "FuzzyPovertyR", "galamm", "gamstransfer", "GAS", "gasfluxes", "gateR", "geofi", "geosimilarity", "geppe", "gert", "GET", "GetTDData", "GFDrmst", "ggpath", "ggrepel", "ghapps", "giacR", "GLCMTextures", "glmmPen", "glmmrBase", "googleAnalyticsR", "googletraffic", "GPCMlasso", "gpindex", "gpuR", "grates", "grattanInflators", "GRIDCOPULA", "gridGeometry", "groupTesting", "GRShiny", "gsw", "gwid", "gWidgets2tcltk", "GWmodel", "Haplin", "hce", "heteromixgm", "HierPortfolios", "highs", "HMC", "hmstimer", "HSAUR3", "hstats", "htsr", "httr2", "hunspell", "HydroPortailStats", "ibdsim2", "IceSat2R", "ichimoku", "iClusterVB", "icosa", "ilabelled", "ImputeRobust", "incubate", "indicspecies", "iNEXT.3D", "INLAspacetime", "IPEDSuploadables", "iplots", "ipolygrowth", "irtawsi", "irtGUI", "irtpwr", "irtQ", "irtreliability", "IsoriX", "iSTATS", "jagshelper", "jmvcore", "joyn", "jqr", "jrt", "jsmodule", "jstager", "KappaGUI", "kequate", "kernelshap", "kernlab", "kit", "klassR", "KoboconnectR", "KSgeneral", "L1pack", "LBI", "leaflet.extras", "lefko3", "lessR", "lgpr", "lingtypology", "lme4breeding", "loon", "loon.ggplot", "loon.shiny", "loon.tourr", "lordif", "LRTesteR", "MachineShop", "magickGUI", "makepipe", "malariaAtlas", "MALDIquant", "mathml", "matrixStats", "MazamaLocationUtils", "MBA", "mboost", "MCARtest", "mcb", "mco", "MCPModGeneral", "mdatools", "mdsr", "Mega2R", "metajam", "MethEvolSIM", "microsimulation", "minic", "miniGUI", "minqa", "mirai", "mirt", "mirtCAT", "mispitools", "missRanger", "mobr", "moc.gapbk", "modsem", "MPV", "mrbayes", "MSCquartets", "msm", "mstDIF", "MultBiplotR", "multibias", "multibiplotGUI", "multIntTestFunc", "mvs", "mxfda", "myClim", "nanonext", "ncdf4", "ndtv", "neotoma2", "nestedcv", "NetFACS", "nflplotR", "NGBVS", "nlgm", "NlinTS", "nlme", "nomnoml", "oce", "octopucs", "od", "omopgenerics", "opencpu", "OpenMx", "openssl", "openxlsx2", "optbdmaeAT", "optrcdmaeAT", "ordinal", "ordinalpattern", "outlierensembles", "pairwiseCI", "paramlink2", "party", "partykit", "pastboon", "patchDVI", "patterncausality", "pbdZMQ", "pcaPP", "pdR", "pedtools", "PerFit", "perms", "PFLR", "phonfieldwork", "piecepackr", "PKPDsim", "plfMA", "plotBart", "PMCMRplus", "PNAR", "PointedSDMs", "polmineR", "poweRbal", "PRA", "PRECAST", "prettifyAddins", "preventr", "PriceIndices", "priceR", "prior3D", "priorCON", "PROsetta", "proteomicsCV", "PubChemR", "pubh", "pubmed.mineR", "qcpm", "QTE.RD", "qtkit", "qtl", "qualtRics", "quanteda.textplots", "Racmacs", "rapiclient", "rapidsplithalf", "rassta", "ravetools", "rayrender", "Rbeast", "RblDataLicense", "Rblpapi", "RclusTool", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos", "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "Rcplex", "RcppCWB", "RcppMeCab", "RcppParallel", "readrba", "referenceIntervals", "refineR", "refseqR", "relMix", "resilience", "ResultModelManager", "revdbayes", "rfviz", "RGraphSpace", "riAFTBART", "rio", "rjags", "rjson", "Rlabkey", "rmarchingcubes", "rmcfs", "RmecabKo", "RMySQL", "rnpn", "robotstxt", "robsurvey", "robustbase", "ROI.plugin.cplex", "ROI.plugin.highs", "ROI.plugin.qpoases", "rollup", "rolog", "ROracle", "rplum", "RPointCloud", "Rpoppler", "rrcov", "Rsagacmd", "RSP", "rstpm2", "rswipl", "rtide", "rTwig", "rugarch", "rytstat", "s3fs", "saeHB.panel.beta", "sampcompR", "SAMtool", "scDIFtest", "sched", "secr", "servr", "SGP", "shiny.benchmark", "shinyIRT", "ShinyItemAnalysis", "shinytest2", "SIAmodules", "Signac", "SigTree", "simCAT", "SimDesign", "SimNPH", "simodels", "simplePHENOTYPES", "SingleCaseES", "skedastic", "skpr", "sna", "snplinkage", "SNSequate", "soptdmaeA", "spatstat.explore", "speakeasyR", "Spectran", "spTimer", "SSDforR", "stan4bart", "startupmsg", "statnet", "stats4teaching", "stcpR6", "streamDAG", "strvalidator", "surface", "switchboard", "tabledown", "tables", "tcltk2", "TestAnaAPP", "TextMiningGUI", "tfrmtbuilder", "tiledb", "tkImgR", "tmsens", "TreeDist", "TTAinterfaceTrendAnalysis", "TVMM", "uHMM", "voi", "vvtableau", "wbacon", "webshot2", "ymd", "YTAnalytics"); pkgs = setdiff(pkgs, c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); pkgs = foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' backend_options: kubernetes: ### PROD-parallel diff --git a/.woodpecker/warm_up_build.yaml b/.woodpecker/warm_up_build.yaml index 4a28f3b..0b0cfd6 100644 --- a/.woodpecker/warm_up_build.yaml +++ b/.woodpecker/warm_up_build.yaml @@ -5,30 +5,42 @@ skip_clone: true matrix: include: - # - PLATFORM: redhat-9 - # BLOCK: 20099:21169 - # ARCH: arm64 - # - PLATFORM: redhat-8 - # BLOCK: 20099:21169 - # ARCH: arm64 - # - PLATFORM: ubuntu-2404 - # BLOCK: 20099:21169 - # ARCH: arm64 - # - PLATFORM: ubuntu-2204 - # BLOCK: 20099:21169 - # ARCH: arm64 - - PLATFORM: alpine-320 - BLOCK: 6547:length(pkgs) + - PLATFORM: redhat-9 + BLOCK: 20099:21169 ARCH: arm64 NCPUS: 2 MEMORY: 28Gi - STRATEGY: sequential - - PLATFORM: alpine-320 - BLOCK: 663:length(pkgs) - ARCH: amd64 + STRATEGY: multisession + - PLATFORM: redhat-8 + BLOCK: 20099:21169 + ARCH: arm64 NCPUS: 2 MEMORY: 28Gi - STRATEGY: sequential + STRATEGY: multisession + - PLATFORM: ubuntu-2404 + BLOCK: 20099:21169 + ARCH: arm64 + NCPUS: 2 + MEMORY: 28Gi + STRATEGY: multisession + - PLATFORM: ubuntu-2204 + BLOCK: 20099:21169 + ARCH: arm64 + NCPUS: 2 + MEMORY: 28Gi + STRATEGY: multisession + # - PLATFORM: alpine-320 + # BLOCK: 7065:length(pkgs) + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 28Gi + # STRATEGY: multisession + # - PLATFORM: alpine-320 + # BLOCK: 1090:length(pkgs) + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 28Gi + # STRATEGY: multisession steps: - name: 'Warming up' image: busybox:latest