clean workflow files

2024-11-26 22:56:58 +01:00 · 2024-11-26 22:56:58 +01:00 · fe6eb6a85b
commit fe6eb6a85b
parent 3d5a815004
1 changed files with 33 additions and 263 deletions
--- a/.woodpecker/build-redhat-9-amd64.yaml
+++ b/.woodpecker/build-redhat-9-amd64.yaml
@ -9,30 +9,6 @@ when:
 skip_clone: true

 steps:
-  # - name: 'Warming up'
-  #   image: busybox:latest
-  #   failure: ignore
-  #   commands:
-  #     - sleep 300
-  #     - echo "Warmed Up"
-  #   backend_options:
-  #     kubernetes:
-  #       resources:
-  #         requests:
-  #           memory: 14Gi
-  #           cpu: 2000m
-  #         limits:
-  #           memory: 14Gi
-  #           cpu: 2000m
-  #       nodeSelector:
-  #         kubernetes.io/arch: amd64
-  #         node.kubernetes.io/instance-type: ccx23
-  #       tolerations:
-  #         - key: 'CI'
-  #           operator: 'Equal'
-  #           value: 'true'
-  #           effect: 'NoSchedule'
-
  - name: 'Build binaries'
    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
    image: docker.io/devxygmbh/amd64-binaries-r-redhat-9
@ -54,22 +30,15 @@ steps:
    # volumes:
    #   - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache
    commands:
-      # download latest package version (avoid having to rebuild image every time)
-      # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
      - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
      - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR
      # for some reason paks pkgcache is often corrupt
      - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/
      - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")'
      - R -q -e 'packageVersion("bincraftR")'
-      # more complicated system dependencies which pak cannot resolve
      - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
-      # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point
-      ### Check
-      # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists)
-      # - Rglpk
-      # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(devxy = 'https://cran.devxy.io/amd64/rhel9/latest', CRAN = 'https://cloud.r-project.org'))); library(dplyr); library(bincraftR); cran_pkgs = tools::CRAN_package_db()[['Package']]; '%nin%' <- Negate('%in%'); foo = query_metadata_table() |> filter(platform == 'redhat-9', arch == 'amd64') |> distinct(name) |> pull(name); pkgs <- dput(cran_pkgs[cran_pkgs %nin% foo][0:300]); options(progressr.enable = TRUE); pkgs = c('later', 'MediaNews', 'RInno', 'sbmSDP', 'sbo', 'sboost', 'sbrl',  'SC.MEB', 'Scalelink', 'ScaleSpikeSlab', 'scalpel', 'scar', 'ScatterDensity',  'scattermore', 'SCBiclust', 'scCAN', 'SCCI', 'scclust', 'sccore',  'SCdeconR', 'scDHA', 'scellpam', 'SCEPtER', 'SCEPtERbinary',  'scGate', 'scGOclust', 'sched', 'scINSIGHT', 'SCIntRuler', 'scISR',  'scistreer', 'scITD', 'scMappR', 'scModels', 'scOntoMatch', 'scoper',  'scorematchingad', 'scorepeak', 'scoringRules', 'SCORNET', 'scout',  'scplot', 'scquantum', 'SCRIP', 'scrm', 'ScRNAIMM', 'scRNAtools',  'scrypt', 'scs', 'scTenifoldKnk', 'SCtools', 'SDAR', 'sdcMicro',  'sdcTable', 'sde', 'sdetorus', 'sdmTMB', 'SDMtune', 'sdpdth',  'sdprisk', 'sdpt3r', 'sdrt', 'sdwd', 'SEA', 'SearchTrees', 'seas',  'SECP', 'secretbase', 'secsse', 'sectorgap', 'secure', 'seededlda',  'SeedMatchR', 'seeker', 'seer', 'segclust2d', 'segmag', 'segmenTier',  'segmentr', 'segMGarch', 'seismicRoll', 'SEL', 'selectiveInference',  'selectspm', 'SELF', 'SemiCompRisks', 'semidist', 'semnar', 'semver',  'SenSpe', 'sentencepiece', 'sentiment.ai', 'sentometrics', 'sentopics',  'sephora', 'seq2R', 'SeqDetect', 'seqgendiff', 'seqHMM', 'seqinr',  'SeqKat', 'seqminer', 'SeqNet', 'seqtrie', 'SequenceSpikeSlab',  'SequentialDesign', 'sequoia', 'seriation', 'serocalculator',  'serrsBayes', 'Seurat', 'SeuratObject', 'sfcr', 'sfcurve', 'sfdct',  'sfheaders', 'sfhotspot', 'SFS', 'SFSI', 'sgd', 'SGDinference',  'sgeostat', 'SGL', 'sglasso', 'sgmodel', 'sgolay', 'SGPR', 'sgsR',  'shapr', 'shapviz', 'sharpeRratio', 'sharpPen', 'SheetReader',  'ShellChron', 'SherlockHolmes', 'shide', 'ShiftConvolvePoibin',  'shiftR', 'shiny.blueprint', 'shiny.fluent', 'shiny.reglog',  'shinyCohortBuilder', 'shinydbauth', 'shinyExprPortal', 'shinyKGode',  'ShinyLink', 'shinymanager', 'shinyMolBio', 'shinypivottabler',  'shinyQueryBuilder', 'shinyrecipes', 'shinyTime', 'skedastic',  'spectrino', 'surveyvoi', 'taskscheduleR', 'theft', 'theftdlc',  'trtswitch', 'ursa'); pkgs = setdiff(pkgs, c('RInno', 'MediaNews', 'later')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, platform = 'redhat-9'))"
+      # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd'
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(devxy = 'https://cran.devxy.io/amd64/rhel9/latest', CRAN = 'https://cloud.r-project.org'))); library(dplyr); library(bincraftR); cran_pkgs = tools::CRAN_package_db()[['Package']]; '%nin%' <- Negate('%in%'); foo = query_metadata_table() |> filter(platform == 'redhat-9', arch == 'amd64') |> distinct(name) |> pull(name); pkgs <- dput(cran_pkgs[cran_pkgs %nin% foo][0:300]); options(progressr.enable = TRUE); pkgs = c('later', 'MediaNews', 'RInno', 'sbmSDP', 'sbo', 'sboost', 'sbrl',  'SC.MEB', 'Scalelink', 'ScaleSpikeSlab', 'scalpel', 'scar', 'ScatterDensity',  'scattermore', 'SCBiclust', 'scCAN', 'SCCI', 'scclust', 'sccore',  'SCdeconR', 'scDHA', 'scellpam', 'SCEPtER', 'SCEPtERbinary',  'scGate', 'scGOclust', 'sched', 'scINSIGHT', 'SCIntRuler', 'scISR',  'scistreer', 'scITD', 'scMappR', 'scModels', 'scOntoMatch', 'scoper',  'scorematchingad', 'scorepeak', 'scoringRules', 'SCORNET', 'scout',  'scplot', 'scquantum', 'SCRIP', 'scrm', 'ScRNAIMM', 'scRNAtools',  'scrypt', 'scs', 'scTenifoldKnk', 'SCtools', 'SDAR', 'sdcMicro',  'sdcTable', 'sde', 'sdetorus', 'sdmTMB', 'SDMtune', 'sdpdth',  'sdprisk', 'sdpt3r', 'sdrt', 'sdwd', 'SEA', 'SearchTrees', 'seas',  'SECP', 'secretbase', 'secsse', 'sectorgap', 'secure', 'seededlda',  'SeedMatchR', 'seeker', 'seer', 'segclust2d', 'segmag', 'segmenTier',  'segmentr', 'segMGarch', 'seismicRoll', 'SEL', 'selectiveInference',  'selectspm', 'SELF', 'SemiCompRisks', 'semidist', 'semnar', 'semver',  'SenSpe', 'sentencepiece', 'sentiment.ai', 'sentometrics', 'sentopics',  'sephora', 'seq2R', 'SeqDetect', 'seqgendiff', 'seqHMM', 'seqinr',  'SeqKat', 'seqminer', 'SeqNet', 'seqtrie', 'SequenceSpikeSlab',  'SequentialDesign', 'sequoia', 'seriation', 'serocalculator',  'serrsBayes', 'Seurat', 'SeuratObject', 'sfcr', 'sfcurve', 'sfdct',  'sfheaders', 'sfhotspot', 'SFS', 'SFSI', 'sgd', 'SGDinference',  'sgeostat', 'SGL', 'sglasso', 'sgmodel', 'sgolay', 'SGPR', 'sgsR',  'shapr', 'shapviz', 'sharpeRratio', 'sharpPen', 'SheetReader',  'ShellChron', 'SherlockHolmes', 'shide', 'ShiftConvolvePoibin',  'shiftR', 'shiny.blueprint', 'shiny.fluent', 'shiny.reglog',  'shinyCohortBuilder', 'shinydbauth', 'shinyExprPortal', 'shinyKGode',  'ShinyLink', 'shinymanager', 'shinyMolBio', 'shinypivottabler',  'shinyQueryBuilder', 'shinyrecipes', 'shinyTime', 'skedastic',  'spectrino', 'surveyvoi', 'taskscheduleR', 'theft', 'theftdlc',  'trtswitch', 'ursa'); pkgs = setdiff(pkgs, c('RInno', 'MediaNews', 'later', 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, platform = 'redhat-9'))"
      ### Manual pkg builds
      # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178
      # RInno: Windows-only
@ -103,7 +72,6 @@ steps:
      # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, debug = FALSE, force = TRUE, platform = "amd64"))'
    backend_options:
      kubernetes:
-        ### PROD-parallel
        resources:
          requests:
            memory: 5Gi
@ -111,22 +79,6 @@ steps:
          limits:
            memory: 20Gi
            cpu: 3000m
-        ### PROD-sequential
-        # resources:
-        #   requests:
-        #     memory: 14Gi
-        #     cpu: 2000m
-        #   limits:
-        #     memory: 14Gi
-        #     cpu: 3000m
-        # DEBUG
-        # resources:
-        #   requests:
-        #     memory: 4Gi
-        #     cpu: 2000m
-        #   limits:
-        #     memory: 4Gi
-        #     cpu: 2000m
        nodeSelector:
          kubernetes.io/arch: amd64
          node.kubernetes.io/instance-type: AX42