From 29aa2a9b4c28a9d0d18f1626e2e292dd1d4882b8 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 21 Oct 2024 22:33:12 +0200 Subject: [PATCH] chore: reorganize CI workflow files --- .woodpecker/build-alpine-320-amd64.yaml | 136 +++++++++++++++++ .woodpecker/build-alpine-320-arm64.yaml | 136 +++++++++++++++++ .woodpecker/build-redhat-8-amd64.yaml | 136 +++++++++++++++++ .woodpecker/build-redhat-8-arm64.yaml | 136 +++++++++++++++++ .woodpecker/build-redhat-9-amd64.yaml | 136 +++++++++++++++++ .woodpecker/build-redhat-9-arm64.yaml | 136 +++++++++++++++++ .woodpecker/build-ubuntu-2204-amd64.yaml | 141 ++++++++++++++++++ .woodpecker/build-ubuntu-2204-arm64.yaml | 136 +++++++++++++++++ ...uild.yaml => build-ubuntu-2404-amd64.yaml} | 136 ++++++----------- .woodpecker/build-ubuntu-2404-arm64.yaml | 136 +++++++++++++++++ .../process-updates-alpine-320-amd64.yaml | 117 +++++++++++++++ .../process-updates-alpine-320-arm64.yaml | 122 +++++++++++++++ .../process-updates-redhat-8-amd64.yaml | 117 +++++++++++++++ .../process-updates-redhat-8-arm64.yaml | 122 +++++++++++++++ .../process-updates-redhat-9-amd64.yaml | 117 +++++++++++++++ .../process-updates-redhat-9-arm64.yaml | 122 +++++++++++++++ .../process-updates-ubuntu-2204-amd64.yaml | 117 +++++++++++++++ .../process-updates-ubuntu-2204-arm64.yaml | 117 +++++++++++++++ .../process-updates-ubuntu-2404-amd64.yaml | 117 +++++++++++++++ .../process-updates-ubuntu-2404-arm64.yaml | 122 +++++++++++++++ .woodpecker/process_cran_updates.yaml | 139 ----------------- .woodpecker/update-package-index.yaml | 30 ++-- .woodpecker/warm_up_build.yaml | 120 --------------- .woodpecker/warm_up_process_cran_updates.yaml | 82 ---------- 24 files changed, 2488 insertions(+), 438 deletions(-) create mode 100644 .woodpecker/build-alpine-320-amd64.yaml create mode 100644 .woodpecker/build-alpine-320-arm64.yaml create mode 100644 .woodpecker/build-redhat-8-amd64.yaml create mode 100644 .woodpecker/build-redhat-8-arm64.yaml create mode 100644 .woodpecker/build-redhat-9-amd64.yaml create mode 100644 .woodpecker/build-redhat-9-arm64.yaml create mode 100644 .woodpecker/build-ubuntu-2204-amd64.yaml create mode 100644 .woodpecker/build-ubuntu-2204-arm64.yaml rename .woodpecker/{build.yaml => build-ubuntu-2404-amd64.yaml} (55%) create mode 100644 .woodpecker/build-ubuntu-2404-arm64.yaml create mode 100644 .woodpecker/process-updates-alpine-320-amd64.yaml create mode 100644 .woodpecker/process-updates-alpine-320-arm64.yaml create mode 100644 .woodpecker/process-updates-redhat-8-amd64.yaml create mode 100644 .woodpecker/process-updates-redhat-8-arm64.yaml create mode 100644 .woodpecker/process-updates-redhat-9-amd64.yaml create mode 100644 .woodpecker/process-updates-redhat-9-arm64.yaml create mode 100644 .woodpecker/process-updates-ubuntu-2204-amd64.yaml create mode 100644 .woodpecker/process-updates-ubuntu-2204-arm64.yaml create mode 100644 .woodpecker/process-updates-ubuntu-2404-amd64.yaml create mode 100644 .woodpecker/process-updates-ubuntu-2404-arm64.yaml delete mode 100644 .woodpecker/process_cran_updates.yaml delete mode 100644 .woodpecker/warm_up_build.yaml delete mode 100644 .woodpecker/warm_up_process_cran_updates.yaml diff --git a/.woodpecker/build-alpine-320-amd64.yaml b/.woodpecker/build-alpine-320-amd64.yaml new file mode 100644 index 0000000..549c48d --- /dev/null +++ b/.woodpecker/build-alpine-320-amd64.yaml @@ -0,0 +1,136 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "alpine-320-amd64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Build binaries' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-alpine-320 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: alpine-320 + BLOCK: 21424:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + INSTANCE_TYPE: cpx51 + STRATEGY: multisession + volumes: + - amd64-binaries-r-dep-cache-alpine-320:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + # more complicated system dependencies which pak cannot resolve + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + ### Check + # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) + # - Rglpk + # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'alpine-320'))" + ### Manual pkg builds + # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 + # RInno: Windows-only + # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) + # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved + # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ + # ROracle: sysdep missing -> create rule + # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 + # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex + # RmecabKo: fatal error: mecab.h: No such file or directory + # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 + # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 + # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex + # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 + # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 + # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed + # Racmacs -> rmarchingcubes + # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: + # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 + # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail + # clrng -> gpuR + # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything + # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 + # filters: -> email sent + + # magickGUI, biplotbootGUI + # ymd (alpine, voi) + # tfrmtbuilder -> V8 (alpine) + # Rcmdr (alpine) + # readstata13 (alpine) + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "amd64"))' + backend_options: + kubernetes: + ### PROD-parallel + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + ### PROD-sequential + # resources: + # requests: + # memory: 29Gi + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.woodpecker/build-alpine-320-arm64.yaml b/.woodpecker/build-alpine-320-arm64.yaml new file mode 100644 index 0000000..6a31c71 --- /dev/null +++ b/.woodpecker/build-alpine-320-arm64.yaml @@ -0,0 +1,136 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "alpine-320-arm64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo 'Warmed Up' + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Build binaries' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/arm64-binaries-r-alpine-320 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: alpine-320 + BLOCK: 20896:length(pkgs) + ARCH: arm64 + NCPUS: 2 + MEMORY: 29Gi + INSTANCE_TYPE: cax41 + STRATEGY: multisession + volumes: + - arm64-binaries-r-dep-cache-alpine-320:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + # more complicated system dependencies which pak cannot resolve + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + ### Check + # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) + # - Rglpk + # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'alpine-320'))" + ### Manual pkg builds + # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 + # RInno: Windows-only + # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) + # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved + # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ + # ROracle: sysdep missing -> create rule + # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 + # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex + # RmecabKo: fatal error: mecab.h: No such file or directory + # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 + # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 + # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex + # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 + # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 + # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed + # Racmacs -> rmarchingcubes + # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: + # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 + # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail + # clrng -> gpuR + # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything + # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 + # filters: -> email sent + + # magickGUI, biplotbootGUI + # ymd (alpine, voi) + # tfrmtbuilder -> V8 (alpine) + # Rcmdr (alpine) + # readstata13 (alpine) + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( 'RcmdrPlugin.orloca', 'RcmdrPlugin.PcaRobust', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.RMTCJags', 'RcmdrPlugin.ROC', 'RcmdrPlugin.sos' , 'RcmdrPlugin.survival', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.temis', 'RcmdrPlugin.UCA', 'RcmdrPlugin.WorldFlora', 'streamDAG', 'stan4bart', 'snplinkage', 'saeHB.panel.beta', 'rTwig', 'rswipl', 'RPointCloud', 'rPDBapi', 'rolog', 'rmcfs', 'rjags', 'ResultModelManager', 'RcppCWB', 'RcmdrPlugin.WorldFlora', 'RcmdrPlugin.UCA', 'RcmdrPlugin.temis', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.survival', 'RcmdrPlugin.sos', 'RcmdrPlugin.ROC', 'RcmdrPlugin.RMTCJags', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.PcaRobust', 'RclusTool', 'Rbeast', 'rayrender', 'quanteda.textmodels', 'polmineR', 'plotBart', 'NlinTS', 'ndtv', 'overtureR', 'mathml', 'magickGUI', 'lgpr', 'KappaGUI', 'historicalborrow', 'GSODR', 'GRIDCOPULA', 'googleAnalyticsR', 'fluidsynth', 'EventDetectGUI', 'evalITR', 'EBcoBART', 'deeptrafo', 'dbarts', 'CRE', 'countSTAR', 'bartMan', 'bartCause', 'altmeta', 'EthSEQ', 'dartR.spatial', 'dartR.sim', 'gwid', 'Mega2R', 'priorCON', 'prior3D', 'MCARtest', 'designmatch', 'highs', 'ROI.plugin.highs', 'GPCMlasso'); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'alpine-320'))' + backend_options: + kubernetes: + ### PROD-parallel + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + ### PROD-sequential + # resources: + # requests: + # memory: 29Gi + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.woodpecker/build-redhat-8-amd64.yaml b/.woodpecker/build-redhat-8-amd64.yaml new file mode 100644 index 0000000..f24333d --- /dev/null +++ b/.woodpecker/build-redhat-8-amd64.yaml @@ -0,0 +1,136 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "redhat-8-amd64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Build binaries' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-redhat-8 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: redhat-8 + BLOCK: 20588:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + INSTANCE_TYPE: cpx51 + STRATEGY: multisession + volumes: + - amd64-binaries-r-dep-cache-redhat-8:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + # more complicated system dependencies which pak cannot resolve + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + ### Check + # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) + # - Rglpk + # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'redhat-8'))" + ### Manual pkg builds + # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 + # RInno: Windows-only + # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) + # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved + # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ + # ROracle: sysdep missing -> create rule + # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 + # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex + # RmecabKo: fatal error: mecab.h: No such file or directory + # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 + # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 + # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex + # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 + # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 + # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed + # Racmacs -> rmarchingcubes + # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: + # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 + # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail + # clrng -> gpuR + # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything + # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 + # filters: -> email sent + + # magickGUI, biplotbootGUI + # ymd (alpine, voi) + # tfrmtbuilder -> V8 (alpine) + # Rcmdr (alpine) + # readstata13 (alpine) + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "amd64"))' + backend_options: + kubernetes: + ### PROD-parallel + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + ### PROD-sequential + # resources: + # requests: + # memory: 29Gi + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.woodpecker/build-redhat-8-arm64.yaml b/.woodpecker/build-redhat-8-arm64.yaml new file mode 100644 index 0000000..b40dc6b --- /dev/null +++ b/.woodpecker/build-redhat-8-arm64.yaml @@ -0,0 +1,136 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "redhat-8-arm64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Build binaries' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/arm64-binaries-r-redhat-8 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: redhat-8 + BLOCK: 20327:length(pkgs) + ARCH: arm64 + NCPUS: 2 + MEMORY: 29Gi + INSTANCE_TYPE: cax41 + STRATEGY: multisession + volumes: + - arm64-binaries-r-dep-cache-redhat-8:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + # more complicated system dependencies which pak cannot resolve + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + ### Check + # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) + # - Rglpk + # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'redhat-8'))" + ### Manual pkg builds + # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 + # RInno: Windows-only + # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) + # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved + # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ + # ROracle: sysdep missing -> create rule + # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 + # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex + # RmecabKo: fatal error: mecab.h: No such file or directory + # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 + # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 + # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex + # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 + # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 + # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed + # Racmacs -> rmarchingcubes + # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: + # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 + # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail + # clrng -> gpuR + # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything + # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 + # filters: -> email sent + + # magickGUI, biplotbootGUI + # ymd (alpine, voi) + # tfrmtbuilder -> V8 (alpine) + # Rcmdr (alpine) + # readstata13 (alpine) + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "arm64"))' + backend_options: + kubernetes: + ### PROD-parallel + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + ### PROD-sequential + # resources: + # requests: + # memory: 29Gi + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.woodpecker/build-redhat-9-amd64.yaml b/.woodpecker/build-redhat-9-amd64.yaml new file mode 100644 index 0000000..e16468d --- /dev/null +++ b/.woodpecker/build-redhat-9-amd64.yaml @@ -0,0 +1,136 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "redhat-9-amd64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Build binaries' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-redhat-9 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: redhat-9 + BLOCK: 19127:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + INSTANCE_TYPE: cpx51 + STRATEGY: multisession + volumes: + - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + # more complicated system dependencies which pak cannot resolve + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + ### Check + # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) + # - Rglpk + # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'redhat-9'))" + ### Manual pkg builds + # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 + # RInno: Windows-only + # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) + # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved + # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ + # ROracle: sysdep missing -> create rule + # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 + # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex + # RmecabKo: fatal error: mecab.h: No such file or directory + # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 + # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 + # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex + # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 + # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 + # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed + # Racmacs -> rmarchingcubes + # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: + # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 + # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail + # clrng -> gpuR + # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything + # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 + # filters: -> email sent + + # magickGUI, biplotbootGUI + # ymd (alpine, voi) + # tfrmtbuilder -> V8 (alpine) + # Rcmdr (alpine) + # readstata13 (alpine) + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "amd64"))' + backend_options: + kubernetes: + ### PROD-parallel + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + ### PROD-sequential + # resources: + # requests: + # memory: 29Gi + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.woodpecker/build-redhat-9-arm64.yaml b/.woodpecker/build-redhat-9-arm64.yaml new file mode 100644 index 0000000..bb62de3 --- /dev/null +++ b/.woodpecker/build-redhat-9-arm64.yaml @@ -0,0 +1,136 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "redhat-9-arm64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Build binaries' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/arm64-binaries-r-redhat-9 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: redhat-9 + BLOCK: 19336:length(pkgs) + ARCH: arm64 + NCPUS: 2 + MEMORY: 29Gi + INSTANCE_TYPE: cax41 + STRATEGY: multisession + volumes: + - arm64-binaries-r-dep-cache-redhat-9:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + # more complicated system dependencies which pak cannot resolve + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + ### Check + # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) + # - Rglpk + # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'redhat-9'))" + ### Manual pkg builds + # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 + # RInno: Windows-only + # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) + # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved + # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ + # ROracle: sysdep missing -> create rule + # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 + # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex + # RmecabKo: fatal error: mecab.h: No such file or directory + # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 + # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 + # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex + # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 + # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 + # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed + # Racmacs -> rmarchingcubes + # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: + # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 + # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail + # clrng -> gpuR + # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything + # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 + # filters: -> email sent + + # magickGUI, biplotbootGUI + # ymd (alpine, voi) + # tfrmtbuilder -> V8 (alpine) + # Rcmdr (alpine) + # readstata13 (alpine) + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "arm64"))' + backend_options: + kubernetes: + ### PROD-parallel + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + ### PROD-sequential + # resources: + # requests: + # memory: 29Gi + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.woodpecker/build-ubuntu-2204-amd64.yaml b/.woodpecker/build-ubuntu-2204-amd64.yaml new file mode 100644 index 0000000..7b2a064 --- /dev/null +++ b/.woodpecker/build-ubuntu-2204-amd64.yaml @@ -0,0 +1,141 @@ +when: + # - event: manual + # evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "ubuntu-2204-amd64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + environment: + PLATFORM: ubuntu-2204 + BLOCK: 21389:length(pkgs) + NCPUS: 2 + STRATEGY: multisession + commands: + - sleep 1 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Build binaries' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-amd64 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ubuntu-2204 + BLOCK: 21389:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + INSTANCE_TYPE: cpx51 + STRATEGY: multisession + volumes: + - amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + # more complicated system dependencies which pak cannot resolve + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + ### Check + # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) + # - Rglpk + # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'ubuntu-2204'))" + ### Manual pkg builds + # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 + # RInno: Windows-only + # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) + # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved + # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ + # ROracle: sysdep missing -> create rule + # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 + # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex + # RmecabKo: fatal error: mecab.h: No such file or directory + # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 + # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 + # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex + # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 + # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 + # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed + # Racmacs -> rmarchingcubes + # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: + # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 + # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail + # clrng -> gpuR + # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything + # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 + # filters: -> email sent + + # magickGUI, biplotbootGUI + # ymd (alpine, voi) + # tfrmtbuilder -> V8 (alpine) + # Rcmdr (alpine) + # readstata13 (alpine) + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "amd64"))' + backend_options: + kubernetes: + ### PROD-parallel + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + ### PROD-sequential + # resources: + # requests: + # memory: 29Gi + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.woodpecker/build-ubuntu-2204-arm64.yaml b/.woodpecker/build-ubuntu-2204-arm64.yaml new file mode 100644 index 0000000..2cafe1f --- /dev/null +++ b/.woodpecker/build-ubuntu-2204-arm64.yaml @@ -0,0 +1,136 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "ubuntu-2204-arm64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Build binaries' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2204 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ubuntu-2204 + BLOCK: 20653:length(pkgs) + ARCH: arm64 + NCPUS: 2 + MEMORY: 29Gi + INSTANCE_TYPE: cax41 + STRATEGY: multisession + volumes: + - arm64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/gert + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + # more complicated system dependencies which pak cannot resolve + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + ### Check + # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) + # - Rglpk + # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'ubuntu-2204'))" + ### Manual pkg builds + # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 + # RInno: Windows-only + # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) + # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved + # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ + # ROracle: sysdep missing -> create rule + # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 + # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex + # RmecabKo: fatal error: mecab.h: No such file or directory + # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 + # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 + # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex + # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 + # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 + # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed + # Racmacs -> rmarchingcubes + # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: + # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 + # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail + # clrng -> gpuR + # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything + # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 + # filters: -> email sent + + # magickGUI, biplotbootGUI + # ymd (alpine, voi) + # tfrmtbuilder -> V8 (alpine) + # Rcmdr (alpine) + # readstata13 (alpine) + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "arm64"))' + backend_options: + kubernetes: + ### PROD-parallel + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + ### PROD-sequential + # resources: + # requests: + # memory: 29Gi + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.woodpecker/build.yaml b/.woodpecker/build-ubuntu-2404-amd64.yaml similarity index 55% rename from .woodpecker/build.yaml rename to .woodpecker/build-ubuntu-2404-amd64.yaml index 4a9f9c9..9b0b106 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build-ubuntu-2404-amd64.yaml @@ -1,89 +1,44 @@ when: + # - event: manual + # evaluate: 'build == "all"' - event: manual - evaluate: 'task == "build"' + evaluate: 'build == "ubuntu-2404-amd64"' skip_clone: true -depends_on: - - warm_up_build - -matrix: - include: - - PLATFORM: redhat-9 - BLOCK: 14736:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cax41 - STRATEGY: multisession - - PLATFORM: redhat-8 - BLOCK: 14065:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cax41 - STRATEGY: multisession - - PLATFORM: ubuntu-2404 - BLOCK: 16953:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cax41 - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - BLOCK: 19280:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cax41 - STRATEGY: multisession - - PLATFORM: alpine-320 - BLOCK: 15871:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cax41 - STRATEGY: multisession - # ### AMD - - PLATFORM: redhat-9 - BLOCK: 17739:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cpx51 - STRATEGY: multisession - - PLATFORM: redhat-8 - BLOCK: 17534:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cpx51 - STRATEGY: multisession - - PLATFORM: ubuntu-2404 - BLOCK: 20732:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cpx51 - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - BLOCK: 21389:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cpx51 - STRATEGY: multisession - - PLATFORM: alpine-320 - BLOCK: 15:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cpx51 - STRATEGY: multisession steps: - - name: 'Build binaries (${PLATFORM}: ${BLOCK})' + - name: 'Warming up' + image: busybox:latest + failure: ignore + environment: + PLATFORM: ubuntu-2204 + BLOCK: 21389:length(pkgs) + NCPUS: 2 + STRATEGY: multisession + commands: + - sleep 1 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Build binaries' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] - image: docker.io/devxygmbh/${ARCH}-binaries-r-${PLATFORM} + image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 pull: true # to ensure the PACKAGES upload step is being executed failure: ignore @@ -93,8 +48,15 @@ steps: R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ubuntu-2404 + BLOCK: 21389:length(pkgs) + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + INSTANCE_TYPE: cpx51 + STRATEGY: multisession volumes: - - ${ARCH}-binaries-r-dep-cache-${PLATFORM}:/mnt/cache + - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache commands: # download latest package version (avoid having to rebuild image every time) # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 @@ -111,7 +73,7 @@ steps: # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) # - Rglpk # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run - - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c("biplotbootGUI", "Rsymphony", "PortfolioAnalytics", "ROI.plugin.symphony", "adea", "RInno", "skedastic", "RPushbullet", "SimNPH", "SimDesign","simCAT", "SIAmodules", "shinytest2", "shinyIRT", "ShinyItemAnalysis", "shiny.benchmark", "scDIFtest", "RSP", "ROracle", "ROI.plugin.qpoases", "ROI.plugin.cplex", "RmecabKo", "rmarchingcubes", "RcppMeCab", "Rcplex", "ravetools", "Rbeast", "Rblpapi", "RcppEnsmallen", "qtkit", "priorCON", "prior3D", "PerFit", "outlierensembles", "nomnoml", "mxfda", "MultBiplotR", "mstDIF", "mdsr", "makepipe", "lordif", "KSgeneral", "kequate", "jstager", "jrt", "irtreliability", "irtQ", "irtpwr", "irtawsi", "GRShiny", "gpuR", "googletraffic", "giacR", "gamstransfer", "aflps", "flow", "filters", "faoutlier", "equateIRT", "EnrichIntersect", "EFA.dimensions", "DFIT", "D3mirt", "ctgdist", "CoTiMA", "ConvertPar", "clrng", "bscui", "BifactorIndicesCalculator", "autoFC", "airt")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers("cli"); progressr::handlers(global = TRUE); foo = lapply(pkgs[${BLOCK}], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' + - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c("biplotbootGUI", "Rsymphony", "PortfolioAnalytics", "ROI.plugin.symphony", "adea", "RInno", "skedastic", "RPushbullet", "SimNPH", "SimDesign","simCAT", "SIAmodules", "shinytest2", "shinyIRT", "ShinyItemAnalysis", "shiny.benchmark", "scDIFtest", "RSP", "ROracle", "ROI.plugin.qpoases", "ROI.plugin.cplex", "RmecabKo", "rmarchingcubes", "RcppMeCab", "Rcplex", "ravetools", "Rbeast", "Rblpapi", "RcppEnsmallen", "qtkit", "priorCON", "prior3D", "PerFit", "outlierensembles", "nomnoml", "mxfda", "MultBiplotR", "mstDIF", "mdsr", "makepipe", "lordif", "KSgeneral", "kequate", "jstager", "jrt", "irtreliability", "irtQ", "irtpwr", "irtawsi", "GRShiny", "gpuR", "googletraffic", "giacR", "gamstransfer", "aflps", "flow", "filters", "faoutlier", "equateIRT", "EnrichIntersect", "EFA.dimensions", "DFIT", "D3mirt", "ctgdist", "CoTiMA", "ConvertPar", "clrng", "bscui", "BifactorIndicesCalculator", "autoFC", "airt")); library(bincraftR); future::plan("$STRATEGY", workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers("cli"); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "amd64"))' ### Manual pkg builds # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 # RInno: Windows-only @@ -142,21 +104,21 @@ steps: # tfrmtbuilder -> V8 (alpine) # Rcmdr (alpine) # readstata13 (alpine) - # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "amd64"))' backend_options: kubernetes: ### PROD-parallel resources: requests: - memory: ${MEMORY} + memory: 29Gi cpu: 4000m limits: - memory: ${MEMORY} + memory: 29Gi cpu: 4000m ### PROD-sequential # resources: # requests: - # memory: ${MEMORY} + # memory: 29Gi # cpu: 2000m # limits: # memory: 29Gi @@ -170,8 +132,8 @@ steps: # memory: 4Gi # cpu: 2000m nodeSelector: - kubernetes.io/arch: '${ARCH}' - node.kubernetes.io/instance-type: '${INSTANCE_TYPE}' + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 tolerations: - key: 'CI' operator: 'Equal' diff --git a/.woodpecker/build-ubuntu-2404-arm64.yaml b/.woodpecker/build-ubuntu-2404-arm64.yaml new file mode 100644 index 0000000..f92a504 --- /dev/null +++ b/.woodpecker/build-ubuntu-2404-arm64.yaml @@ -0,0 +1,136 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "ubuntu-2404-arm64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Build binaries' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2404 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ubuntu-2404 + BLOCK: 20126:length(pkgs) + ARCH: arm64 + NCPUS: 2 + MEMORY: 29Gi + INSTANCE_TYPE: cax41 + STRATEGY: multisession + volumes: + - arm64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + # more complicated system dependencies which pak cannot resolve + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + ### Check + # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) + # - Rglpk + # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'ubuntu-2404'))" + ### Manual pkg builds + # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 + # RInno: Windows-only + # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) + # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved + # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ + # ROracle: sysdep missing -> create rule + # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 + # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex + # RmecabKo: fatal error: mecab.h: No such file or directory + # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 + # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 + # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex + # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 + # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 + # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed + # Racmacs -> rmarchingcubes + # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: + # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 + # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail + # clrng -> gpuR + # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything + # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 + # filters: -> email sent + + # magickGUI, biplotbootGUI + # ymd (alpine, voi) + # tfrmtbuilder -> V8 (alpine) + # Rcmdr (alpine) + # readstata13 (alpine) + # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "arm64"))' + backend_options: + kubernetes: + ### PROD-parallel + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + ### PROD-sequential + # resources: + # requests: + # memory: 29Gi + # cpu: 2000m + # limits: + # memory: 29Gi + # cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.woodpecker/process-updates-alpine-320-amd64.yaml b/.woodpecker/process-updates-alpine-320-amd64.yaml new file mode 100644 index 0000000..2f93ca4 --- /dev/null +++ b/.woodpecker/process-updates-alpine-320-amd64.yaml @@ -0,0 +1,117 @@ +when: + - event: [cron] + cron: process-cran-updates-alpine-320-amd64 + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "alpine-320-amd64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Processing Updates' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-alpine-320 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: alpine-320 + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + INSTANCE_TYPE: cpx51 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 40) + volumes: + - amd64-binaries-r-dep-cache-alpine-320:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); pkgs = tools::CRAN_package_db()[[1]][$BLOCK]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('sequential'); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = $INTERVAL, platform = 'alpine-320', process_updated = TRUE, process_new = TRUE)" + backend_options: + kubernetes: + ### PROD + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/process-updates-alpine-320-arm64.yaml b/.woodpecker/process-updates-alpine-320-arm64.yaml new file mode 100644 index 0000000..c84c909 --- /dev/null +++ b/.woodpecker/process-updates-alpine-320-arm64.yaml @@ -0,0 +1,122 @@ +when: + - event: [cron] + cron: process-cran-updates + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "alpine-320-arm64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + environment: + PLATFORM: ubuntu-2204 + BLOCK: 21389:length(pkgs) + NCPUS: 2 + STRATEGY: multisession + commands: + - sleep 1 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Processing Updates' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/arm64-binaries-r-${PLATFORM} + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: alpine-320 + ARCH: arm64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + INSTANCE_TYPE: cax41 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 31) + volumes: + - arm64-binaries-r-dep-cache-${PLATFORM}:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); pkgs = tools::CRAN_package_db()[[1]][$BLOCK]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('sequential'); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = $INTERVAL, platform = 'alpine-320', process_updated = TRUE, process_new = TRUE)" + backend_options: + kubernetes: + ### PROD + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/process-updates-redhat-8-amd64.yaml b/.woodpecker/process-updates-redhat-8-amd64.yaml new file mode 100644 index 0000000..5e8dc35 --- /dev/null +++ b/.woodpecker/process-updates-redhat-8-amd64.yaml @@ -0,0 +1,117 @@ +when: + - event: [cron] + cron: process-cran-updates-redhat-8-amd64 + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "redhat-8-amd64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 1 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Processing Updates' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-redhat-8 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: redhat-8 + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + INSTANCE_TYPE: cpx51 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3) + volumes: + - amd64-binaries-r-dep-cache-redhat-8:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); pkgs = tools::CRAN_package_db()[[1]][$BLOCK]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('sequential'); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE)" + backend_options: + kubernetes: + ### PROD + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/process-updates-redhat-8-arm64.yaml b/.woodpecker/process-updates-redhat-8-arm64.yaml new file mode 100644 index 0000000..c759aec --- /dev/null +++ b/.woodpecker/process-updates-redhat-8-arm64.yaml @@ -0,0 +1,122 @@ +when: + - event: [cron] + cron: process-cran-updates + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "redhat-8-arm64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + environment: + PLATFORM: ubuntu-2204 + BLOCK: 21389:length(pkgs) + NCPUS: 2 + STRATEGY: multisession + commands: + - sleep 1 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Processing Updates' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/arm64-binaries-r-${PLATFORM} + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: redhat-8 + ARCH: arm64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + INSTANCE_TYPE: cax41 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 31) + volumes: + - arm64-binaries-r-dep-cache-${PLATFORM}:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); pkgs = tools::CRAN_package_db()[[1]][$BLOCK]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('sequential'); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE)" + backend_options: + kubernetes: + ### PROD + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/process-updates-redhat-9-amd64.yaml b/.woodpecker/process-updates-redhat-9-amd64.yaml new file mode 100644 index 0000000..403624c --- /dev/null +++ b/.woodpecker/process-updates-redhat-9-amd64.yaml @@ -0,0 +1,117 @@ +when: + - event: [cron] + cron: process-cran-updates-redhat-9-amd64 + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "redhat-9-amd64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Processing Updates' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-redhat-9 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: redhat-9 + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + INSTANCE_TYPE: cpx51 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3) + volumes: + - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); pkgs = tools::CRAN_package_db()[[1]][$BLOCK]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('sequential'); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE)" + backend_options: + kubernetes: + ### PROD + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/process-updates-redhat-9-arm64.yaml b/.woodpecker/process-updates-redhat-9-arm64.yaml new file mode 100644 index 0000000..4868759 --- /dev/null +++ b/.woodpecker/process-updates-redhat-9-arm64.yaml @@ -0,0 +1,122 @@ +when: + - event: [cron] + cron: process-cran-updates + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "redhat-9-arm64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + environment: + PLATFORM: ubuntu-2204 + BLOCK: 21389:length(pkgs) + NCPUS: 2 + STRATEGY: multisession + commands: + - sleep 1 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Processing Updates' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/arm64-binaries-r-${PLATFORM} + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: redhat-9 + ARCH: arm64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + INSTANCE_TYPE: cax41 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 31) + volumes: + - arm64-binaries-r-dep-cache-${PLATFORM}:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); pkgs = tools::CRAN_package_db()[[1]][$BLOCK]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('sequential'); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE)" + backend_options: + kubernetes: + ### PROD + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/process-updates-ubuntu-2204-amd64.yaml b/.woodpecker/process-updates-ubuntu-2204-amd64.yaml new file mode 100644 index 0000000..aa5e36c --- /dev/null +++ b/.woodpecker/process-updates-ubuntu-2204-amd64.yaml @@ -0,0 +1,117 @@ +when: + - event: [cron] + cron: process-cran-updates-ubuntu-2204-amd64 + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "ubuntu-2204-amd64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Processing Updates' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2204 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ubuntu-2204 + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + INSTANCE_TYPE: cpx51 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3) + volumes: + - amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); pkgs = tools::CRAN_package_db()[[1]][$BLOCK]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('sequential'); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE)" + backend_options: + kubernetes: + ### PROD + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/process-updates-ubuntu-2204-arm64.yaml b/.woodpecker/process-updates-ubuntu-2204-arm64.yaml new file mode 100644 index 0000000..30900f0 --- /dev/null +++ b/.woodpecker/process-updates-ubuntu-2204-arm64.yaml @@ -0,0 +1,117 @@ +when: + - event: [cron] + cron: process-cran-updates-ubuntu-2204-arm64 + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "ubuntu-2204-arm64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Processing Updates' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2204 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ubuntu-2204 + ARCH: arm64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + INSTANCE_TYPE: cax41 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3) + volumes: + - arm64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); pkgs = tools::CRAN_package_db()[[1]][$BLOCK]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('sequential'); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE)" + backend_options: + kubernetes: + ### PROD + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/process-updates-ubuntu-2404-amd64.yaml b/.woodpecker/process-updates-ubuntu-2404-amd64.yaml new file mode 100644 index 0000000..e131253 --- /dev/null +++ b/.woodpecker/process-updates-ubuntu-2404-amd64.yaml @@ -0,0 +1,117 @@ +when: + - event: [cron] + cron: process-cran-updates-ubuntu-2404-amd64 + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "ubuntu-2404-amd64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Processing Updates' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ubuntu-2404 + ARCH: amd64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + INSTANCE_TYPE: cpx51 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3) + volumes: + - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); pkgs = tools::CRAN_package_db()[[1]][$BLOCK]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('sequential'); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE)" + backend_options: + kubernetes: + ### PROD + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: cpx51 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/process-updates-ubuntu-2404-arm64.yaml b/.woodpecker/process-updates-ubuntu-2404-arm64.yaml new file mode 100644 index 0000000..d8f396c --- /dev/null +++ b/.woodpecker/process-updates-ubuntu-2404-arm64.yaml @@ -0,0 +1,122 @@ +when: + - event: [cron] + cron: process-cran-updates-ubuntu-2404-arm64 + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "ubuntu-2404-arm64"' + +skip_clone: true + +steps: + - name: 'Warming up' + image: busybox:latest + failure: ignore + environment: + PLATFORM: ubuntu-2204 + BLOCK: 21389:length(pkgs) + NCPUS: 2 + STRATEGY: multisession + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + nodeSelector: + kubernetes.io/arch: arm64 + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'Processing Updates' + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2404 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ubuntu-2404 + ARCH: arm64 + NCPUS: 2 + MEMORY: 29Gi + STRATEGY: multisession + INSTANCE_TYPE: cax41 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3) + volumes: + - arm64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres + # for some reason paks pkgcache is often corrupt + - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ + - R -q -e 'pak::meta_update()' + - R CMD INSTALL . + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); pkgs = tools::CRAN_package_db()[[1]][$BLOCK]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('sequential'); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE)" + backend_options: + kubernetes: + ### PROD + resources: + requests: + memory: 29Gi + cpu: 4000m + limits: + memory: 29Gi + cpu: 4000m + # DEBUG + # resources: + # requests: + # memory: 4Gi + # cpu: 2000m + # limits: + # memory: 4Gi + # cpu: 2000m + nodeSelector: + kubernetes.io/arch: 'arm64' + node.kubernetes.io/instance-type: cax41 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/process_cran_updates.yaml b/.woodpecker/process_cran_updates.yaml deleted file mode 100644 index b0d4151..0000000 --- a/.woodpecker/process_cran_updates.yaml +++ /dev/null @@ -1,139 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates - - event: manual - evaluate: 'task == "process-cran-updates"' - -skip_clone: true - -depends_on: - - warm_up_process_cran_updates - -matrix: - include: - ### ARM - # - PLATFORM: redhat-9 - # ARCH: arm64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # - PLATFORM: redhat-8 - # ARCH: arm64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # - PLATFORM: ubuntu-2404 - # ARCH: arm64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # - PLATFORM: ubuntu-2204 - # ARCH: arm64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # ### AMD - # - PLATFORM: redhat-9 - # ARCH: amd64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # - PLATFORM: redhat-8 - # ARCH: amd64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # - PLATFORM: ubuntu-2404 - # ARCH: amd64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - - PLATFORM: ubuntu-2204 - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - INSTANCE_TYPE: cpx51 - INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 31) - # - PLATFORM: alpine-320 - # ARCH: amd64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - -steps: - - name: 'Processing Updates: (${PLATFORM})' - secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] - image: docker.io/devxygmbh/${ARCH}-binaries-r-${PLATFORM} - pull: true - environment: - GIT_USER: pat-s - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - volumes: - - ${ARCH}-binaries-r-dep-cache-${PLATFORM}:/mnt/cache - commands: - # download latest package version (avoid having to rebuild image every time) - - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres - # for some reason paks pkgcache is often corrupt - - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ - - R -q -e 'pak::meta_update()' - - R CMD INSTALL . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error", repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]][${BLOCK}]; pkgs = setdiff(pkgs, c("biplotbootGUI", "Rsymphony", "PortfolioAnalytics", "ROI.plugin.symphony", "adea", "RInno", "skedastic", "RPushbullet", "SimNPH", "SimDesign","simCAT", "SIAmodules", "shinytest2", "shinyIRT", "ShinyItemAnalysis", "shiny.benchmark", "scDIFtest", "RSP", "ROracle", "ROI.plugin.qpoases", "ROI.plugin.cplex", "RmecabKo", "rmarchingcubes", "RcppMeCab", "Rcplex", "ravetools", "Rbeast", "Rblpapi", "RcppEnsmallen", "qtkit", "priorCON", "prior3D", "PerFit", "outlierensembles", "nomnoml", "mxfda", "MultBiplotR", "mstDIF", "mdsr", "makepipe", "lordif", "KSgeneral", "kequate", "jstager", "jrt", "irtreliability", "irtQ", "irtpwr", "irtawsi", "GRShiny", "gpuR", "googletraffic", "giacR", "gamstransfer", "aflps", "flow", "filters", "faoutlier", "equateIRT", "EnrichIntersect", "EFA.dimensions", "DFIT", "D3mirt", "ctgdist", "CoTiMA", "ConvertPar", "clrng", "bscui", "BifactorIndicesCalculator", "autoFC", "airt")); library(bincraftR); future::plan("sequential"); options(progressr.enable = TRUE); progressr::handlers("cli"); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = ${INTERVAL}, platform = "${PLATFORM}", process_updated = TRUE, process_new = TRUE)' - backend_options: - kubernetes: - ### PROD - resources: - requests: - memory: ${MEMORY} - cpu: 4000m - limits: - memory: ${MEMORY} - cpu: 4000m - # DEBUG - # resources: - # requests: - # memory: 4Gi - # cpu: 2000m - # limits: - # memory: 4Gi - # cpu: 2000m - nodeSelector: - kubernetes.io/arch: '${ARCH}' - node.kubernetes.io/instance-type: '${INSTANCE_TYPE}' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 - when: - - status: [failure] - settings: - url: https://ntfy.devxy.io/ci-builds - token: - from_secret: ntfy_token - title: "CI '${CI_REPO}' failed" - actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; - click: ${CI_BUILD_LINK} - message: | - Workflow: ${CI_WORKFLOW_NAME/\n//} - Commit message: ${CI_COMMIT_MESSAGE/\n//} - Commit Author: ${CI_COMMIT_AUTHOR/\n//} - backend_options: - kubernetes: - resources: - requests: - memory: 100Mi - cpu: 0m - limits: - memory: 300Mi - cpu: 200m diff --git a/.woodpecker/update-package-index.yaml b/.woodpecker/update-package-index.yaml index f380599..246e4bf 100644 --- a/.woodpecker/update-package-index.yaml +++ b/.woodpecker/update-package-index.yaml @@ -1,23 +1,33 @@ when: - event: [cron] cron: update-package-index + - event: manual + evaluate: 'task == "update-package-index"' skip_clone: true matrix: include: - - PLATFORM: redhat-9 - ARCH: arm64 - - PLATFORM: redhat-8 - ARCH: arm64 + # - PLATFORM: redhat-9 + # ARCH: arm64 + # - PLATFORM: redhat-8 + # ARCH: arm64 + # - PLATFORM: ubuntu-2404 + # ARCH: arm64 + # - PLATFORM: ubuntu-2204 + # ARCH: arm64 + # - PLATFORM: alpine-320 + # ARCH: arm64 + # - PLATFORM: redhat-9 + # ARCH: amd64 + # - PLATFORM: redhat-8 + # ARCH: amd64 - PLATFORM: ubuntu-2404 - ARCH: arm64 - - PLATFORM: ubuntu-2204 - ARCH: arm64 - - PLATFORM: alpine-320 - ARCH: arm64 - - PLATFORM: alpine-320 ARCH: amd64 + # - PLATFORM: ubuntu-2204 + # ARCH: amd64 + # - PLATFORM: alpine-320 + # ARCH: amd64 steps: - name: Upload PACKAGES files secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] diff --git a/.woodpecker/warm_up_build.yaml b/.woodpecker/warm_up_build.yaml deleted file mode 100644 index ae3a156..0000000 --- a/.woodpecker/warm_up_build.yaml +++ /dev/null @@ -1,120 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build"' - -skip_clone: true - -matrix: - include: - - PLATFORM: redhat-9 - BLOCK: 14736:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cax41 - STRATEGY: multisession - - PLATFORM: redhat-8 - BLOCK: 14065:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cax41 - STRATEGY: multisession - - PLATFORM: ubuntu-2404 - BLOCK: 16953:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cax41 - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - BLOCK: 19280:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cax41 - STRATEGY: multisession - - PLATFORM: alpine-320 - BLOCK: 15871:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cax41 - STRATEGY: multisession - # ### AMD - - PLATFORM: redhat-9 - BLOCK: 17739:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cpx51 - STRATEGY: multisession - - PLATFORM: redhat-8 - BLOCK: 17534:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cpx51 - STRATEGY: multisession - - PLATFORM: ubuntu-2404 - BLOCK: 20732:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cpx51 - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - BLOCK: 21389:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cpx51 - STRATEGY: multisession - - PLATFORM: alpine-320 - BLOCK: 15901:length(pkgs) - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - INSTANCE_TYPE: cpx51 - STRATEGY: multisession -steps: - - name: 'Warming up' - image: busybox:latest - failure: ignore - commands: - - sleep 1 - - echo "Warmed Up" - backend_options: - kubernetes: - ### PROD-parallel - resources: - requests: - memory: ${MEMORY} - cpu: 12000m - limits: - memory: ${MEMORY} - cpu: 16000m - ### PROD-sequential - # resources: - # requests: - # memory: ${MEMORY} - # cpu: 2000m - # limits: - # memory: 29Gi - # cpu: 4000m - # DEBUG - # resources: - # requests: - # memory: 4Gi - # cpu: 2000m - # limits: - # memory: 4Gi - # cpu: 2000m - nodeSelector: - kubernetes.io/arch: '${ARCH}' - node.kubernetes.io/instance-type: '${INSTANCE_TYPE}' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.woodpecker/warm_up_process_cran_updates.yaml b/.woodpecker/warm_up_process_cran_updates.yaml deleted file mode 100644 index 4b4a913..0000000 --- a/.woodpecker/warm_up_process_cran_updates.yaml +++ /dev/null @@ -1,82 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates - - event: manual - evaluate: 'task == "process-cran-updates"' - -skip_clone: true - -matrix: - include: - ### ARM - # - PLATFORM: redhat-9 - # ARCH: arm64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # - PLATFORM: redhat-8 - # ARCH: arm64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # - PLATFORM: ubuntu-2404 - # ARCH: arm64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # - PLATFORM: ubuntu-2204 - # ARCH: arm64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # ### AMD - # - PLATFORM: redhat-9 - # ARCH: amd64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # - PLATFORM: redhat-8 - # ARCH: amd64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - # - PLATFORM: ubuntu-2404 - # ARCH: amd64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - - PLATFORM: ubuntu-2204 - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - # - PLATFORM: alpine-320 - # ARCH: amd64 - # NCPUS: 2 - # MEMORY: 29Gi - # STRATEGY: multisession - -steps: - - name: 'Warming up' - image: busybox:latest - failure: ignore - commands: - - sleep 1 - - echo "Warmed Up" - - backend_options: - kubernetes: - resources: - requests: - memory: 12Gi - cpu: 700m - limits: - memory: 14Gi - cpu: 4000m - nodeSelector: - kubernetes.io/arch: '${ARCH}' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule'