From 78b58c85921ee10d89e688756696dbc0aaf02fd2 Mon Sep 17 00:00:00 2001 From: pat-s Date: Mon, 21 Oct 2024 12:42:14 +0200 Subject: [PATCH] process updates --- .woodpecker/build.yaml | 5 +- .woodpecker/process_cran_updates.yaml | 99 ++++++++++--------- .woodpecker/warm_up_build.yaml | 3 +- .woodpecker/warm_up_process_cran_updates.yaml | 86 ++++++++-------- 4 files changed, 100 insertions(+), 93 deletions(-) diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index 7f9580f..4a9f9c9 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -1,5 +1,6 @@ when: - event: [manual] + - event: manual + evaluate: 'task == "build"' skip_clone: true @@ -73,7 +74,7 @@ matrix: INSTANCE_TYPE: cpx51 STRATEGY: multisession - PLATFORM: alpine-320 - BLOCK: 15901:length(pkgs) + BLOCK: 15:length(pkgs) ARCH: amd64 NCPUS: 2 MEMORY: 29Gi diff --git a/.woodpecker/process_cran_updates.yaml b/.woodpecker/process_cran_updates.yaml index 93ba3e5..b0d4151 100644 --- a/.woodpecker/process_cran_updates.yaml +++ b/.woodpecker/process_cran_updates.yaml @@ -1,6 +1,8 @@ when: - event: [cron] cron: process-cran-updates + - event: manual + evaluate: 'task == "process-cran-updates"' skip_clone: true @@ -10,52 +12,54 @@ depends_on: matrix: include: ### ARM - - PLATFORM: redhat-9 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: redhat-8 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2404 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - ### AMD - - PLATFORM: redhat-9 - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: redhat-8 - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2404 - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession + # - PLATFORM: redhat-9 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: redhat-8 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2404 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2204 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # ### AMD + # - PLATFORM: redhat-9 + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: redhat-8 + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2404 + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession - PLATFORM: ubuntu-2204 ARCH: amd64 NCPUS: 2 MEMORY: 29Gi STRATEGY: multisession - - PLATFORM: alpine-320 - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession + INSTANCE_TYPE: cpx51 + INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 31) + # - PLATFORM: alpine-320 + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession steps: - name: 'Processing Updates: (${PLATFORM})' @@ -73,26 +77,24 @@ steps: commands: # download latest package version (avoid having to rebuild image every time) - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /opt/R/*/lib/R/library/RPostgres /mnt/cache/R-pkgs/00LOCK-RPostgres + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres # for some reason paks pkgcache is often corrupt - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ - R -q -e 'pak::meta_update()' - - rm -rf /mnt/cache/R-pkgs/RPostgres /mnt/cache/pkgcache/R/pkgcache - - R -q -e 'install.packages("RPostgres", quiet = TRUE, repos = structure(c(devxy = "https://cran.devxy.io/${ARCH}/${PKGS_ID}/latest", CRAN = "https://cloud.r-project.org")))' - R CMD INSTALL . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error", repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]][${BLOCK}]; pkgs = setdiff(pkgs, c("Rsymphony", "PortfolioAnalytics", "ROI.plugin.symphony", "RInno", "gpuR", "BRugs", "clrng", "Rcplex", "ROI.plugin.cplex", "ROracle")); library(bincraftR); future::plan("sequential"); options(progressr.enable = TRUE); progressr::handlers("cli"); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = lubridate::interval(lubridate::today() - 3, lubridate::today() - 3), platform = "${PLATFORM}", process_updated = TRUE, process_new = TRUE)' + - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error", repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]][${BLOCK}]; pkgs = setdiff(pkgs, c("biplotbootGUI", "Rsymphony", "PortfolioAnalytics", "ROI.plugin.symphony", "adea", "RInno", "skedastic", "RPushbullet", "SimNPH", "SimDesign","simCAT", "SIAmodules", "shinytest2", "shinyIRT", "ShinyItemAnalysis", "shiny.benchmark", "scDIFtest", "RSP", "ROracle", "ROI.plugin.qpoases", "ROI.plugin.cplex", "RmecabKo", "rmarchingcubes", "RcppMeCab", "Rcplex", "ravetools", "Rbeast", "Rblpapi", "RcppEnsmallen", "qtkit", "priorCON", "prior3D", "PerFit", "outlierensembles", "nomnoml", "mxfda", "MultBiplotR", "mstDIF", "mdsr", "makepipe", "lordif", "KSgeneral", "kequate", "jstager", "jrt", "irtreliability", "irtQ", "irtpwr", "irtawsi", "GRShiny", "gpuR", "googletraffic", "giacR", "gamstransfer", "aflps", "flow", "filters", "faoutlier", "equateIRT", "EnrichIntersect", "EFA.dimensions", "DFIT", "D3mirt", "ctgdist", "CoTiMA", "ConvertPar", "clrng", "bscui", "BifactorIndicesCalculator", "autoFC", "airt")); library(bincraftR); future::plan("sequential"); options(progressr.enable = TRUE); progressr::handlers("cli"); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = ${INTERVAL}, platform = "${PLATFORM}", process_updated = TRUE, process_new = TRUE)' backend_options: kubernetes: ### PROD resources: requests: - memory: 12Gi - cpu: 700m + memory: ${MEMORY} + cpu: 4000m limits: - memory: 14Gi + memory: ${MEMORY} cpu: 4000m # DEBUG # resources: @@ -104,6 +106,7 @@ steps: # cpu: 2000m nodeSelector: kubernetes.io/arch: '${ARCH}' + node.kubernetes.io/instance-type: '${INSTANCE_TYPE}' tolerations: - key: 'CI' operator: 'Equal' diff --git a/.woodpecker/warm_up_build.yaml b/.woodpecker/warm_up_build.yaml index b4b571b..ae3a156 100644 --- a/.woodpecker/warm_up_build.yaml +++ b/.woodpecker/warm_up_build.yaml @@ -1,5 +1,6 @@ when: - event: [manual] + - event: manual + evaluate: 'task == "build"' skip_clone: true diff --git a/.woodpecker/warm_up_process_cran_updates.yaml b/.woodpecker/warm_up_process_cran_updates.yaml index 3a8ec34..4b4a913 100644 --- a/.woodpecker/warm_up_process_cran_updates.yaml +++ b/.woodpecker/warm_up_process_cran_updates.yaml @@ -1,65 +1,67 @@ when: - event: [cron] cron: process-cran-updates + - event: manual + evaluate: 'task == "process-cran-updates"' skip_clone: true matrix: include: ### ARM - - PLATFORM: redhat-9 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: redhat-8 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2404 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2204 - ARCH: arm64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - ### AMD - - PLATFORM: redhat-9 - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: redhat-8 - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession - - PLATFORM: ubuntu-2404 - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession + # - PLATFORM: redhat-9 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: redhat-8 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2404 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2204 + # ARCH: arm64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # ### AMD + # - PLATFORM: redhat-9 + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: redhat-8 + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession + # - PLATFORM: ubuntu-2404 + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession - PLATFORM: ubuntu-2204 ARCH: amd64 NCPUS: 2 MEMORY: 29Gi STRATEGY: multisession - - PLATFORM: alpine-320 - ARCH: amd64 - NCPUS: 2 - MEMORY: 29Gi - STRATEGY: multisession + # - PLATFORM: alpine-320 + # ARCH: amd64 + # NCPUS: 2 + # MEMORY: 29Gi + # STRATEGY: multisession steps: - name: 'Warming up' image: busybox:latest failure: ignore commands: - - sleep 200 + - sleep 1 - echo "Warmed Up" backend_options: