From 93eec1418c59f2d4797a5e10d055763e1652d3af Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 10 Nov 2024 16:54:37 +0100 Subject: [PATCH] individual CI files for package index update --- .woodpecker/build-alpine-320-amd64.yaml | 2 +- .woodpecker/build-redhat-8-amd64.yaml | 2 +- .woodpecker/build-redhat-9-amd64.yaml | 2 +- .woodpecker/build-ubuntu-2204-amd64.yaml | 2 +- ...update-package-index-alpine-320-amd64.yaml | 2 + .../update-package-index-redhat-8-amd64.yaml | 2 + .../update-package-index-redhat-9-amd64.yaml | 2 + ...pdate-package-index-ubuntu-2204-amd64.yaml | 2 + ...pdate-package-index-ubuntu-2404-amd64.yaml | 2 + .woodpecker/update-package-index.yaml | 88 ------------------- 10 files changed, 14 insertions(+), 92 deletions(-) delete mode 100644 .woodpecker/update-package-index.yaml diff --git a/.woodpecker/build-alpine-320-amd64.yaml b/.woodpecker/build-alpine-320-amd64.yaml index bb4f398..3af9534 100644 --- a/.woodpecker/build-alpine-320-amd64.yaml +++ b/.woodpecker/build-alpine-320-amd64.yaml @@ -22,7 +22,7 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache PLATFORM: alpine-320 - BLOCK: 2436:2993 + BLOCK: 2991:2993 ARCH: amd64 NCPUS: 4 # memory: 20Gi diff --git a/.woodpecker/build-redhat-8-amd64.yaml b/.woodpecker/build-redhat-8-amd64.yaml index e295e04..8935b86 100644 --- a/.woodpecker/build-redhat-8-amd64.yaml +++ b/.woodpecker/build-redhat-8-amd64.yaml @@ -22,7 +22,7 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache PLATFORM: redhat-8 - BLOCK: 1561:1833 + BLOCK: 1833:1833 ARCH: amd64 NCPUS: 4 # MEMORY: 14Gi diff --git a/.woodpecker/build-redhat-9-amd64.yaml b/.woodpecker/build-redhat-9-amd64.yaml index 5e19840..14ab5e2 100644 --- a/.woodpecker/build-redhat-9-amd64.yaml +++ b/.woodpecker/build-redhat-9-amd64.yaml @@ -46,7 +46,7 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache PLATFORM: redhat-9 - BLOCK: 1664:2428 + BLOCK: 2394:2428 ARCH: amd64 NCPUS: 4 INSTANCE_TYPE: AX42 diff --git a/.woodpecker/build-ubuntu-2204-amd64.yaml b/.woodpecker/build-ubuntu-2204-amd64.yaml index 675696f..ae74812 100644 --- a/.woodpecker/build-ubuntu-2204-amd64.yaml +++ b/.woodpecker/build-ubuntu-2204-amd64.yaml @@ -46,7 +46,7 @@ steps: # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) # - Rglpk # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(devxy = 'https://cran.devxy.io/amd64/jammy/latest', CRAN = 'https://cloud.r-project.org'))); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'redatam', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'ubuntu-2204'))" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(devxy = 'https://cran.devxy.io/amd64/jammy/latest', CRAN = 'https://cloud.r-project.org'))); pkgs <- c('webshot2', 'tabledown', 'TestAnaAPP', 'trekcolors', 'Spectran', 'SimNPH', 'RPushbullet', 'SimDesign', 'simCAT', 'SIAmodules', 'shinytest2', 'ShinyItemAnalysis', 'shinyIRT', 'shiny.benchmark', 'scDIFtest', 'RSP', 'qtkit', 'PROsetta', 'prettifyAddins', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtGUI', 'irtawsi', 'GRShiny', 'GPCMlasso', 'googletraffic', 'giacR', 'flps', 'flow', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt'); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'ubuntu-2204'))" ### Manual pkg builds # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 # RInno: Windows-only diff --git a/.woodpecker/update-package-index-alpine-320-amd64.yaml b/.woodpecker/update-package-index-alpine-320-amd64.yaml index 6ad796a..4737d0d 100644 --- a/.woodpecker/update-package-index-alpine-320-amd64.yaml +++ b/.woodpecker/update-package-index-alpine-320-amd64.yaml @@ -1,4 +1,6 @@ when: + - event: [cron] + cron: update-package-index-alpine-320-amd64 - event: [cron] cron: update-package-index - event: manual diff --git a/.woodpecker/update-package-index-redhat-8-amd64.yaml b/.woodpecker/update-package-index-redhat-8-amd64.yaml index 53e38fc..8df7702 100644 --- a/.woodpecker/update-package-index-redhat-8-amd64.yaml +++ b/.woodpecker/update-package-index-redhat-8-amd64.yaml @@ -1,6 +1,8 @@ when: - event: [cron] cron: update-package-index-redhat-8-amd64 + - event: [cron] + cron: update-package-index - event: manual evaluate: 'update_package_index == "redhat-8-amd64"' diff --git a/.woodpecker/update-package-index-redhat-9-amd64.yaml b/.woodpecker/update-package-index-redhat-9-amd64.yaml index e654b69..bd81252 100644 --- a/.woodpecker/update-package-index-redhat-9-amd64.yaml +++ b/.woodpecker/update-package-index-redhat-9-amd64.yaml @@ -1,6 +1,8 @@ when: - event: [cron] cron: update-package-index-redhat-9-amd64 + - event: [cron] + cron: update-package-index - event: manual evaluate: 'update_package_index == "redhat-9-amd64"' diff --git a/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml b/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml index 4e645bc..adf4e73 100644 --- a/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml +++ b/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml @@ -1,6 +1,8 @@ when: - event: [cron] cron: update-package-index-ubuntu-2204-amd64 + - event: [cron] + cron: update-package-index - event: manual evaluate: 'update_package_index == "ubuntu-2204-amd64"' diff --git a/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml b/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml index 38599f3..5776d45 100644 --- a/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml +++ b/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml @@ -1,6 +1,8 @@ when: - event: [cron] cron: update-package-index-ubuntu-2404-amd64 + - event: [cron] + cron: update-package-index - event: manual evaluate: 'update_package_index == "ubuntu-2404-amd64"' diff --git a/.woodpecker/update-package-index.yaml b/.woodpecker/update-package-index.yaml deleted file mode 100644 index 493be78..0000000 --- a/.woodpecker/update-package-index.yaml +++ /dev/null @@ -1,88 +0,0 @@ -when: - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'task == "update-package-index"' - -skip_clone: true - -matrix: - include: - - PLATFORM: redhat-9 - ARCH: arm64 - - PLATFORM: redhat-8 - ARCH: arm64 - - PLATFORM: ubuntu-2404 - ARCH: arm64 - - PLATFORM: ubuntu-2204 - ARCH: arm64 - - PLATFORM: alpine-320 - ARCH: arm64 - - PLATFORM: redhat-9 - ARCH: amd64 - - PLATFORM: redhat-8 - ARCH: amd64 - - PLATFORM: ubuntu-2404 - ARCH: amd64 - - PLATFORM: ubuntu-2204 - ARCH: amd64 - - PLATFORM: alpine-320 - ARCH: amd64 -steps: - - name: Upload PACKAGES files - secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] - image: docker.io/devxygmbh/${ARCH}-binaries-r-${PLATFORM} - pull: true - environment: - GIT_USER: pat-s - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - ${ARCH}-binaries-r-dep-cache-${PLATFORM}:/mnt/cache - commands: - # download latest package version (avoid having to rebuild image every time) - # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . - - rm -rf /mnt/cache/packages/* - - rm -rf /mnt/cache/R-pkgs/stringr - - R CMD INSTALL . - - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' - backend_options: - kubernetes: - resources: - requests: - memory: 2000Mi - cpu: 2000m - limits: - memory: 2000Mi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: '${ARCH}' - - - name: "ntfy" - image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 - when: - - status: [failure] - settings: - url: https://ntfy.devxy.io/ci-builds - token: - from_secret: ntfy_token - title: "CI '${CI_REPO}' failed" - actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; - click: ${CI_BUILD_LINK} - message: | - Workflow: ${CI_WORKFLOW_NAME/\n//} - Commit message: ${CI_COMMIT_MESSAGE/\n//} - Commit Author: ${CI_COMMIT_AUTHOR/\n//} - backend_options: - kubernetes: - resources: - requests: - memory: 100Mi - cpu: 0m - limits: - memory: 300Mi - cpu: 200m