136 lines
5 KiB
YAML
136 lines
5 KiB
YAML
when:
|
|
- event: [cron]
|
|
cron: process-cran-updates
|
|
|
|
skip_clone: true
|
|
|
|
depends_on:
|
|
- warm_up_process_cran_updates
|
|
|
|
matrix:
|
|
include:
|
|
### ARM
|
|
- PLATFORM: redhat-9
|
|
ARCH: arm64
|
|
NCPUS: 2
|
|
MEMORY: 29Gi
|
|
STRATEGY: multisession
|
|
- PLATFORM: redhat-8
|
|
ARCH: arm64
|
|
NCPUS: 2
|
|
MEMORY: 29Gi
|
|
STRATEGY: multisession
|
|
- PLATFORM: ubuntu-2404
|
|
ARCH: arm64
|
|
NCPUS: 2
|
|
MEMORY: 29Gi
|
|
STRATEGY: multisession
|
|
- PLATFORM: ubuntu-2204
|
|
ARCH: arm64
|
|
NCPUS: 2
|
|
MEMORY: 29Gi
|
|
STRATEGY: multisession
|
|
### AMD
|
|
- PLATFORM: redhat-9
|
|
ARCH: amd64
|
|
NCPUS: 2
|
|
MEMORY: 29Gi
|
|
STRATEGY: multisession
|
|
- PLATFORM: redhat-8
|
|
ARCH: amd64
|
|
NCPUS: 2
|
|
MEMORY: 29Gi
|
|
STRATEGY: multisession
|
|
- PLATFORM: ubuntu-2404
|
|
ARCH: amd64
|
|
NCPUS: 2
|
|
MEMORY: 29Gi
|
|
STRATEGY: multisession
|
|
- PLATFORM: ubuntu-2204
|
|
ARCH: amd64
|
|
NCPUS: 2
|
|
MEMORY: 29Gi
|
|
STRATEGY: multisession
|
|
- PLATFORM: alpine-320
|
|
ARCH: amd64
|
|
NCPUS: 2
|
|
MEMORY: 29Gi
|
|
STRATEGY: multisession
|
|
|
|
steps:
|
|
- name: 'Processing Updates: (${PLATFORM})'
|
|
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
|
|
image: docker.io/devxygmbh/${ARCH}-binaries-r-${PLATFORM}
|
|
pull: true
|
|
environment:
|
|
GIT_USER: pat-s
|
|
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
|
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
|
CCACHE_DIR: /mnt/cache/ccache
|
|
volumes:
|
|
- ${ARCH}-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
|
|
commands:
|
|
# download latest package version (avoid having to rebuild image every time)
|
|
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
|
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /opt/R/*/lib/R/library/RPostgres /mnt/cache/R-pkgs/00LOCK-RPostgres
|
|
# for some reason paks pkgcache is often corrupt
|
|
- rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/
|
|
- R -q -e 'pak::meta_update()'
|
|
- rm -rf /mnt/cache/R-pkgs/RPostgres /mnt/cache/pkgcache/R/pkgcache
|
|
- R -q -e 'install.packages("RPostgres", quiet = TRUE, repos = structure(c(devxy = "https://cran.devxy.io/${ARCH}/${PKGS_ID}/latest", CRAN = "https://cloud.r-project.org")))'
|
|
- R CMD INSTALL .
|
|
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
|
# set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point
|
|
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
|
|
- xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error", repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]][${BLOCK}]; pkgs = setdiff(pkgs, c("Rsymphony", "PortfolioAnalytics", "ROI.plugin.symphony", "RInno", "gpuR", "BRugs", "clrng", "Rcplex", "ROI.plugin.cplex", "ROracle")); library(bincraftR); future::plan("sequential"); options(progressr.enable = TRUE); progressr::handlers("cli"); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = lubridate::interval(lubridate::today() - 3, lubridate::today() - 3), platform = "${PLATFORM}", process_updated = TRUE, process_new = TRUE)'
|
|
backend_options:
|
|
kubernetes:
|
|
### PROD
|
|
resources:
|
|
requests:
|
|
memory: 12Gi
|
|
cpu: 700m
|
|
limits:
|
|
memory: 14Gi
|
|
cpu: 4000m
|
|
# DEBUG
|
|
# resources:
|
|
# requests:
|
|
# memory: 4Gi
|
|
# cpu: 2000m
|
|
# limits:
|
|
# memory: 4Gi
|
|
# cpu: 2000m
|
|
nodeSelector:
|
|
kubernetes.io/arch: '${ARCH}'
|
|
tolerations:
|
|
- key: 'CI'
|
|
operator: 'Equal'
|
|
value: 'true'
|
|
effect: 'NoSchedule'
|
|
|
|
- name: 'ntfy'
|
|
image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
|
|
when:
|
|
- status: [failure]
|
|
settings:
|
|
url: https://ntfy.devxy.io/ci-builds
|
|
token:
|
|
from_secret: ntfy_token
|
|
title: "CI '${CI_REPO}' failed"
|
|
actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
|
|
click: ${CI_BUILD_LINK}
|
|
message: |
|
|
Workflow: ${CI_WORKFLOW_NAME/\n//}
|
|
Commit message: ${CI_COMMIT_MESSAGE/\n//}
|
|
Commit Author: ${CI_COMMIT_AUTHOR/\n//}
|
|
backend_options:
|
|
kubernetes:
|
|
resources:
|
|
requests:
|
|
memory: 100Mi
|
|
cpu: 0m
|
|
limits:
|
|
memory: 300Mi
|
|
cpu: 200m
|