devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 11 Pull requests Releases Activity

process updates

Browse source
This commit is contained in:
Patrick Schratz 2024-10-21 12:42:14 +02:00
commit 78b58c8592
Signed by: pat-s
GPG key ID: 3C6318841EF78925
4 changed files with 100 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.woodpecker/build.yaml
View file

@ -1,5 +1,6 @@
when: when:
event: [manual] - event: manual
evaluate: 'task == "build"'
skip_clone: true skip_clone: true
@ -73,7 +74,7 @@ matrix:
INSTANCE_TYPE: cpx51 INSTANCE_TYPE: cpx51
STRATEGY: multisession STRATEGY: multisession
- PLATFORM: alpine-320 - PLATFORM: alpine-320
BLOCK: 15901:length(pkgs) BLOCK: 15:length(pkgs)
ARCH: amd64 ARCH: amd64
NCPUS: 2 NCPUS: 2
MEMORY: 29Gi MEMORY: 29Gi

99
.woodpecker/process_cran_updates.yaml
View file

@ -1,6 +1,8 @@
when: when:
- event: [cron] - event: [cron]
cron: process-cran-updates cron: process-cran-updates
- event: manual
evaluate: 'task == "process-cran-updates"'
skip_clone: true skip_clone: true
@ -10,52 +12,54 @@ depends_on:
matrix: matrix:
include: include:
### ARM ### ARM
- PLATFORM: redhat-9 # - PLATFORM: redhat-9
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: redhat-8 # - PLATFORM: redhat-8
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2404 # - PLATFORM: ubuntu-2404
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2204 # - PLATFORM: ubuntu-2204
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
### AMD # ### AMD
- PLATFORM: redhat-9 # - PLATFORM: redhat-9
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: redhat-8 # - PLATFORM: redhat-8
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2404 # - PLATFORM: ubuntu-2404
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2204 - PLATFORM: ubuntu-2204
ARCH: amd64 ARCH: amd64
NCPUS: 2 NCPUS: 2
MEMORY: 29Gi MEMORY: 29Gi
STRATEGY: multisession STRATEGY: multisession
- PLATFORM: alpine-320 INSTANCE_TYPE: cpx51
ARCH: amd64 INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 31)
NCPUS: 2 # - PLATFORM: alpine-320
MEMORY: 29Gi # ARCH: amd64
STRATEGY: multisession # NCPUS: 2
# MEMORY: 29Gi
# STRATEGY: multisession
steps: steps:
- name: 'Processing Updates: (${PLATFORM})' - name: 'Processing Updates: (${PLATFORM})'
@ -73,26 +77,24 @@ steps:
commands: commands:
# download latest package version (avoid having to rebuild image every time) # download latest package version (avoid having to rebuild image every time)
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
- rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /opt/R/*/lib/R/library/RPostgres /mnt/cache/R-pkgs/00LOCK-RPostgres - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-RPostgres
# for some reason paks pkgcache is often corrupt # for some reason paks pkgcache is often corrupt
- rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/ - rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/
- R -q -e 'pak::meta_update()' - R -q -e 'pak::meta_update()'
- rm -rf /mnt/cache/R-pkgs/RPostgres /mnt/cache/pkgcache/R/pkgcache
- R -q -e 'install.packages("RPostgres", quiet = TRUE, repos = structure(c(devxy = "https://cran.devxy.io/${ARCH}/${PKGS_ID}/latest", CRAN = "https://cloud.r-project.org")))'
- R CMD INSTALL . - R CMD INSTALL .
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
# set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
- xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error", repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]][${BLOCK}]; pkgs = setdiff(pkgs, c("Rsymphony", "PortfolioAnalytics", "ROI.plugin.symphony", "RInno", "gpuR", "BRugs", "clrng", "Rcplex", "ROI.plugin.cplex", "ROracle")); library(bincraftR); future::plan("sequential"); options(progressr.enable = TRUE); progressr::handlers("cli"); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = lubridate::interval(lubridate::today() - 3, lubridate::today() - 3), platform = "${PLATFORM}", process_updated = TRUE, process_new = TRUE)' - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error", repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]][${BLOCK}]; pkgs = setdiff(pkgs, c("biplotbootGUI", "Rsymphony", "PortfolioAnalytics", "ROI.plugin.symphony", "adea", "RInno", "skedastic", "RPushbullet", "SimNPH", "SimDesign","simCAT", "SIAmodules", "shinytest2", "shinyIRT", "ShinyItemAnalysis", "shiny.benchmark", "scDIFtest", "RSP", "ROracle", "ROI.plugin.qpoases", "ROI.plugin.cplex", "RmecabKo", "rmarchingcubes", "RcppMeCab", "Rcplex", "ravetools", "Rbeast", "Rblpapi", "RcppEnsmallen", "qtkit", "priorCON", "prior3D", "PerFit", "outlierensembles", "nomnoml", "mxfda", "MultBiplotR", "mstDIF", "mdsr", "makepipe", "lordif", "KSgeneral", "kequate", "jstager", "jrt", "irtreliability", "irtQ", "irtpwr", "irtawsi", "GRShiny", "gpuR", "googletraffic", "giacR", "gamstransfer", "aflps", "flow", "filters", "faoutlier", "equateIRT", "EnrichIntersect", "EFA.dimensions", "DFIT", "D3mirt", "ctgdist", "CoTiMA", "ConvertPar", "clrng", "bscui", "BifactorIndicesCalculator", "autoFC", "airt")); library(bincraftR); future::plan("sequential"); options(progressr.enable = TRUE); progressr::handlers("cli"); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); process_cran_updates(interval = ${INTERVAL}, platform = "${PLATFORM}", process_updated = TRUE, process_new = TRUE)'
backend_options: backend_options:
kubernetes: kubernetes:
### PROD ### PROD
resources: resources:
requests: requests:
memory: 12Gi memory: ${MEMORY}
cpu: 700m cpu: 4000m
limits: limits:
memory: 14Gi memory: ${MEMORY}
cpu: 4000m cpu: 4000m
# DEBUG # DEBUG
# resources: # resources:
@ -104,6 +106,7 @@ steps:
# cpu: 2000m # cpu: 2000m
nodeSelector: nodeSelector:
kubernetes.io/arch: '${ARCH}' kubernetes.io/arch: '${ARCH}'
node.kubernetes.io/instance-type: '${INSTANCE_TYPE}'
tolerations: tolerations:
- key: 'CI' - key: 'CI'
operator: 'Equal' operator: 'Equal'

3
.woodpecker/warm_up_build.yaml
View file

@ -1,5 +1,6 @@
when: when:
event: [manual] - event: manual
evaluate: 'task == "build"'
skip_clone: true skip_clone: true

86
.woodpecker/warm_up_process_cran_updates.yaml
View file

@ -1,65 +1,67 @@
when: when:
- event: [cron] - event: [cron]
cron: process-cran-updates cron: process-cran-updates
- event: manual
evaluate: 'task == "process-cran-updates"'
skip_clone: true skip_clone: true
matrix: matrix:
include: include:
### ARM ### ARM
- PLATFORM: redhat-9 # - PLATFORM: redhat-9
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: redhat-8 # - PLATFORM: redhat-8
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2404 # - PLATFORM: ubuntu-2404
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2204 # - PLATFORM: ubuntu-2204
ARCH: arm64 # ARCH: arm64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
### AMD # ### AMD
- PLATFORM: redhat-9 # - PLATFORM: redhat-9
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: redhat-8 # - PLATFORM: redhat-8
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2404 # - PLATFORM: ubuntu-2404
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
- PLATFORM: ubuntu-2204 - PLATFORM: ubuntu-2204
ARCH: amd64 ARCH: amd64
NCPUS: 2 NCPUS: 2
MEMORY: 29Gi MEMORY: 29Gi
STRATEGY: multisession STRATEGY: multisession
- PLATFORM: alpine-320 # - PLATFORM: alpine-320
ARCH: amd64 # ARCH: amd64
NCPUS: 2 # NCPUS: 2
MEMORY: 29Gi # MEMORY: 29Gi
STRATEGY: multisession # STRATEGY: multisession
steps: steps:
- name: 'Warming up' - name: 'Warming up'
image: busybox:latest image: busybox:latest
failure: ignore failure: ignore
commands: commands:
- sleep 200 - sleep 1
- echo "Warmed Up" - echo "Warmed Up"
backend_options: backend_options: