devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 10 Pull requests 1 Releases Activity

chore: fix temporary hickup for pkg redatam

Browse source
This commit is contained in:
Patrick Schratz 2024-11-07 10:11:23 +01:00
commit 9953ebe31f
Signed by: pat-s
GPG key ID: 3C6318841EF78925
16 changed files with 358 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.woodpecker/build-alpine-320-amd64.yaml
View file

@ -22,10 +22,10 @@ steps:
R_LIBS_USER: /mnt/cache/R-pkgs
CCACHE_DIR: /mnt/cache/ccache
PLATFORM: alpine-320
BLOCK: 1880:2993
BLOCK: 2436:2993
ARCH: amd64
NCPUS: 4
# MEMORY: 18Gi
# memory: 20Gi
# INSTANCE_TYPE: ccx23
STRATEGY: sequential
# volumes:
@ -85,7 +85,7 @@ steps:
memory: 5Gi
cpu: 2000m
limits:
memory: 18Gi
memory: 20Gi
cpu: 4000m
nodeSelector:
kubernetes.io/arch: amd64

4
.woodpecker/build-redhat-8-amd64.yaml
View file

@ -22,7 +22,7 @@ steps:
R_LIBS_USER: /mnt/cache/R-pkgs
CCACHE_DIR: /mnt/cache/ccache
PLATFORM: redhat-8
BLOCK: 737:1833
BLOCK: 1561:1833
ARCH: amd64
NCPUS: 4
# MEMORY: 14Gi
@ -86,7 +86,7 @@ steps:
memory: 5Gi
cpu: 2000m
limits:
memory: 14Gi
memory: 20Gi
cpu: 4000m
### PROD-sequential
# resources:

5
.woodpecker/build-redhat-9-amd64.yaml
View file

@ -46,10 +46,9 @@ steps:
R_LIBS_USER: /mnt/cache/R-pkgs
CCACHE_DIR: /mnt/cache/ccache
PLATFORM: redhat-9
BLOCK: 715:2428
BLOCK: 1664:2428
ARCH: amd64
NCPUS: 4
MEMORY: 14Gi
INSTANCE_TYPE: AX42
STRATEGY: sequential
# volumes:
@ -110,7 +109,7 @@ steps:
memory: 5Gi
cpu: 2000m
limits:
memory: 14Gi
memory: 20Gi
cpu: 4000m
### PROD-sequential
# resources:

2
.woodpecker/build-ubuntu-2204-amd64.yaml
View file

@ -22,7 +22,7 @@ steps:
R_LIBS_USER: /mnt/cache/R-pkgs
CCACHE_DIR: /mnt/cache/ccache
PLATFORM: ubuntu-2204
BLOCK: 2128:4755
BLOCK: 4223:4755
ARCH: amd64
NCPUS: 4
# MEMORY: 14Gi

2
.woodpecker/build-ubuntu-2404-amd64.yaml
View file

@ -22,7 +22,7 @@ steps:
R_LIBS_USER: /mnt/cache/R-pkgs
CCACHE_DIR: /mnt/cache/ccache
PLATFORM: ubuntu-2404
BLOCK: 1886:4298
BLOCK: 3025:4298
ARCH: amd64
NCPUS: 4
MEMORY: 14Gi

4
.woodpecker/process-updates-alpine-320-amd64.yaml
View file

@ -43,10 +43,10 @@ steps:
### PROD
resources:
requests:
memory: 14Gi
memory: 18Gi
cpu: 3000m
limits:
memory: 14Gi
memory: 18Gi
cpu: 3000m
nodeSelector:
kubernetes.io/arch: amd64

10
.woodpecker/process-updates-redhat-8-amd64.yaml
View file

@ -19,10 +19,10 @@ steps:
# kubernetes:
# resources:
# requests:
# memory: 14Gi
# memory: 18Gi
# cpu: 3000m
# limits:
# memory: 14Gi
# memory: 18Gi
# cpu: 3000m
# nodeSelector:
# kubernetes.io/arch: amd64
@ -46,7 +46,7 @@ steps:
PLATFORM: redhat-8
ARCH: amd64
NCPUS: 2
# MEMORY: 14Gi
# MEMORY: 18Gi
STRATEGY: sequential
# INSTANCE_TYPE: cpx41
INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3)
@ -69,10 +69,10 @@ steps:
### PROD
resources:
requests:
memory: 14Gi
memory: 18Gi
cpu: 3000m
limits:
memory: 14Gi
memory: 18Gi
cpu: 3000m
nodeSelector:
kubernetes.io/arch: amd64

6
.woodpecker/process-updates-redhat-9-amd64.yaml
View file

@ -22,7 +22,7 @@ steps:
PLATFORM: redhat-9
ARCH: amd64
NCPUS: 2
MEMORY: 14Gi
MEMORY: 18Gi
STRATEGY: sequential
INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3)
# volumes:
@ -44,10 +44,10 @@ steps:
### PROD
resources:
requests:
memory: 14Gi
memory: 18Gi
cpu: 3000m
limits:
memory: 14Gi
memory: 18Gi
cpu: 3000m
nodeSelector:
kubernetes.io/arch: amd64

38
.woodpecker/process-updates-ubuntu-2204-amd64.yaml
View file

@ -9,30 +9,6 @@ when:
skip_clone: true
steps:
# - name: 'Warming up'
# image: busybox:latest
# failure: ignore
# commands:
# - sleep 300
# - echo "Warmed Up"
# backend_options:
# kubernetes:
# resources:
# requests:
# memory: 14Gi
# cpu: 3000m
# limits:
# memory: 14Gi
# cpu: 3000m
# nodeSelector:
# kubernetes.io/arch: amd64
# node.kubernetes.io/instance-type: ccx23
# tolerations:
# - key: 'CI'
# operator: 'Equal'
# value: 'true'
# effect: 'NoSchedule'
- name: 'Processing Updates'
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2204
@ -46,7 +22,7 @@ steps:
PLATFORM: ubuntu-2204
ARCH: amd64
NCPUS: 2
MEMORY: 14Gi
MEMORY: 18Gi
STRATEGY: sequential
# INSTANCE_TYPE: ccx23
INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3)
@ -69,19 +45,11 @@ steps:
### PROD
resources:
requests:
memory: 14Gi
memory: 18Gi
cpu: 3000m
limits:
memory: 14Gi
memory: 18Gi
cpu: 3000m
# DEBUG
# resources:
# requests:
# memory: 4Gi
# cpu: 2000m
# limits:
# memory: 4Gi
# cpu: 2000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: AX42

8
.woodpecker/process-updates-ubuntu-2404-amd64.yaml
View file

@ -24,8 +24,8 @@ steps:
NCPUS: 1
STRATEGY: sequential
INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3)
volumes:
- amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache
# volumes:
# - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache
commands:
# download latest package version (avoid having to rebuild image every time)
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
@ -43,10 +43,10 @@ steps:
### PROD
resources:
requests:
memory: 14Gi
memory: 18Gi
cpu: 3000m
limits:
memory: 14Gi
memory: 18Gi
cpu: 3000m
nodeSelector:
kubernetes.io/arch: amd64

66
.woodpecker/update-package-index-alpine-320-amd64.yaml Normal file
View file

@ -0,0 +1,66 @@
when:
- event: [cron]
cron: update-package-index
- event: manual
evaluate: 'update_package_index == "alpine-320-amd64"'
skip_clone: true
steps:
- name: Upload PACKAGES files
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
image: docker.io/devxygmbh/amd64-binaries-r-alpine-320
pull: true
environment:
GIT_USER: pat-s
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
R_LIBS_USER: /mnt/cache/R-pkgs
volumes:
- amd64-binaries-r-dep-cache-alpine-320:/mnt/cache
commands:
# download latest package version (avoid having to rebuild image every time)
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
- rm -rf /mnt/cache/packages/*
- rm -rf /mnt/cache/R-pkgs/stringr
- R CMD INSTALL .
- R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
- cd /mnt/cache/packages
- R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()'
backend_options:
kubernetes:
resources:
requests:
memory: 2000Mi
cpu: 2000m
limits:
memory: 2000Mi
cpu: 2000m
nodeSelector:
kubernetes.io/arch: 'amd64'
- name: "ntfy"
image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
when:
- status: [failure]
settings:
url: https://ntfy.devxy.io/ci-builds
token:
from_secret: ntfy_token
title: "CI '${CI_REPO}' failed"
actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
click: ${CI_BUILD_LINK}
message: |
Workflow: ${CI_WORKFLOW_NAME/\n//}
Commit message: ${CI_COMMIT_MESSAGE/\n//}
Commit Author: ${CI_COMMIT_AUTHOR/\n//}
backend_options:
kubernetes:
resources:
requests:
memory: 100Mi
cpu: 0m
limits:
memory: 300Mi
cpu: 200m

66
.woodpecker/update-package-index-redhat-8-amd64.yaml Normal file
View file

@ -0,0 +1,66 @@
when:
- event: [cron]
cron: update-package-index-redhat-8-amd64
- event: manual
evaluate: 'update_package_index == "redhat-8-amd64"'
skip_clone: true
steps:
- name: Upload PACKAGES files
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
image: docker.io/devxygmbh/amd64-binaries-r-redhat-8
pull: true
environment:
GIT_USER: pat-s
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
R_LIBS_USER: /mnt/cache/R-pkgs
volumes:
- amd64-binaries-r-dep-cache-redhat-8:/mnt/cache
commands:
# download latest package version (avoid having to rebuild image every time)
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
- rm -rf /mnt/cache/packages/*
- rm -rf /mnt/cache/R-pkgs/stringr
- R CMD INSTALL .
- R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
- cd /mnt/cache/packages
- R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()'
backend_options:
kubernetes:
resources:
requests:
memory: 2000Mi
cpu: 2000m
limits:
memory: 2000Mi
cpu: 2000m
nodeSelector:
kubernetes.io/arch: 'amd64'
- name: "ntfy"
image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
when:
- status: [failure]
settings:
url: https://ntfy.devxy.io/ci-builds
token:
from_secret: ntfy_token
title: "CI '${CI_REPO}' failed"
actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
click: ${CI_BUILD_LINK}
message: |
Workflow: ${CI_WORKFLOW_NAME/\n//}
Commit message: ${CI_COMMIT_MESSAGE/\n//}
Commit Author: ${CI_COMMIT_AUTHOR/\n//}
backend_options:
kubernetes:
resources:
requests:
memory: 100Mi
cpu: 0m
limits:
memory: 300Mi
cpu: 200m

66
.woodpecker/update-package-index-redhat-9-amd64.yaml Normal file
View file

@ -0,0 +1,66 @@
when:
- event: [cron]
cron: update-package-index-redhat-9-amd64
- event: manual
evaluate: 'update_package_index == "redhat-9-amd64"'
skip_clone: true
steps:
- name: Upload PACKAGES files
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
image: docker.io/devxygmbh/amd64-binaries-r-redhat-9
pull: true
environment:
GIT_USER: pat-s
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
R_LIBS_USER: /mnt/cache/R-pkgs
volumes:
- amd64-binaries-r-dep-cache-redhat-9:/mnt/cache
commands:
# download latest package version (avoid having to rebuild image every time)
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
- rm -rf /mnt/cache/packages/*
- rm -rf /mnt/cache/R-pkgs/stringr
- R CMD INSTALL .
- R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
- cd /mnt/cache/packages
- R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()'
backend_options:
kubernetes:
resources:
requests:
memory: 2000Mi
cpu: 2000m
limits:
memory: 2000Mi
cpu: 2000m
nodeSelector:
kubernetes.io/arch: 'amd64'
- name: "ntfy"
image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
when:
- status: [failure]
settings:
url: https://ntfy.devxy.io/ci-builds
token:
from_secret: ntfy_token
title: "CI '${CI_REPO}' failed"
actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
click: ${CI_BUILD_LINK}
message: |
Workflow: ${CI_WORKFLOW_NAME/\n//}
Commit message: ${CI_COMMIT_MESSAGE/\n//}
Commit Author: ${CI_COMMIT_AUTHOR/\n//}
backend_options:
kubernetes:
resources:
requests:
memory: 100Mi
cpu: 0m
limits:
memory: 300Mi
cpu: 200m

66
.woodpecker/update-package-index-ubuntu-2204-amd64.yaml Normal file
View file

@ -0,0 +1,66 @@
when:
- event: [cron]
cron: update-package-index-ubuntu-2204-amd64
- event: manual
evaluate: 'update_package_index == "ubuntu-2204-amd64"'
skip_clone: true
steps:
- name: Upload PACKAGES files
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2204
pull: true
environment:
GIT_USER: pat-s
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
R_LIBS_USER: /mnt/cache/R-pkgs
volumes:
- amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache
commands:
# download latest package version (avoid having to rebuild image every time)
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
- rm -rf /mnt/cache/packages/*
- rm -rf /mnt/cache/R-pkgs/stringr
- R CMD INSTALL .
- R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
- cd /mnt/cache/packages
- R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()'
backend_options:
kubernetes:
resources:
requests:
memory: 2000Mi
cpu: 2000m
limits:
memory: 2000Mi
cpu: 2000m
nodeSelector:
kubernetes.io/arch: 'amd64'
- name: "ntfy"
image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
when:
- status: [failure]
settings:
url: https://ntfy.devxy.io/ci-builds
token:
from_secret: ntfy_token
title: "CI '${CI_REPO}' failed"
actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
click: ${CI_BUILD_LINK}
message: |
Workflow: ${CI_WORKFLOW_NAME/\n//}
Commit message: ${CI_COMMIT_MESSAGE/\n//}
Commit Author: ${CI_COMMIT_AUTHOR/\n//}
backend_options:
kubernetes:
resources:
requests:
memory: 100Mi
cpu: 0m
limits:
memory: 300Mi
cpu: 200m

66
.woodpecker/update-package-index-ubuntu-2404-amd64.yaml Normal file
View file

@ -0,0 +1,66 @@
when:
- event: [cron]
cron: update-package-index-ubuntu-2404-amd64
- event: manual
evaluate: 'update_package_index == "ubuntu-2404-amd64"'
skip_clone: true
steps:
- name: Upload PACKAGES files
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404
pull: true
environment:
GIT_USER: pat-s
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
R_LIBS_USER: /mnt/cache/R-pkgs
volumes:
- amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache
commands:
# download latest package version (avoid having to rebuild image every time)
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
- rm -rf /mnt/cache/packages/*
- rm -rf /mnt/cache/R-pkgs/stringr
- R CMD INSTALL .
- R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
- cd /mnt/cache/packages
- R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()'
backend_options:
kubernetes:
resources:
requests:
memory: 2000Mi
cpu: 2000m
limits:
memory: 2000Mi
cpu: 2000m
nodeSelector:
kubernetes.io/arch: 'amd64'
- name: "ntfy"
image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
when:
- status: [failure]
settings:
url: https://ntfy.devxy.io/ci-builds
token:
from_secret: ntfy_token
title: "CI '${CI_REPO}' failed"
actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
click: ${CI_BUILD_LINK}
message: |
Workflow: ${CI_WORKFLOW_NAME/\n//}
Commit message: ${CI_COMMIT_MESSAGE/\n//}
Commit Author: ${CI_COMMIT_AUTHOR/\n//}
backend_options:
kubernetes:
resources:
requests:
memory: 100Mi
cpu: 0m
limits:
memory: 300Mi
cpu: 200m

2
R/process_cran_updates.R
View file

@ -30,6 +30,8 @@ process_cran_updates <- function(
process_removed = TRUE) {
# Get list of updated and new packages for a specific day
updated_pkgs <- get_updated_cran_packages(interval)
# FIXME: temporary hickup for package redatam
updated_pkgs <- updated_pkgs[updated_pkgs$name != 'redatam', ]
new_pkgs <- get_new_cran_packages(interval)
all_pkgs <- dplyr::bind_rows(updated_pkgs, new_pkgs)