diff --git a/.woodpecker/build-alpine-320-amd64.yaml b/.woodpecker/build-alpine-320-amd64.yaml index 897f9b1..bb4f398 100644 --- a/.woodpecker/build-alpine-320-amd64.yaml +++ b/.woodpecker/build-alpine-320-amd64.yaml @@ -22,10 +22,10 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache PLATFORM: alpine-320 - BLOCK: 1880:2993 + BLOCK: 2436:2993 ARCH: amd64 NCPUS: 4 - # MEMORY: 18Gi + # memory: 20Gi # INSTANCE_TYPE: ccx23 STRATEGY: sequential # volumes: @@ -85,7 +85,7 @@ steps: memory: 5Gi cpu: 2000m limits: - memory: 18Gi + memory: 20Gi cpu: 4000m nodeSelector: kubernetes.io/arch: amd64 diff --git a/.woodpecker/build-redhat-8-amd64.yaml b/.woodpecker/build-redhat-8-amd64.yaml index 70182e3..e295e04 100644 --- a/.woodpecker/build-redhat-8-amd64.yaml +++ b/.woodpecker/build-redhat-8-amd64.yaml @@ -22,7 +22,7 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache PLATFORM: redhat-8 - BLOCK: 737:1833 + BLOCK: 1561:1833 ARCH: amd64 NCPUS: 4 # MEMORY: 14Gi @@ -86,7 +86,7 @@ steps: memory: 5Gi cpu: 2000m limits: - memory: 14Gi + memory: 20Gi cpu: 4000m ### PROD-sequential # resources: diff --git a/.woodpecker/build-redhat-9-amd64.yaml b/.woodpecker/build-redhat-9-amd64.yaml index 5016259..5e19840 100644 --- a/.woodpecker/build-redhat-9-amd64.yaml +++ b/.woodpecker/build-redhat-9-amd64.yaml @@ -46,10 +46,9 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache PLATFORM: redhat-9 - BLOCK: 715:2428 + BLOCK: 1664:2428 ARCH: amd64 NCPUS: 4 - MEMORY: 14Gi INSTANCE_TYPE: AX42 STRATEGY: sequential # volumes: @@ -110,7 +109,7 @@ steps: memory: 5Gi cpu: 2000m limits: - memory: 14Gi + memory: 20Gi cpu: 4000m ### PROD-sequential # resources: diff --git a/.woodpecker/build-ubuntu-2204-amd64.yaml b/.woodpecker/build-ubuntu-2204-amd64.yaml index dc50a0d..675696f 100644 --- a/.woodpecker/build-ubuntu-2204-amd64.yaml +++ b/.woodpecker/build-ubuntu-2204-amd64.yaml @@ -22,7 +22,7 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache PLATFORM: ubuntu-2204 - BLOCK: 2128:4755 + BLOCK: 4223:4755 ARCH: amd64 NCPUS: 4 # MEMORY: 14Gi diff --git a/.woodpecker/build-ubuntu-2404-amd64.yaml b/.woodpecker/build-ubuntu-2404-amd64.yaml index 26d608e..1378562 100644 --- a/.woodpecker/build-ubuntu-2404-amd64.yaml +++ b/.woodpecker/build-ubuntu-2404-amd64.yaml @@ -22,7 +22,7 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache PLATFORM: ubuntu-2404 - BLOCK: 1886:4298 + BLOCK: 3025:4298 ARCH: amd64 NCPUS: 4 MEMORY: 14Gi diff --git a/.woodpecker/process-updates-alpine-320-amd64.yaml b/.woodpecker/process-updates-alpine-320-amd64.yaml index 9379a07..ae0885e 100644 --- a/.woodpecker/process-updates-alpine-320-amd64.yaml +++ b/.woodpecker/process-updates-alpine-320-amd64.yaml @@ -43,10 +43,10 @@ steps: ### PROD resources: requests: - memory: 14Gi + memory: 18Gi cpu: 3000m limits: - memory: 14Gi + memory: 18Gi cpu: 3000m nodeSelector: kubernetes.io/arch: amd64 diff --git a/.woodpecker/process-updates-redhat-8-amd64.yaml b/.woodpecker/process-updates-redhat-8-amd64.yaml index 53c1505..4f3356f 100644 --- a/.woodpecker/process-updates-redhat-8-amd64.yaml +++ b/.woodpecker/process-updates-redhat-8-amd64.yaml @@ -19,10 +19,10 @@ steps: # kubernetes: # resources: # requests: - # memory: 14Gi + # memory: 18Gi # cpu: 3000m # limits: - # memory: 14Gi + # memory: 18Gi # cpu: 3000m # nodeSelector: # kubernetes.io/arch: amd64 @@ -46,7 +46,7 @@ steps: PLATFORM: redhat-8 ARCH: amd64 NCPUS: 2 - # MEMORY: 14Gi + # MEMORY: 18Gi STRATEGY: sequential # INSTANCE_TYPE: cpx41 INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3) @@ -69,10 +69,10 @@ steps: ### PROD resources: requests: - memory: 14Gi + memory: 18Gi cpu: 3000m limits: - memory: 14Gi + memory: 18Gi cpu: 3000m nodeSelector: kubernetes.io/arch: amd64 diff --git a/.woodpecker/process-updates-redhat-9-amd64.yaml b/.woodpecker/process-updates-redhat-9-amd64.yaml index 0084275..353ad18 100644 --- a/.woodpecker/process-updates-redhat-9-amd64.yaml +++ b/.woodpecker/process-updates-redhat-9-amd64.yaml @@ -22,7 +22,7 @@ steps: PLATFORM: redhat-9 ARCH: amd64 NCPUS: 2 - MEMORY: 14Gi + MEMORY: 18Gi STRATEGY: sequential INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3) # volumes: @@ -44,10 +44,10 @@ steps: ### PROD resources: requests: - memory: 14Gi + memory: 18Gi cpu: 3000m limits: - memory: 14Gi + memory: 18Gi cpu: 3000m nodeSelector: kubernetes.io/arch: amd64 diff --git a/.woodpecker/process-updates-ubuntu-2204-amd64.yaml b/.woodpecker/process-updates-ubuntu-2204-amd64.yaml index fcd761e..261f30c 100644 --- a/.woodpecker/process-updates-ubuntu-2204-amd64.yaml +++ b/.woodpecker/process-updates-ubuntu-2204-amd64.yaml @@ -9,30 +9,6 @@ when: skip_clone: true steps: - # - name: 'Warming up' - # image: busybox:latest - # failure: ignore - # commands: - # - sleep 300 - # - echo "Warmed Up" - # backend_options: - # kubernetes: - # resources: - # requests: - # memory: 14Gi - # cpu: 3000m - # limits: - # memory: 14Gi - # cpu: 3000m - # nodeSelector: - # kubernetes.io/arch: amd64 - # node.kubernetes.io/instance-type: ccx23 - # tolerations: - # - key: 'CI' - # operator: 'Equal' - # value: 'true' - # effect: 'NoSchedule' - - name: 'Processing Updates' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2204 @@ -46,7 +22,7 @@ steps: PLATFORM: ubuntu-2204 ARCH: amd64 NCPUS: 2 - MEMORY: 14Gi + MEMORY: 18Gi STRATEGY: sequential # INSTANCE_TYPE: ccx23 INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3) @@ -69,19 +45,11 @@ steps: ### PROD resources: requests: - memory: 14Gi + memory: 18Gi cpu: 3000m limits: - memory: 14Gi + memory: 18Gi cpu: 3000m - # DEBUG - # resources: - # requests: - # memory: 4Gi - # cpu: 2000m - # limits: - # memory: 4Gi - # cpu: 2000m nodeSelector: kubernetes.io/arch: amd64 node.kubernetes.io/instance-type: AX42 diff --git a/.woodpecker/process-updates-ubuntu-2404-amd64.yaml b/.woodpecker/process-updates-ubuntu-2404-amd64.yaml index c0f9455..525eb3f 100644 --- a/.woodpecker/process-updates-ubuntu-2404-amd64.yaml +++ b/.woodpecker/process-updates-ubuntu-2404-amd64.yaml @@ -24,8 +24,8 @@ steps: NCPUS: 1 STRATEGY: sequential INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3) - volumes: - - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache + # volumes: + # - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache commands: # download latest package version (avoid having to rebuild image every time) - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . @@ -43,10 +43,10 @@ steps: ### PROD resources: requests: - memory: 14Gi + memory: 18Gi cpu: 3000m limits: - memory: 14Gi + memory: 18Gi cpu: 3000m nodeSelector: kubernetes.io/arch: amd64 diff --git a/.woodpecker/update-package-index-alpine-320-amd64.yaml b/.woodpecker/update-package-index-alpine-320-amd64.yaml new file mode 100644 index 0000000..6ad796a --- /dev/null +++ b/.woodpecker/update-package-index-alpine-320-amd64.yaml @@ -0,0 +1,66 @@ +when: + - event: [cron] + cron: update-package-index + - event: manual + evaluate: 'update_package_index == "alpine-320-amd64"' + +skip_clone: true + +steps: + - name: Upload PACKAGES files + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-alpine-320 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + volumes: + - amd64-binaries-r-dep-cache-alpine-320:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/packages/* + - rm -rf /mnt/cache/R-pkgs/stringr + - R CMD INSTALL . + - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' + - cd /mnt/cache/packages + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + backend_options: + kubernetes: + resources: + requests: + memory: 2000Mi + cpu: 2000m + limits: + memory: 2000Mi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: 'amd64' + + - name: "ntfy" + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/update-package-index-redhat-8-amd64.yaml b/.woodpecker/update-package-index-redhat-8-amd64.yaml new file mode 100644 index 0000000..53e38fc --- /dev/null +++ b/.woodpecker/update-package-index-redhat-8-amd64.yaml @@ -0,0 +1,66 @@ +when: + - event: [cron] + cron: update-package-index-redhat-8-amd64 + - event: manual + evaluate: 'update_package_index == "redhat-8-amd64"' + +skip_clone: true + +steps: + - name: Upload PACKAGES files + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-redhat-8 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + volumes: + - amd64-binaries-r-dep-cache-redhat-8:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/packages/* + - rm -rf /mnt/cache/R-pkgs/stringr + - R CMD INSTALL . + - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' + - cd /mnt/cache/packages + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + backend_options: + kubernetes: + resources: + requests: + memory: 2000Mi + cpu: 2000m + limits: + memory: 2000Mi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: 'amd64' + + - name: "ntfy" + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/update-package-index-redhat-9-amd64.yaml b/.woodpecker/update-package-index-redhat-9-amd64.yaml new file mode 100644 index 0000000..e654b69 --- /dev/null +++ b/.woodpecker/update-package-index-redhat-9-amd64.yaml @@ -0,0 +1,66 @@ +when: + - event: [cron] + cron: update-package-index-redhat-9-amd64 + - event: manual + evaluate: 'update_package_index == "redhat-9-amd64"' + +skip_clone: true + +steps: + - name: Upload PACKAGES files + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-redhat-9 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + volumes: + - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/packages/* + - rm -rf /mnt/cache/R-pkgs/stringr + - R CMD INSTALL . + - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' + - cd /mnt/cache/packages + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + backend_options: + kubernetes: + resources: + requests: + memory: 2000Mi + cpu: 2000m + limits: + memory: 2000Mi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: 'amd64' + + - name: "ntfy" + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml b/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml new file mode 100644 index 0000000..4e645bc --- /dev/null +++ b/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml @@ -0,0 +1,66 @@ +when: + - event: [cron] + cron: update-package-index-ubuntu-2204-amd64 + - event: manual + evaluate: 'update_package_index == "ubuntu-2204-amd64"' + +skip_clone: true + +steps: + - name: Upload PACKAGES files + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2204 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + volumes: + - amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/packages/* + - rm -rf /mnt/cache/R-pkgs/stringr + - R CMD INSTALL . + - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' + - cd /mnt/cache/packages + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + backend_options: + kubernetes: + resources: + requests: + memory: 2000Mi + cpu: 2000m + limits: + memory: 2000Mi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: 'amd64' + + - name: "ntfy" + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml b/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml new file mode 100644 index 0000000..38599f3 --- /dev/null +++ b/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml @@ -0,0 +1,66 @@ +when: + - event: [cron] + cron: update-package-index-ubuntu-2404-amd64 + - event: manual + evaluate: 'update_package_index == "ubuntu-2404-amd64"' + +skip_clone: true + +steps: + - name: Upload PACKAGES files + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] + image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 + pull: true + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + volumes: + - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . + - rm -rf /mnt/cache/packages/* + - rm -rf /mnt/cache/R-pkgs/stringr + - R CMD INSTALL . + - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' + - cd /mnt/cache/packages + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + backend_options: + kubernetes: + resources: + requests: + memory: 2000Mi + cpu: 2000m + limits: + memory: 2000Mi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: 'amd64' + + - name: "ntfy" + image: codeberg.org/l-x/woodpecker-ntfy:1.3.1 + when: + - status: [failure] + settings: + url: https://ntfy.devxy.io/ci-builds + token: + from_secret: ntfy_token + title: "CI '${CI_REPO}' failed" + actions: view, Open build, ${CI_PIPELINE_URL}, clear=false; + click: ${CI_BUILD_LINK} + message: | + Workflow: ${CI_WORKFLOW_NAME/\n//} + Commit message: ${CI_COMMIT_MESSAGE/\n//} + Commit Author: ${CI_COMMIT_AUTHOR/\n//} + backend_options: + kubernetes: + resources: + requests: + memory: 100Mi + cpu: 0m + limits: + memory: 300Mi + cpu: 200m diff --git a/R/process_cran_updates.R b/R/process_cran_updates.R index bbf831e..5b1c0cd 100644 --- a/R/process_cran_updates.R +++ b/R/process_cran_updates.R @@ -30,6 +30,8 @@ process_cran_updates <- function( process_removed = TRUE) { # Get list of updated and new packages for a specific day updated_pkgs <- get_updated_cran_packages(interval) + # FIXME: temporary hickup for package redatam + updated_pkgs <- updated_pkgs[updated_pkgs$name != 'redatam', ] new_pkgs <- get_new_cran_packages(interval) all_pkgs <- dplyr::bind_rows(updated_pkgs, new_pkgs)