chore: fix temporary hickup for pkg redatam

.woodpecker/build-alpine-320-amd64.yaml

@@ -22,10 +22,10 @@ steps:
       R_LIBS_USER: /mnt/cache/R-pkgs
       CCACHE_DIR: /mnt/cache/ccache
       PLATFORM: alpine-320
-      BLOCK: 1880:2993
+      BLOCK: 2436:2993
       ARCH: amd64
       NCPUS: 4
-      # MEMORY: 18Gi
+      # memory: 20Gi
       # INSTANCE_TYPE: ccx23
       STRATEGY: sequential
     # volumes:
@@ -85,7 +85,7 @@ steps:
             memory: 5Gi
             cpu: 2000m
           limits:
-            memory: 18Gi
+            memory: 20Gi
             cpu: 4000m
         nodeSelector:
           kubernetes.io/arch: amd64

.woodpecker/build-redhat-8-amd64.yaml

@@ -22,7 +22,7 @@ steps:
       R_LIBS_USER: /mnt/cache/R-pkgs
       CCACHE_DIR: /mnt/cache/ccache
       PLATFORM: redhat-8
-      BLOCK: 737:1833
+      BLOCK: 1561:1833
       ARCH: amd64
       NCPUS: 4
       # MEMORY: 14Gi
@@ -86,7 +86,7 @@ steps:
             memory: 5Gi
             cpu: 2000m
           limits:
-            memory: 14Gi
+            memory: 20Gi
             cpu: 4000m
         ### PROD-sequential
         # resources:

.woodpecker/build-redhat-9-amd64.yaml

@@ -46,10 +46,9 @@ steps:
       R_LIBS_USER: /mnt/cache/R-pkgs
       CCACHE_DIR: /mnt/cache/ccache
       PLATFORM: redhat-9
-      BLOCK: 715:2428
+      BLOCK: 1664:2428
       ARCH: amd64
       NCPUS: 4
-      MEMORY: 14Gi
       INSTANCE_TYPE: AX42
       STRATEGY: sequential
     # volumes:
@@ -110,7 +109,7 @@ steps:
             memory: 5Gi
             cpu: 2000m
           limits:
-            memory: 14Gi
+            memory: 20Gi
             cpu: 4000m
         ### PROD-sequential
         # resources:

.woodpecker/build-ubuntu-2204-amd64.yaml

@@ -22,7 +22,7 @@ steps:
       R_LIBS_USER: /mnt/cache/R-pkgs
       CCACHE_DIR: /mnt/cache/ccache
       PLATFORM: ubuntu-2204
-      BLOCK: 2128:4755
+      BLOCK: 4223:4755
       ARCH: amd64
       NCPUS: 4
       # MEMORY: 14Gi

.woodpecker/build-ubuntu-2404-amd64.yaml

@@ -22,7 +22,7 @@ steps:
       R_LIBS_USER: /mnt/cache/R-pkgs
       CCACHE_DIR: /mnt/cache/ccache
       PLATFORM: ubuntu-2404
-      BLOCK: 1886:4298
+      BLOCK: 3025:4298
       ARCH: amd64
       NCPUS: 4
       MEMORY: 14Gi

.woodpecker/process-updates-alpine-320-amd64.yaml

@@ -43,10 +43,10 @@ steps:
         ### PROD
         resources:
           requests:
-            memory: 14Gi
+            memory: 18Gi
             cpu: 3000m
           limits:
-            memory: 14Gi
+            memory: 18Gi
             cpu: 3000m
         nodeSelector:
           kubernetes.io/arch: amd64

.woodpecker/process-updates-redhat-8-amd64.yaml

@@ -19,10 +19,10 @@ steps:
   #     kubernetes:
   #       resources:
   #         requests:
-  #           memory: 14Gi
+  #           memory: 18Gi
   #           cpu: 3000m
   #         limits:
-  #           memory: 14Gi
+  #           memory: 18Gi
   #           cpu: 3000m
   #       nodeSelector:
   #         kubernetes.io/arch: amd64
@@ -46,7 +46,7 @@ steps:
       PLATFORM: redhat-8
       ARCH: amd64
       NCPUS: 2
-      # MEMORY: 14Gi
+      # MEMORY: 18Gi
       STRATEGY: sequential
       # INSTANCE_TYPE: cpx41
       INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3)
@@ -69,10 +69,10 @@ steps:
         ### PROD
         resources:
           requests:
-            memory: 14Gi
+            memory: 18Gi
             cpu: 3000m
           limits:
-            memory: 14Gi
+            memory: 18Gi
             cpu: 3000m
         nodeSelector:
           kubernetes.io/arch: amd64

.woodpecker/process-updates-redhat-9-amd64.yaml

@@ -22,7 +22,7 @@ steps:
       PLATFORM: redhat-9
       ARCH: amd64
       NCPUS: 2
-      MEMORY: 14Gi
+      MEMORY: 18Gi
       STRATEGY: sequential
       INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3)
     # volumes:
@@ -44,10 +44,10 @@ steps:
         ### PROD
         resources:
           requests:
-            memory: 14Gi
+            memory: 18Gi
             cpu: 3000m
           limits:
-            memory: 14Gi
+            memory: 18Gi
             cpu: 3000m
         nodeSelector:
           kubernetes.io/arch: amd64

.woodpecker/process-updates-ubuntu-2204-amd64.yaml

@@ -9,30 +9,6 @@ when:
 skip_clone: true
 
 steps:
-  # - name: 'Warming up'
-  #   image: busybox:latest
-  #   failure: ignore
-  #   commands:
-  #     - sleep 300
-  #     - echo "Warmed Up"
-  #   backend_options:
-  #     kubernetes:
-  #       resources:
-  #         requests:
-  #           memory: 14Gi
-  #           cpu: 3000m
-  #         limits:
-  #           memory: 14Gi
-  #           cpu: 3000m
-  #       nodeSelector:
-  #         kubernetes.io/arch: amd64
-  #         node.kubernetes.io/instance-type: ccx23
-  #       tolerations:
-  #         - key: 'CI'
-  #           operator: 'Equal'
-  #           value: 'true'
-  #           effect: 'NoSchedule'
-
   - name: 'Processing Updates'
     secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
     image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2204
@@ -46,7 +22,7 @@ steps:
       PLATFORM: ubuntu-2204
       ARCH: amd64
       NCPUS: 2
-      MEMORY: 14Gi
+      MEMORY: 18Gi
       STRATEGY: sequential
       # INSTANCE_TYPE: ccx23
       INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3)
@@ -69,19 +45,11 @@ steps:
         ### PROD
         resources:
           requests:
-            memory: 14Gi
+            memory: 18Gi
             cpu: 3000m
           limits:
-            memory: 14Gi
+            memory: 18Gi
             cpu: 3000m
-        # DEBUG
-        # resources:
-        #   requests:
-        #     memory: 4Gi
-        #     cpu: 2000m
-        #   limits:
-        #     memory: 4Gi
-        #     cpu: 2000m
         nodeSelector:
           kubernetes.io/arch: amd64
           node.kubernetes.io/instance-type: AX42

.woodpecker/process-updates-ubuntu-2404-amd64.yaml

@@ -24,8 +24,8 @@ steps:
       NCPUS: 1
       STRATEGY: sequential
       INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3)
-    volumes:
+    # volumes:
-      - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache
+    #   - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache
     commands:
       # download latest package version (avoid having to rebuild image every time)
       - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
@@ -43,10 +43,10 @@ steps:
         ### PROD
         resources:
           requests:
-            memory: 14Gi
+            memory: 18Gi
             cpu: 3000m
           limits:
-            memory: 14Gi
+            memory: 18Gi
             cpu: 3000m
         nodeSelector:
           kubernetes.io/arch: amd64

.woodpecker/update-package-index-alpine-320-amd64.yaml

@@ -0,0 +1,66 @@
+when:
+  - event: [cron]
+    cron: update-package-index
+  - event: manual
+    evaluate: 'update_package_index == "alpine-320-amd64"'
+
+skip_clone: true
+
+steps:
+  - name: Upload PACKAGES files
+    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
+    image: docker.io/devxygmbh/amd64-binaries-r-alpine-320
+    pull: true
+    environment:
+      GIT_USER: pat-s
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+    volumes:
+      - amd64-binaries-r-dep-cache-alpine-320:/mnt/cache
+    commands:
+      # download latest package version (avoid having to rebuild image every time)
+      # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
+      - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
+      - rm -rf /mnt/cache/packages/*
+      - rm -rf /mnt/cache/R-pkgs/stringr
+      - R CMD INSTALL .
+      - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
+      - cd /mnt/cache/packages
+      - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()'
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 2000Mi
+            cpu: 2000m
+          limits:
+            memory: 2000Mi
+            cpu: 2000m
+        nodeSelector:
+          kubernetes.io/arch: 'amd64'
+
+  - name: "ntfy"
+    image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
+    when:
+      - status: [failure]
+    settings:
+      url: https://ntfy.devxy.io/ci-builds
+      token:
+        from_secret: ntfy_token
+      title: "CI '${CI_REPO}' failed"
+      actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
+      click: ${CI_BUILD_LINK}
+      message: |
+        Workflow: ${CI_WORKFLOW_NAME/\n//}
+        Commit message: ${CI_COMMIT_MESSAGE/\n//}
+        Commit Author: ${CI_COMMIT_AUTHOR/\n//}
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 100Mi
+            cpu: 0m
+          limits:
+            memory: 300Mi
+            cpu: 200m

.woodpecker/update-package-index-redhat-8-amd64.yaml

@@ -0,0 +1,66 @@
+when:
+  - event: [cron]
+    cron: update-package-index-redhat-8-amd64
+  - event: manual
+    evaluate: 'update_package_index == "redhat-8-amd64"'
+
+skip_clone: true
+
+steps:
+  - name: Upload PACKAGES files
+    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
+    image: docker.io/devxygmbh/amd64-binaries-r-redhat-8
+    pull: true
+    environment:
+      GIT_USER: pat-s
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+    volumes:
+      - amd64-binaries-r-dep-cache-redhat-8:/mnt/cache
+    commands:
+      # download latest package version (avoid having to rebuild image every time)
+      # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
+      - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
+      - rm -rf /mnt/cache/packages/*
+      - rm -rf /mnt/cache/R-pkgs/stringr
+      - R CMD INSTALL .
+      - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
+      - cd /mnt/cache/packages
+      - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()'
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 2000Mi
+            cpu: 2000m
+          limits:
+            memory: 2000Mi
+            cpu: 2000m
+        nodeSelector:
+          kubernetes.io/arch: 'amd64'
+
+  - name: "ntfy"
+    image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
+    when:
+      - status: [failure]
+    settings:
+      url: https://ntfy.devxy.io/ci-builds
+      token:
+        from_secret: ntfy_token
+      title: "CI '${CI_REPO}' failed"
+      actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
+      click: ${CI_BUILD_LINK}
+      message: |
+        Workflow: ${CI_WORKFLOW_NAME/\n//}
+        Commit message: ${CI_COMMIT_MESSAGE/\n//}
+        Commit Author: ${CI_COMMIT_AUTHOR/\n//}
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 100Mi
+            cpu: 0m
+          limits:
+            memory: 300Mi
+            cpu: 200m

.woodpecker/update-package-index-redhat-9-amd64.yaml

@@ -0,0 +1,66 @@
+when:
+  - event: [cron]
+    cron: update-package-index-redhat-9-amd64
+  - event: manual
+    evaluate: 'update_package_index == "redhat-9-amd64"'
+
+skip_clone: true
+
+steps:
+  - name: Upload PACKAGES files
+    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
+    image: docker.io/devxygmbh/amd64-binaries-r-redhat-9
+    pull: true
+    environment:
+      GIT_USER: pat-s
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+    volumes:
+      - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache
+    commands:
+      # download latest package version (avoid having to rebuild image every time)
+      # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
+      - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
+      - rm -rf /mnt/cache/packages/*
+      - rm -rf /mnt/cache/R-pkgs/stringr
+      - R CMD INSTALL .
+      - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
+      - cd /mnt/cache/packages
+      - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()'
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 2000Mi
+            cpu: 2000m
+          limits:
+            memory: 2000Mi
+            cpu: 2000m
+        nodeSelector:
+          kubernetes.io/arch: 'amd64'
+
+  - name: "ntfy"
+    image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
+    when:
+      - status: [failure]
+    settings:
+      url: https://ntfy.devxy.io/ci-builds
+      token:
+        from_secret: ntfy_token
+      title: "CI '${CI_REPO}' failed"
+      actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
+      click: ${CI_BUILD_LINK}
+      message: |
+        Workflow: ${CI_WORKFLOW_NAME/\n//}
+        Commit message: ${CI_COMMIT_MESSAGE/\n//}
+        Commit Author: ${CI_COMMIT_AUTHOR/\n//}
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 100Mi
+            cpu: 0m
+          limits:
+            memory: 300Mi
+            cpu: 200m

.woodpecker/update-package-index-ubuntu-2204-amd64.yaml

@@ -0,0 +1,66 @@
+when:
+  - event: [cron]
+    cron: update-package-index-ubuntu-2204-amd64
+  - event: manual
+    evaluate: 'update_package_index == "ubuntu-2204-amd64"'
+
+skip_clone: true
+
+steps:
+  - name: Upload PACKAGES files
+    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
+    image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2204
+    pull: true
+    environment:
+      GIT_USER: pat-s
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+    volumes:
+      - amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache
+    commands:
+      # download latest package version (avoid having to rebuild image every time)
+      # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
+      - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
+      - rm -rf /mnt/cache/packages/*
+      - rm -rf /mnt/cache/R-pkgs/stringr
+      - R CMD INSTALL .
+      - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
+      - cd /mnt/cache/packages
+      - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()'
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 2000Mi
+            cpu: 2000m
+          limits:
+            memory: 2000Mi
+            cpu: 2000m
+        nodeSelector:
+          kubernetes.io/arch: 'amd64'
+
+  - name: "ntfy"
+    image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
+    when:
+      - status: [failure]
+    settings:
+      url: https://ntfy.devxy.io/ci-builds
+      token:
+        from_secret: ntfy_token
+      title: "CI '${CI_REPO}' failed"
+      actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
+      click: ${CI_BUILD_LINK}
+      message: |
+        Workflow: ${CI_WORKFLOW_NAME/\n//}
+        Commit message: ${CI_COMMIT_MESSAGE/\n//}
+        Commit Author: ${CI_COMMIT_AUTHOR/\n//}
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 100Mi
+            cpu: 0m
+          limits:
+            memory: 300Mi
+            cpu: 200m

.woodpecker/update-package-index-ubuntu-2404-amd64.yaml

@@ -0,0 +1,66 @@
+when:
+  - event: [cron]
+    cron: update-package-index-ubuntu-2404-amd64
+  - event: manual
+    evaluate: 'update_package_index == "ubuntu-2404-amd64"'
+
+skip_clone: true
+
+steps:
+  - name: Upload PACKAGES files
+    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
+    image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404
+    pull: true
+    environment:
+      GIT_USER: pat-s
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+    volumes:
+      - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache
+    commands:
+      # download latest package version (avoid having to rebuild image every time)
+      # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
+      - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git .
+      - rm -rf /mnt/cache/packages/*
+      - rm -rf /mnt/cache/R-pkgs/stringr
+      - R CMD INSTALL .
+      - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
+      - cd /mnt/cache/packages
+      - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()'
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 2000Mi
+            cpu: 2000m
+          limits:
+            memory: 2000Mi
+            cpu: 2000m
+        nodeSelector:
+          kubernetes.io/arch: 'amd64'
+
+  - name: "ntfy"
+    image: codeberg.org/l-x/woodpecker-ntfy:1.3.1
+    when:
+      - status: [failure]
+    settings:
+      url: https://ntfy.devxy.io/ci-builds
+      token:
+        from_secret: ntfy_token
+      title: "CI '${CI_REPO}' failed"
+      actions: view, Open build, ${CI_PIPELINE_URL}, clear=false;
+      click: ${CI_BUILD_LINK}
+      message: |
+        Workflow: ${CI_WORKFLOW_NAME/\n//}
+        Commit message: ${CI_COMMIT_MESSAGE/\n//}
+        Commit Author: ${CI_COMMIT_AUTHOR/\n//}
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 100Mi
+            cpu: 0m
+          limits:
+            memory: 300Mi
+            cpu: 200m

R/process_cran_updates.R

@@ -30,6 +30,8 @@ process_cran_updates <- function(
     process_removed = TRUE) {
   # Get list of updated and new packages for a specific day
   updated_pkgs <- get_updated_cran_packages(interval)
+  # FIXME: temporary hickup for package redatam
+  updated_pkgs <- updated_pkgs[updated_pkgs$name != 'redatam', ]
   new_pkgs <- get_new_cran_packages(interval)
 
   all_pkgs <- dplyr::bind_rows(updated_pkgs, new_pkgs)