From a90d6b843135c836883d68b2dd72e9547ee29b80 Mon Sep 17 00:00:00 2001 From: pat-s Date: Fri, 22 Nov 2024 20:49:58 +0100 Subject: [PATCH] feat: parallelize package index update --- .woodpecker/update-package-index-alpine-320-amd64.yaml | 10 +++++----- .woodpecker/update-package-index-alpine-320-arm64.yaml | 10 +++++----- .woodpecker/update-package-index-redhat-8-amd64.yaml | 8 ++++---- .woodpecker/update-package-index-redhat-8-arm64.yaml | 10 +++++----- .woodpecker/update-package-index-redhat-9-amd64.yaml | 10 +++++----- .woodpecker/update-package-index-redhat-9-arm64.yaml | 10 +++++----- .../update-package-index-ubuntu-2204-amd64.yaml | 10 +++++----- .../update-package-index-ubuntu-2204-arm64.yaml | 10 +++++----- .../update-package-index-ubuntu-2404-amd64.yaml | 6 +++--- .../update-package-index-ubuntu-2404-arm64.yaml | 10 +++++----- 10 files changed, 47 insertions(+), 47 deletions(-) diff --git a/.woodpecker/update-package-index-alpine-320-amd64.yaml b/.woodpecker/update-package-index-alpine-320-amd64.yaml index 72abadb..a74dee2 100644 --- a/.woodpecker/update-package-index-alpine-320-amd64.yaml +++ b/.woodpecker/update-package-index-alpine-320-amd64.yaml @@ -25,19 +25,19 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . # - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")' - R -q -e 'packageVersion("bincraftR")' - - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = future::availableCores()); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' backend_options: kubernetes: resources: requests: - memory: 2000Mi + memory: 3000Mi cpu: 2000m limits: - memory: 2000Mi - cpu: 2000m + memory: 3000Mi + cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' diff --git a/.woodpecker/update-package-index-alpine-320-arm64.yaml b/.woodpecker/update-package-index-alpine-320-arm64.yaml index 0daefc5..211fadc 100644 --- a/.woodpecker/update-package-index-alpine-320-arm64.yaml +++ b/.woodpecker/update-package-index-alpine-320-arm64.yaml @@ -25,19 +25,19 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . # - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")' - R -q -e 'packageVersion("bincraftR")' - - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = future::availableCores()); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' backend_options: kubernetes: resources: requests: - memory: 2000Mi + memory: 3000Mi cpu: 2000m limits: - memory: 2000Mi - cpu: 2000m + memory: 3000Mi + cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64' diff --git a/.woodpecker/update-package-index-redhat-8-amd64.yaml b/.woodpecker/update-package-index-redhat-8-amd64.yaml index 9a17002..d40b94e 100644 --- a/.woodpecker/update-package-index-redhat-8-amd64.yaml +++ b/.woodpecker/update-package-index-redhat-8-amd64.yaml @@ -27,16 +27,16 @@ steps: # - rm -rf /mnt/cache/packages/* - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = future::availableCores()); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' backend_options: kubernetes: resources: requests: - memory: 2000Mi + memory: 3000Mi cpu: 2000m limits: - memory: 2000Mi - cpu: 2000m + memory: 3000Mi + cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' diff --git a/.woodpecker/update-package-index-redhat-8-arm64.yaml b/.woodpecker/update-package-index-redhat-8-arm64.yaml index fa7732d..12681dd 100644 --- a/.woodpecker/update-package-index-redhat-8-arm64.yaml +++ b/.woodpecker/update-package-index-redhat-8-arm64.yaml @@ -25,19 +25,19 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . # - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")' - R -q -e 'packageVersion("bincraftR")' - - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = future::availableCores()); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' backend_options: kubernetes: resources: requests: - memory: 2000Mi + memory: 3000Mi cpu: 2000m limits: - memory: 2000Mi - cpu: 2000m + memory: 3000Mi + cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64' diff --git a/.woodpecker/update-package-index-redhat-9-amd64.yaml b/.woodpecker/update-package-index-redhat-9-amd64.yaml index 5f9d829..42679ed 100644 --- a/.woodpecker/update-package-index-redhat-9-amd64.yaml +++ b/.woodpecker/update-package-index-redhat-9-amd64.yaml @@ -25,19 +25,19 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . # - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")' - R -q -e 'packageVersion("bincraftR")' - - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = future::availableCores()); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' backend_options: kubernetes: resources: requests: - memory: 2000Mi + memory: 3000Mi cpu: 2000m limits: - memory: 2000Mi - cpu: 2000m + memory: 3000Mi + cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' diff --git a/.woodpecker/update-package-index-redhat-9-arm64.yaml b/.woodpecker/update-package-index-redhat-9-arm64.yaml index 598a8aa..e5c49e1 100644 --- a/.woodpecker/update-package-index-redhat-9-arm64.yaml +++ b/.woodpecker/update-package-index-redhat-9-arm64.yaml @@ -25,19 +25,19 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . # - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")' - R -q -e 'packageVersion("bincraftR")' - - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = future::availableCores()); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' backend_options: kubernetes: resources: requests: - memory: 2000Mi + memory: 3000Mi cpu: 2000m limits: - memory: 2000Mi - cpu: 2000m + memory: 3000Mi + cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64' diff --git a/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml b/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml index 29eb1e3..31b5628 100644 --- a/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml +++ b/.woodpecker/update-package-index-ubuntu-2204-amd64.yaml @@ -25,19 +25,19 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . # - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")' - R -q -e 'packageVersion("bincraftR")' - - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(); warnings()' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = future::availableCores()); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(); warnings()' backend_options: kubernetes: resources: requests: - memory: 2000Mi + memory: 3000Mi cpu: 2000m limits: - memory: 2000Mi - cpu: 2000m + memory: 3000Mi + cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' diff --git a/.woodpecker/update-package-index-ubuntu-2204-arm64.yaml b/.woodpecker/update-package-index-ubuntu-2204-arm64.yaml index af58927..929c4b6 100644 --- a/.woodpecker/update-package-index-ubuntu-2204-arm64.yaml +++ b/.woodpecker/update-package-index-ubuntu-2204-arm64.yaml @@ -25,19 +25,19 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . # - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")' - R -q -e 'packageVersion("bincraftR")' - - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = future::availableCores()); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' backend_options: kubernetes: resources: requests: - memory: 2000Mi + memory: 3000Mi cpu: 2000m limits: - memory: 2000Mi - cpu: 2000m + memory: 3000Mi + cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64' diff --git a/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml b/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml index 95a4f6a..875f074 100644 --- a/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml +++ b/.woodpecker/update-package-index-ubuntu-2404-amd64.yaml @@ -25,18 +25,18 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . # # - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")' - R -q -e 'packageVersion("bincraftR")' - - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - cd /mnt/cache/packages - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = future::availableCores()); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' backend_options: kubernetes: resources: requests: - memory: 2000Mi + memory: 3000Mi cpu: 2000m limits: - memory: 2000Mi + memory: 3000Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' diff --git a/.woodpecker/update-package-index-ubuntu-2404-arm64.yaml b/.woodpecker/update-package-index-ubuntu-2404-arm64.yaml index 1ca0aec..cad5633 100644 --- a/.woodpecker/update-package-index-ubuntu-2404-arm64.yaml +++ b/.woodpecker/update-package-index-ubuntu-2404-arm64.yaml @@ -25,19 +25,19 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/r-pkg-binaries.git . # - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")' - R -q -e 'packageVersion("bincraftR")' - - R -q -e 'pak::pak(c("pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 2); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = future::availableCores()); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index()' backend_options: kubernetes: resources: requests: - memory: 2000Mi + memory: 3000Mi cpu: 2000m limits: - memory: 2000Mi - cpu: 2000m + memory: 3000Mi + cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64'