devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 10 Pull requests Releases Activity

skip upload if file does not exist locally

Browse source
This commit is contained in:
Patrick Schratz 2024-07-27 21:24:44 +02:00
commit f888e4aa86
Signed by: pat-s
GPG key ID: 3C6318841EF78925
2 changed files with 48 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

87
.woodpecker/build.yaml
View file

@ -14,47 +14,46 @@ when:
# - PLATFORM: opensuse-155 # - PLATFORM: opensuse-155
# PLATFORM_ID: suse155 # PLATFORM_ID: suse155
steps: steps:
- name: Build binaries # - name: Build binaries
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] # secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
image: docker.io/pats22/arm-binaries-r-ubi9 # image: docker.io/pats22/arm-binaries-r-ubi9
pull: true # pull: true
# to ensure the PACKAGES upload step is being executed # # to ensure the PACKAGES upload step is being executed
failure: ignore # failure: ignore
environment: # environment:
GIT_USER: pat-s # GIT_USER: pat-s
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves # # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
R_PKG_CACHE_DIR: /mnt/cache/pkgcache # R_PKG_CACHE_DIR: /mnt/cache/pkgcache
R_LIBS_USER: /mnt/cache/R-pkgs # R_LIBS_USER: /mnt/cache/R-pkgs
CCACHE_DIR: /mnt/cache/ccache # CCACHE_DIR: /mnt/cache/ccache
volumes: # volumes:
- arm-binaries-r-dep-cache-rhel9:/mnt/cache # - arm-binaries-r-dep-cache-rhel9:/mnt/cache
commands: # commands:
# download latest package version (avoid having to rebuild image every time) # # download latest package version (avoid having to rebuild image every time)
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 # # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries # - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries
- rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries # - rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries
- R -q -e 'pak::pak("dyfanjones/s3fs")' # - R -q -e 'pak::pak("dyfanjones/s3fs")'
- R -q -e 'install.packages(".", repos = NULL, quiet = FALSE); packageVersion("rBinaries")' # - R -q -e 'pak::local_install(); packageVersion("rBinaries")'
- mkdir -p /root/.R && echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" > /root/.R/Makevars # - mkdir -p /root/.R && echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" > /root/.R/Makevars
# manually install some packages into the cache so that some builds don't fail # # manually install some packages into the cache so that some builds don't fail
# Hmisc -> ABCanalysis # # Hmisc -> ABCanalysis
- R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))' # - R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))'
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries /mnt/cache/ccache # # more complicated system dependencies which pak cannot resolve
- R -q -e 'options(crayon.enabled = TRUE, Ncpus = 6, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][26:40]; library(rBinaries); future::plan("multisession", workers = 5, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))' # -
backend_options: # - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries /mnt/cache/ccache
kubernetes: # - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 6, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][26:40]; library(rBinaries); future::plan("multisession", workers = 5, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))'
resources: # backend_options:
requests: # kubernetes:
memory: 25Gi # resources:
cpu: 12000m # requests:
limits: # memory: 25Gi
memory: 28Gi # cpu: 12000m
cpu: 16000m # limits:
nodeSelector: # memory: 28Gi
kubernetes.io/arch: 'arm64' # cpu: 16000m
# - name: Sleep # nodeSelector:
# image: alpine:3.20 # kubernetes.io/arch: 'arm64'
# commands: sleep 10
- name: Upload PACKAGES files - name: Upload PACKAGES files
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
image: docker.io/pats22/arm-binaries-r-ubi9 image: docker.io/pats22/arm-binaries-r-ubi9
@ -70,17 +69,17 @@ steps:
# download latest package version (avoid having to rebuild image every time) # download latest package version (avoid having to rebuild image every time)
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries
- R -q -e 'install.packages(".", repos = NULL, quiet = TRUE); packageVersion("rBinaries")' - R -q -e 'pak::local_install(); packageVersion("rBinaries")'
- R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()' - R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()'
- R -q -e 'files = list.files("/mnt/cache/binaries/", include.dirs = FALSE, full.names = TRUE, recursive = TRUE); foo = file.remove(files)' - R -q -e 'files = list.files("/mnt/cache/binaries/", include.dirs = FALSE, full.names = TRUE, recursive = TRUE); foo = file.remove(files)'
backend_options: backend_options:
kubernetes: kubernetes:
resources: resources:
requests: requests:
memory: 300Mi memory: 1000Mi
cpu: 0m cpu: 0m
limits: limits:
memory: 500Mi memory: 1300Mi
cpu: 1000m cpu: 1000m
nodeSelector: nodeSelector:
kubernetes.io/arch: 'arm64' kubernetes.io/arch: 'arm64'

5
R/upload.R
View file

@ -19,6 +19,11 @@ upload_single_binary_to_s3 <- function(
local_bin_path <- set_bin_path(r_minor_version, build_for_minor, local_build_root = local_build_root, codename) local_bin_path <- set_bin_path(r_minor_version, build_for_minor, local_build_root = local_build_root, codename)
remote_bin_path <- set_bin_path(r_minor_version, build_for_minor, local_build_root = bucket, codename) remote_bin_path <- set_bin_path(r_minor_version, build_for_minor, local_build_root = bucket, codename)
if (!file.exists(sprintf("%s/%s", local_bin_path, tarball_name))) {
cli::cli_alert("{.fun upload_single_binary_to_s3}: File {.pkg {package_name}} {.field {tag}} does not exist locally - skipping upload.")
return(TRUE)
}
if (debug) { if (debug) {
cli::cli_alert_warning("DEBUG: local_bin_path: {local_bin_path}") cli::cli_alert_warning("DEBUG: local_bin_path: {local_bin_path}")
cli::cli_alert_warning("DEBUG: remote_bin_path: {remote_bin_path}") cli::cli_alert_warning("DEBUG: remote_bin_path: {remote_bin_path}")