diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index 8eccf96..ca5f706 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -14,47 +14,46 @@ when: # - PLATFORM: opensuse-155 # PLATFORM_ID: suse155 steps: - - name: Build binaries - secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] - image: docker.io/pats22/arm-binaries-r-ubi9 - pull: true - # to ensure the PACKAGES upload step is being executed - failure: ignore - environment: - GIT_USER: pat-s - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - volumes: - - arm-binaries-r-dep-cache-rhel9:/mnt/cache - commands: - # download latest package version (avoid having to rebuild image every time) - # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries - - rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries - - R -q -e 'pak::pak("dyfanjones/s3fs")' - - R -q -e 'install.packages(".", repos = NULL, quiet = FALSE); packageVersion("rBinaries")' - - mkdir -p /root/.R && echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" > /root/.R/Makevars - # manually install some packages into the cache so that some builds don't fail - # Hmisc -> ABCanalysis - - R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries /mnt/cache/ccache - - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 6, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][26:40]; library(rBinaries); future::plan("multisession", workers = 5, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))' - backend_options: - kubernetes: - resources: - requests: - memory: 25Gi - cpu: 12000m - limits: - memory: 28Gi - cpu: 16000m - nodeSelector: - kubernetes.io/arch: 'arm64' - # - name: Sleep - # image: alpine:3.20 - # commands: sleep 10 + # - name: Build binaries + # secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] + # image: docker.io/pats22/arm-binaries-r-ubi9 + # pull: true + # # to ensure the PACKAGES upload step is being executed + # failure: ignore + # environment: + # GIT_USER: pat-s + # # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + # R_PKG_CACHE_DIR: /mnt/cache/pkgcache + # R_LIBS_USER: /mnt/cache/R-pkgs + # CCACHE_DIR: /mnt/cache/ccache + # volumes: + # - arm-binaries-r-dep-cache-rhel9:/mnt/cache + # commands: + # # download latest package version (avoid having to rebuild image every time) + # # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + # - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries + # - rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries + # - R -q -e 'pak::pak("dyfanjones/s3fs")' + # - R -q -e 'pak::local_install(); packageVersion("rBinaries")' + # - mkdir -p /root/.R && echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" > /root/.R/Makevars + # # manually install some packages into the cache so that some builds don't fail + # # Hmisc -> ABCanalysis + # - R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))' + # # more complicated system dependencies which pak cannot resolve + # - + # - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries /mnt/cache/ccache + # - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 6, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][26:40]; library(rBinaries); future::plan("multisession", workers = 5, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))' + # backend_options: + # kubernetes: + # resources: + # requests: + # memory: 25Gi + # cpu: 12000m + # limits: + # memory: 28Gi + # cpu: 16000m + # nodeSelector: + # kubernetes.io/arch: 'arm64' - name: Upload PACKAGES files secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] image: docker.io/pats22/arm-binaries-r-ubi9 @@ -70,17 +69,17 @@ steps: # download latest package version (avoid having to rebuild image every time) # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries - - R -q -e 'install.packages(".", repos = NULL, quiet = TRUE); packageVersion("rBinaries")' + - R -q -e 'pak::local_install(); packageVersion("rBinaries")' - R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()' - R -q -e 'files = list.files("/mnt/cache/binaries/", include.dirs = FALSE, full.names = TRUE, recursive = TRUE); foo = file.remove(files)' backend_options: kubernetes: resources: requests: - memory: 300Mi + memory: 1000Mi cpu: 0m limits: - memory: 500Mi + memory: 1300Mi cpu: 1000m nodeSelector: kubernetes.io/arch: 'arm64' diff --git a/R/upload.R b/R/upload.R index c99096f..f0868bb 100644 --- a/R/upload.R +++ b/R/upload.R @@ -19,6 +19,11 @@ upload_single_binary_to_s3 <- function( local_bin_path <- set_bin_path(r_minor_version, build_for_minor, local_build_root = local_build_root, codename) remote_bin_path <- set_bin_path(r_minor_version, build_for_minor, local_build_root = bucket, codename) + if (!file.exists(sprintf("%s/%s", local_bin_path, tarball_name))) { + cli::cli_alert("{.fun upload_single_binary_to_s3}: File {.pkg {package_name}} {.field {tag}} does not exist locally - skipping upload.") + return(TRUE) + } + if (debug) { cli::cli_alert_warning("DEBUG: local_bin_path: {local_bin_path}") cli::cli_alert_warning("DEBUG: remote_bin_path: {remote_bin_path}")