From 6f1b8dfa8da98d0600eaa6654cdc5354ea7d138d Mon Sep 17 00:00:00 2001 From: pat-s Date: Sat, 27 Jul 2024 22:38:58 +0200 Subject: [PATCH] new build image + index upload fixes --- .woodpecker/build.yaml | 82 +++++++++++++++++++++--------------------- R/packages_index.R | 20 +++++------ 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index ca5f706..6d71591 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -14,46 +14,45 @@ when: # - PLATFORM: opensuse-155 # PLATFORM_ID: suse155 steps: - # - name: Build binaries - # secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] - # image: docker.io/pats22/arm-binaries-r-ubi9 - # pull: true - # # to ensure the PACKAGES upload step is being executed - # failure: ignore - # environment: - # GIT_USER: pat-s - # # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - # R_PKG_CACHE_DIR: /mnt/cache/pkgcache - # R_LIBS_USER: /mnt/cache/R-pkgs - # CCACHE_DIR: /mnt/cache/ccache - # volumes: - # - arm-binaries-r-dep-cache-rhel9:/mnt/cache - # commands: - # # download latest package version (avoid having to rebuild image every time) - # # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - # - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries - # - rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries - # - R -q -e 'pak::pak("dyfanjones/s3fs")' - # - R -q -e 'pak::local_install(); packageVersion("rBinaries")' - # - mkdir -p /root/.R && echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" > /root/.R/Makevars - # # manually install some packages into the cache so that some builds don't fail - # # Hmisc -> ABCanalysis - # - R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))' - # # more complicated system dependencies which pak cannot resolve - # - - # - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries /mnt/cache/ccache - # - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 6, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][26:40]; library(rBinaries); future::plan("multisession", workers = 5, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))' - # backend_options: - # kubernetes: - # resources: - # requests: - # memory: 25Gi - # cpu: 12000m - # limits: - # memory: 28Gi - # cpu: 16000m - # nodeSelector: - # kubernetes.io/arch: 'arm64' + - name: Build binaries + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] + image: docker.io/pats22/arm-binaries-r-ubi9 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + volumes: + - arm-binaries-r-dep-cache-rhel9:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries + - rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries + - R -q -e 'pak::pak("dyfanjones/s3fs")' + - R -q -e 'pak::local_install(); packageVersion("rBinaries")' + - mkdir -p /root/.R && echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" > /root/.R/Makevars + # manually install some packages into the cache so that some builds don't fail + # Hmisc -> ABCanalysis + - R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))' + # more complicated system dependencies which pak cannot resolve + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries /mnt/cache/ccache + - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 6, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][26:40]; library(rBinaries); future::plan("multisession", workers = 5, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))' + backend_options: + kubernetes: + resources: + requests: + memory: 25Gi + cpu: 12000m + limits: + memory: 28Gi + cpu: 16000m + nodeSelector: + kubernetes.io/arch: 'arm64' - name: Upload PACKAGES files secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] image: docker.io/pats22/arm-binaries-r-ubi9 @@ -70,8 +69,9 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries - R -q -e 'pak::local_install(); packageVersion("rBinaries")' + - R -q -e 'pak::pak("dyfanjones/s3fs")' + - cd /mnt/cache/binaries/__linux__/rhel9/latest/src/contrib - R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()' - - R -q -e 'files = list.files("/mnt/cache/binaries/", include.dirs = FALSE, full.names = TRUE, recursive = TRUE); foo = file.remove(files)' backend_options: kubernetes: resources: diff --git a/R/packages_index.R b/R/packages_index.R index e99fb8c..10cf318 100644 --- a/R/packages_index.R +++ b/R/packages_index.R @@ -54,11 +54,11 @@ upload_package_index <- function( endpoint = "https://s3.eu-central-003.backblazeb2.com", region = "eu-central-003", bucket = "devxy-arm64-r-binaries", - local_build_root = "/mnt/cache/binaries/", + local_build_root = ".", codename = NULL, r_minor_version = NULL, build_for_minor = FALSE, debug = FALSE) { - cli::cli_alert("{.fun create_package_index}: Uploading PACKAGES* files to S3.") + cli::cli_alert("{.fun upload_package_index}: Uploading PACKAGES* files to S3.") codename <- set_codename(codename) if (is.null(r_minor_version)) { @@ -66,11 +66,11 @@ upload_package_index <- function( } if (!build_for_minor) { - local_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", local_build_root, codename) - remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", bucket, codename) + local_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib", local_build_root, codename) + remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib", bucket, codename) } else { - dir_out_bin <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", local_build_root, codename, r_minor_version) - remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version) + dir_out_bin <- sprintf("%s/__linux__/%s/%s/latest/src/contrib", local_build_root, codename, r_minor_version) + remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib", bucket, codename, r_minor_version) } s3fs::s3_file_system( @@ -82,18 +82,18 @@ upload_package_index <- function( pkg_count = length(s3fs::s3_dir_ls(remote_bin_dir)) - 4 t1 <- Sys.time() - cranlike::update_PACKAGES(remote_bin_dir) + cranlike::update_PACKAGES(sprintf("s3://%s", remote_bin_dir)) total_build_time <- round(Sys.time() - t1, 2) cli::cli_alert("Time updating PACKAGES index for {.field {pkg_count}} packages: {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.") purrr::walk2( - sprintf("%s/%s", local_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), + c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz"), sprintf("%s/%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), \(x, y) s3fs::s3_file_upload(x, y, overwrite = TRUE) ) - pkgs_upload_local <- sprintf("%sPACKAGES", local_bin_dir) - pkgs_upload_remote <- sprintf("%sPACKAGES", remote_bin_dir) + # pkgs_upload_local <- sprintf("%sPACKAGES", local_bin_dir) + # pkgs_upload_remote <- sprintf("%sPACKAGES", remote_bin_dir) # if (debug) { # cli::cli_alert_warning("DEBUG: PACKAGES UPLOAD local dir: {pkgs_upload_local}")