devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 11 Pull requests 1 Releases Activity

new build image + index upload fixes

Browse source
This commit is contained in:
Patrick Schratz 2024-07-27 22:38:58 +02:00
commit 6f1b8dfa8d
Signed by: pat-s
GPG key ID: 3C6318841EF78925
2 changed files with 51 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

82
.woodpecker/build.yaml
View file

@ -14,46 +14,45 @@ when:
# - PLATFORM: opensuse-155 # - PLATFORM: opensuse-155
# PLATFORM_ID: suse155 # PLATFORM_ID: suse155
steps: steps:
# - name: Build binaries - name: Build binaries
# secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
# image: docker.io/pats22/arm-binaries-r-ubi9 image: docker.io/pats22/arm-binaries-r-ubi9
# pull: true pull: true
# # to ensure the PACKAGES upload step is being executed # to ensure the PACKAGES upload step is being executed
# failure: ignore failure: ignore
# environment: environment:
# GIT_USER: pat-s GIT_USER: pat-s
# # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
# R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_PKG_CACHE_DIR: /mnt/cache/pkgcache
# R_LIBS_USER: /mnt/cache/R-pkgs R_LIBS_USER: /mnt/cache/R-pkgs
# CCACHE_DIR: /mnt/cache/ccache CCACHE_DIR: /mnt/cache/ccache
# volumes: volumes:
# - arm-binaries-r-dep-cache-rhel9:/mnt/cache - arm-binaries-r-dep-cache-rhel9:/mnt/cache
# commands: commands:
# # download latest package version (avoid having to rebuild image every time) # download latest package version (avoid having to rebuild image every time)
# # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
# - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries
# - rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries - rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries
# - R -q -e 'pak::pak("dyfanjones/s3fs")' - R -q -e 'pak::pak("dyfanjones/s3fs")'
# - R -q -e 'pak::local_install(); packageVersion("rBinaries")' - R -q -e 'pak::local_install(); packageVersion("rBinaries")'
# - mkdir -p /root/.R && echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" > /root/.R/Makevars - mkdir -p /root/.R && echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" > /root/.R/Makevars
# # manually install some packages into the cache so that some builds don't fail # manually install some packages into the cache so that some builds don't fail
# # Hmisc -> ABCanalysis # Hmisc -> ABCanalysis
# - R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))' - R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))'
# # more complicated system dependencies which pak cannot resolve # more complicated system dependencies which pak cannot resolve
# - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries /mnt/cache/ccache
# - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries /mnt/cache/ccache - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 6, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][26:40]; library(rBinaries); future::plan("multisession", workers = 5, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))'
# - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 6, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][26:40]; library(rBinaries); future::plan("multisession", workers = 5, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))' backend_options:
# backend_options: kubernetes:
# kubernetes: resources:
# resources: requests:
# requests: memory: 25Gi
# memory: 25Gi cpu: 12000m
# cpu: 12000m limits:
# limits: memory: 28Gi
# memory: 28Gi cpu: 16000m
# cpu: 16000m nodeSelector:
# nodeSelector: kubernetes.io/arch: 'arm64'
# kubernetes.io/arch: 'arm64'
- name: Upload PACKAGES files - name: Upload PACKAGES files
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
image: docker.io/pats22/arm-binaries-r-ubi9 image: docker.io/pats22/arm-binaries-r-ubi9
@ -70,8 +69,9 @@ steps:
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries
- R -q -e 'pak::local_install(); packageVersion("rBinaries")' - R -q -e 'pak::local_install(); packageVersion("rBinaries")'
- R -q -e 'pak::pak("dyfanjones/s3fs")'
- cd /mnt/cache/binaries/__linux__/rhel9/latest/src/contrib
- R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()' - R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()'
- R -q -e 'files = list.files("/mnt/cache/binaries/", include.dirs = FALSE, full.names = TRUE, recursive = TRUE); foo = file.remove(files)'
backend_options: backend_options:
kubernetes: kubernetes:
resources: resources:

20
R/packages_index.R
View file

@ -54,11 +54,11 @@ upload_package_index <- function(
endpoint = "https://s3.eu-central-003.backblazeb2.com", endpoint = "https://s3.eu-central-003.backblazeb2.com",
region = "eu-central-003", region = "eu-central-003",
bucket = "devxy-arm64-r-binaries", bucket = "devxy-arm64-r-binaries",
local_build_root = "/mnt/cache/binaries/", local_build_root = ".",
codename = NULL, r_minor_version = NULL, codename = NULL, r_minor_version = NULL,
build_for_minor = FALSE, build_for_minor = FALSE,
debug = FALSE) { debug = FALSE) {
cli::cli_alert("{.fun create_package_index}: Uploading PACKAGES* files to S3.") cli::cli_alert("{.fun upload_package_index}: Uploading PACKAGES* files to S3.")
codename <- set_codename(codename) codename <- set_codename(codename)
if (is.null(r_minor_version)) { if (is.null(r_minor_version)) {
@ -66,11 +66,11 @@ upload_package_index <- function(
} }
if (!build_for_minor) { if (!build_for_minor) {
local_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", local_build_root, codename) local_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib", local_build_root, codename)
remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", bucket, codename) remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib", bucket, codename)
} else { } else {
dir_out_bin <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", local_build_root, codename, r_minor_version) dir_out_bin <- sprintf("%s/__linux__/%s/%s/latest/src/contrib", local_build_root, codename, r_minor_version)
remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version) remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib", bucket, codename, r_minor_version)
} }
s3fs::s3_file_system( s3fs::s3_file_system(
@ -82,18 +82,18 @@ upload_package_index <- function(
pkg_count = length(s3fs::s3_dir_ls(remote_bin_dir)) - 4 pkg_count = length(s3fs::s3_dir_ls(remote_bin_dir)) - 4
t1 <- Sys.time() t1 <- Sys.time()
cranlike::update_PACKAGES(remote_bin_dir) cranlike::update_PACKAGES(sprintf("s3://%s", remote_bin_dir))
total_build_time <- round(Sys.time() - t1, 2) total_build_time <- round(Sys.time() - t1, 2)
cli::cli_alert("Time updating PACKAGES index for {.field {pkg_count}} packages: {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.") cli::cli_alert("Time updating PACKAGES index for {.field {pkg_count}} packages: {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.")
purrr::walk2( purrr::walk2(
sprintf("%s/%s", local_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz"),
sprintf("%s/%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), sprintf("%s/%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
\(x, y) s3fs::s3_file_upload(x, y, overwrite = TRUE) \(x, y) s3fs::s3_file_upload(x, y, overwrite = TRUE)
) )
pkgs_upload_local <- sprintf("%sPACKAGES", local_bin_dir) # pkgs_upload_local <- sprintf("%sPACKAGES", local_bin_dir)
pkgs_upload_remote <- sprintf("%sPACKAGES", remote_bin_dir) # pkgs_upload_remote <- sprintf("%sPACKAGES", remote_bin_dir)
# if (debug) { # if (debug) {
# cli::cli_alert_warning("DEBUG: PACKAGES UPLOAD local dir: {pkgs_upload_local}") # cli::cli_alert_warning("DEBUG: PACKAGES UPLOAD local dir: {pkgs_upload_local}")