build in new root, upload PACKAGES file at the end
This commit is contained in:
parent
f0af31e5d2
commit
4e24f4e743
GPG key ID:
3C6318841EF78925
5 changed files with 90 additions and 15 deletions
|
|
@ -15,6 +15,45 @@ when:
|
|||
# PLATFORM_ID: suse155
|
||||
steps:
|
||||
- name: Build
|
||||
|
||||
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
|
||||
image: docker.io/pats22/arm-binaries-r-ubi9
|
||||
pull: true
|
||||
# to ensure the PACKAGES upload step is being executed
|
||||
failure: ignore
|
||||
environment:
|
||||
GIT_USER: pat-s
|
||||
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
volumes:
|
||||
- arm-binaries-r-dep-cache-rhel9:/mnt/cache
|
||||
commands:
|
||||
# download latest package version (avoid having to rebuild image every time)
|
||||
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
|
||||
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries
|
||||
- R -q -e 'install.packages(".", repos = NULL, quiet = FALSE); packageVersion("rBinaries")'
|
||||
# manually install some packages into the cache so that some builds don't fail
|
||||
# Hmisc -> ABCanalysis
|
||||
# - R -q -e 'pak::pak("Hmisc")'
|
||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries
|
||||
- R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][7:10]; library(rBinaries); future::plan("multisession", workers = 4, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))'
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: 2000Mi
|
||||
cpu: 2000m
|
||||
limits:
|
||||
memory: 6000Mi
|
||||
cpu: 4050m
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: 'arm64'
|
||||
# - name: Sleep
|
||||
# image: alpine:3.20
|
||||
# commands: sleep 10
|
||||
- name: Upload PACKAGES files
|
||||
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
|
||||
image: docker.io/pats22/arm-binaries-r-ubi9
|
||||
pull: true
|
||||
|
|
@ -30,19 +69,16 @@ steps:
|
|||
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
|
||||
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries
|
||||
- R -q -e 'install.packages(".", repos = NULL, quiet = TRUE); packageVersion("rBinaries")'
|
||||
# manually install some packages into the cache so that some builds don't fail
|
||||
# Hmisc -> ABCanalysis
|
||||
# - R -q -e 'pak::pak("Hmisc")'
|
||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs
|
||||
- R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][3:5]; library(rBinaries); future::plan("multisession", workers = 4, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))'
|
||||
- R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()'
|
||||
- R -q -e 'files = list.files("/mnt/cache/binaries/", include.dirs = FALSE, full.names = TRUE, recursive = TRUE); foo = file.remove(files)'
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: 2000Mi
|
||||
cpu: 2000m
|
||||
memory: 300Mi
|
||||
cpu: 0m
|
||||
limits:
|
||||
memory: 6000Mi
|
||||
cpu: 4050m
|
||||
memory: 500Mi
|
||||
cpu: 1000m
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: 'arm64'
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ export(list_metadata_tables)
|
|||
export(query_metadata_table)
|
||||
export(set_bin_path)
|
||||
export(set_codename)
|
||||
export(upload_package_index)
|
||||
export(upload_single_binary_to_s3)
|
||||
import(progressr)
|
||||
importFrom(DBI,dbConnect)
|
||||
|
|
@ -25,6 +26,7 @@ importFrom(dplyr,group_by)
|
|||
importFrom(dplyr,n)
|
||||
importFrom(dplyr,summarise)
|
||||
importFrom(dplyr,tbl)
|
||||
importFrom(fs,file_size)
|
||||
importFrom(future,plan)
|
||||
importFrom(future.apply,future_mapply)
|
||||
importFrom(gert,git_config_set)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
#' @export
|
||||
build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
||||
r_minor_version = NULL, build_for_minor = TRUE,
|
||||
local_build_root = "/root",
|
||||
local_build_root = "/mnt/cache/binaries/",
|
||||
local_clone_dir = "/tmp",
|
||||
platform = "redhat-9",
|
||||
install_system_dependencies = TRUE,
|
||||
|
|
@ -144,6 +144,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
|||
|
||||
#' @importFrom cli cli_alert
|
||||
#' @importFrom pkgbuild build
|
||||
#' @importFrom fs file_size
|
||||
#' @export
|
||||
build_single_tag <- function(
|
||||
package_name, tag = NULL,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ add_to_package_index <- function(
|
|||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries",
|
||||
local_build_root = "/mnt/cache/binaries",
|
||||
codename = NULL, r_minor_version = NULL,
|
||||
build_for_minor = FALSE,
|
||||
debug = FALSE) {
|
||||
|
|
@ -15,10 +16,10 @@ add_to_package_index <- function(
|
|||
}
|
||||
|
||||
if (!build_for_minor) {
|
||||
local_bin_dir <- sprintf("/root/__linux__/%s/latest/src/contrib/", codename)
|
||||
local_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", bucket, codename)
|
||||
} else {
|
||||
dir_out_bin <- sprintf("/root/__linux__/%s/%s/latest/src/contrib/", codename, r_minor_version)
|
||||
dir_out_bin <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", local_build_root, codename, r_minor_version)
|
||||
remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version)
|
||||
}
|
||||
|
||||
|
|
@ -30,15 +31,55 @@ add_to_package_index <- function(
|
|||
)
|
||||
|
||||
# get latest PACKAGES file from S3
|
||||
if (s3fs::s3_file_exists(sprintf("%s/PACKAGES", remote_bin_dir))) {
|
||||
s3fs::s3_file_download(sprintf("%s/PACKAGES", remote_bin_dir), sprintf("%s/PACKAGES", local_bin_dir), overwrite = TRUE)
|
||||
if (!s3fs::s3_file_exists(sprintf("%s/PACKAGES", remote_bin_dir))) {
|
||||
s3fs::s3_file_download(sprintf("%s/PACKAGES", remote_bin_dir), sprintf("%s/PACKAGES", local_bin_dir))
|
||||
}
|
||||
file_names <- list.files(local_bin_dir, pattern = sprintf("%s*", package_name))
|
||||
|
||||
# list all tarballs for the given package
|
||||
cranlike::add_PACKAGES(file_names, local_bin_dir)
|
||||
|
||||
# dummy <- s3fs::s3_file_upload(pkgs_upload_local, pkgs_upload_remote, overwrite = TRUE)
|
||||
# s3fs::s3_file_upload(
|
||||
# sprintf("%s/%s", local_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
|
||||
# sprintf("%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
|
||||
# overwrite = TRUE
|
||||
# )
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
||||
#' @export
|
||||
upload_package_index <- function(
|
||||
package_name = NULL,
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries",
|
||||
local_build_root = "/mnt/cache/binaries/",
|
||||
codename = NULL, r_minor_version = NULL,
|
||||
build_for_minor = FALSE,
|
||||
debug = FALSE) {
|
||||
cli::cli_alert("{.fun create_package_index}: Uploading PACKAGES* files to S3.")
|
||||
|
||||
codename <- set_codename(codename)
|
||||
if (is.null(r_minor_version)) {
|
||||
r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
|
||||
}
|
||||
|
||||
if (!build_for_minor) {
|
||||
local_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", bucket, codename)
|
||||
} else {
|
||||
dir_out_bin <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", local_build_root, codename, r_minor_version)
|
||||
remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version)
|
||||
}
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
endpoint = endpoint,
|
||||
region_name = region,
|
||||
)
|
||||
|
||||
dummy <- purrr::walk2(
|
||||
sprintf("%s/%s", local_bin_dir, c("PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
|
||||
sprintf("%s/%s", remote_bin_dir, c("PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
|
||||
|
|
@ -56,11 +97,5 @@ add_to_package_index <- function(
|
|||
s3 <- paws.storage::s3(endpoint = endpoint, region = region)
|
||||
s3$put_object(Body = pkgs_upload_local, Bucket = bucket, Key = "__linux__/rhel9/latest/src/contrib/PACKAGES")
|
||||
|
||||
# dummy <- s3fs::s3_file_upload(pkgs_upload_local, pkgs_upload_remote, overwrite = TRUE)
|
||||
# s3fs::s3_file_upload(
|
||||
# sprintf("%s/%s", local_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
|
||||
# sprintf("%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
|
||||
# overwrite = TRUE
|
||||
# )
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ upload_single_binary_to_s3 <- function(
|
|||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries",
|
||||
local_build_root = "/root", codename = NULL,
|
||||
local_build_root = "/mnt/cache/binaries/",
|
||||
codename = NULL,
|
||||
package_name, tag,
|
||||
r_minor_version = NULL, build_for_minor = FALSE,
|
||||
force = FALSE,
|
||||
|
|
|
|||
Loading…
Reference in a new issue