diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index 000d134..af07143 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -15,6 +15,45 @@ when: # PLATFORM_ID: suse155 steps: - name: Build + + secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] + image: docker.io/pats22/arm-binaries-r-ubi9 + pull: true + # to ensure the PACKAGES upload step is being executed + failure: ignore + environment: + GIT_USER: pat-s + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + volumes: + - arm-binaries-r-dep-cache-rhel9:/mnt/cache + commands: + # download latest package version (avoid having to rebuild image every time) + # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 + - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries + - rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries + - R -q -e 'install.packages(".", repos = NULL, quiet = FALSE); packageVersion("rBinaries")' + # manually install some packages into the cache so that some builds don't fail + # Hmisc -> ABCanalysis + # - R -q -e 'pak::pak("Hmisc")' + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries + - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][7:10]; library(rBinaries); future::plan("multisession", workers = 4, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))' + backend_options: + kubernetes: + resources: + requests: + memory: 2000Mi + cpu: 2000m + limits: + memory: 6000Mi + cpu: 4050m + nodeSelector: + kubernetes.io/arch: 'arm64' + # - name: Sleep + # image: alpine:3.20 + # commands: sleep 10 + - name: Upload PACKAGES files secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] image: docker.io/pats22/arm-binaries-r-ubi9 pull: true @@ -30,19 +69,16 @@ steps: # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries - R -q -e 'install.packages(".", repos = NULL, quiet = TRUE); packageVersion("rBinaries")' - # manually install some packages into the cache so that some builds don't fail - # Hmisc -> ABCanalysis - # - R -q -e 'pak::pak("Hmisc")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs - - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][3:5]; library(rBinaries); future::plan("multisession", workers = 4, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))' + - R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()' + - R -q -e 'files = list.files("/mnt/cache/binaries/", include.dirs = FALSE, full.names = TRUE, recursive = TRUE); foo = file.remove(files)' backend_options: kubernetes: resources: requests: - memory: 2000Mi - cpu: 2000m + memory: 300Mi + cpu: 0m limits: - memory: 6000Mi - cpu: 4050m + memory: 500Mi + cpu: 1000m nodeSelector: kubernetes.io/arch: 'arm64' diff --git a/NAMESPACE b/NAMESPACE index 6f6fee7..6e90015 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -9,6 +9,7 @@ export(list_metadata_tables) export(query_metadata_table) export(set_bin_path) export(set_codename) +export(upload_package_index) export(upload_single_binary_to_s3) import(progressr) importFrom(DBI,dbConnect) @@ -25,6 +26,7 @@ importFrom(dplyr,group_by) importFrom(dplyr,n) importFrom(dplyr,summarise) importFrom(dplyr,tbl) +importFrom(fs,file_size) importFrom(future,plan) importFrom(future.apply,future_mapply) importFrom(gert,git_config_set) diff --git a/R/build_binaries.R b/R/build_binaries.R index aeccb20..ad411a6 100644 --- a/R/build_binaries.R +++ b/R/build_binaries.R @@ -7,7 +7,7 @@ #' @export build_binary_package <- function(package_name, tag = NULL, codename = NULL, r_minor_version = NULL, build_for_minor = TRUE, - local_build_root = "/root", + local_build_root = "/mnt/cache/binaries/", local_clone_dir = "/tmp", platform = "redhat-9", install_system_dependencies = TRUE, @@ -144,6 +144,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, #' @importFrom cli cli_alert #' @importFrom pkgbuild build +#' @importFrom fs file_size #' @export build_single_tag <- function( package_name, tag = NULL, diff --git a/R/packages_index.R b/R/packages_index.R index 1afe257..772b037 100644 --- a/R/packages_index.R +++ b/R/packages_index.R @@ -6,6 +6,7 @@ add_to_package_index <- function( endpoint = "https://s3.eu-central-003.backblazeb2.com", region = "eu-central-003", bucket = "devxy-arm64-r-binaries", + local_build_root = "/mnt/cache/binaries", codename = NULL, r_minor_version = NULL, build_for_minor = FALSE, debug = FALSE) { @@ -15,10 +16,10 @@ add_to_package_index <- function( } if (!build_for_minor) { - local_bin_dir <- sprintf("/root/__linux__/%s/latest/src/contrib/", codename) + local_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", local_build_root, codename) remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", bucket, codename) } else { - dir_out_bin <- sprintf("/root/__linux__/%s/%s/latest/src/contrib/", codename, r_minor_version) + dir_out_bin <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", local_build_root, codename, r_minor_version) remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version) } @@ -30,15 +31,55 @@ add_to_package_index <- function( ) # get latest PACKAGES file from S3 - if (s3fs::s3_file_exists(sprintf("%s/PACKAGES", remote_bin_dir))) { - s3fs::s3_file_download(sprintf("%s/PACKAGES", remote_bin_dir), sprintf("%s/PACKAGES", local_bin_dir), overwrite = TRUE) + if (!s3fs::s3_file_exists(sprintf("%s/PACKAGES", remote_bin_dir))) { + s3fs::s3_file_download(sprintf("%s/PACKAGES", remote_bin_dir), sprintf("%s/PACKAGES", local_bin_dir)) } file_names <- list.files(local_bin_dir, pattern = sprintf("%s*", package_name)) # list all tarballs for the given package cranlike::add_PACKAGES(file_names, local_bin_dir) + # dummy <- s3fs::s3_file_upload(pkgs_upload_local, pkgs_upload_remote, overwrite = TRUE) + # s3fs::s3_file_upload( + # sprintf("%s/%s", local_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), + # sprintf("%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), + # overwrite = TRUE + # ) + return(invisible(TRUE)) +} + +#' @export +upload_package_index <- function( + package_name = NULL, + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region = "eu-central-003", + bucket = "devxy-arm64-r-binaries", + local_build_root = "/mnt/cache/binaries/", + codename = NULL, r_minor_version = NULL, + build_for_minor = FALSE, + debug = FALSE) { cli::cli_alert("{.fun create_package_index}: Uploading PACKAGES* files to S3.") + + codename <- set_codename(codename) + if (is.null(r_minor_version)) { + r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string) + } + + if (!build_for_minor) { + local_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", local_build_root, codename) + remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", bucket, codename) + } else { + dir_out_bin <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", local_build_root, codename, r_minor_version) + remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version) + } + + s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"), + aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"), + endpoint = endpoint, + region_name = region, + ) + dummy <- purrr::walk2( sprintf("%s/%s", local_bin_dir, c("PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), sprintf("%s/%s", remote_bin_dir, c("PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), @@ -56,11 +97,5 @@ add_to_package_index <- function( s3 <- paws.storage::s3(endpoint = endpoint, region = region) s3$put_object(Body = pkgs_upload_local, Bucket = bucket, Key = "__linux__/rhel9/latest/src/contrib/PACKAGES") - # dummy <- s3fs::s3_file_upload(pkgs_upload_local, pkgs_upload_remote, overwrite = TRUE) - # s3fs::s3_file_upload( - # sprintf("%s/%s", local_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), - # sprintf("%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), - # overwrite = TRUE - # ) return(invisible(TRUE)) } diff --git a/R/upload.R b/R/upload.R index 28089d2..cb6734e 100644 --- a/R/upload.R +++ b/R/upload.R @@ -4,7 +4,8 @@ upload_single_binary_to_s3 <- function( endpoint = "https://s3.eu-central-003.backblazeb2.com", region = "eu-central-003", bucket = "devxy-arm64-r-binaries", - local_build_root = "/root", codename = NULL, + local_build_root = "/mnt/cache/binaries/", + codename = NULL, package_name, tag, r_minor_version = NULL, build_for_minor = FALSE, force = FALSE,