From 2aed99628057ff299227419ead8f558773385bdf Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 23 Jul 2024 21:45:25 +0200 Subject: [PATCH] add upload function --- R/build_binaries.R | 106 +++++++++++++++++++++++++-------------------- R/upload.R | 61 ++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 47 deletions(-) create mode 100644 R/upload.R diff --git a/R/build_binaries.R b/R/build_binaries.R index 2cf7b59..223cdea 100644 --- a/R/build_binaries.R +++ b/R/build_binaries.R @@ -61,8 +61,6 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, cli::cli_alert_info("Building binaries for {.pkg {package_name[[1]]}} with tags {.strong {tag}}.") - # devtools::install(quiet = TRUE) - # Set up the progress handler progressr::handlers(global = TRUE) progressr::handlers("cli", "debug") @@ -73,11 +71,11 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, tryCatch( { p() - build_single_tag(x, y, dir_out_bin, local_clone_dir, platform = platform) store_build_metadata(x, y, platform, FALSE) + build_single_tag(x, y, dir_out_bin, local_clone_dir, platform = platform) }, error = function(e) { - message("Error in processing package ", x, " with tag ", y, ": ") + message(sprintf("Error in processing package %s with tag %s: %s", x, y, e)) local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y) unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) store_build_metadata(x, y, platform, TRUE) @@ -91,6 +89,22 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, total_build_time <- round(Sys.time() - t1, 2) cli::cli_alert_info("Execution time for {.pkg {package_name[[1]]}} ({length(tag)} tags): {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.") + cli::cli_h2("Uploading") + out <- progressr::with_progress({ + p <- progressr::progressor(along = tag) + future_mapply(function(x, y) { + tryCatch( + { + p() + upload_single_binary_to_s3(package_name = x, tag = y, debug = F) + }, + error = function(e) { + message("Error in uploading package ", x, " with tag ", y, ": ") + } + ) + }, package_name, tag, future.seed = TRUE) + }) + return(invisible(TRUE)) } @@ -137,51 +151,49 @@ build_single_tag <- function( sprintf("https://github.com/cran/%s", package_name), local_clone_dir_single )) - cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.strong {tag}}.") - - t1 <- Sys.time() - pkgbuild::build( - path = sprintf("%s", local_clone_dir_single), - binary = TRUE, vignettes = FALSE, - dest_path = dir_out_bin - ) - - if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) { - cli::cli_alert_info('{.fun build_single_tag}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}') - # remove _aarch64-unknown-linux-gnu part in filename - system2("rsync", args = c( - sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag), - sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag) - )) + if (file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) { + cli::cli_alert_info("{.fun build_single_tag}: 2. Tarball for package {.pkg {package_name}} with tag {.strong {tag}} already exists. Skipping build.") } else { - cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.') + cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.strong {tag}}.") + + t1 <- Sys.time() + pkgbuild::build( + path = sprintf("%s", local_clone_dir_single), + binary = TRUE, vignettes = FALSE, + dest_path = dir_out_bin + ) + + if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) { + cli::cli_alert_info('{.fun build_single_tag}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}') + # remove _aarch64-unknown-linux-gnu part in filename + system2("rsync", args = c( + sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag), + sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag) + )) + } else { + cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.') + } + unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag)) + + cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.") + unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) + + total_build_time <- round(Sys.time() - t1, 2) + + # skip DB connection if package already exists + if (file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) { + con <- DBI::dbConnect(RPostgres::Postgres(), + dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io", + port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS") + ) + dbExecute(con, sprintf( + "UPDATE single_builds set build_duration = %s WHERE package_name = '%s' and platform = '%s' and tag = '%s'", + total_build_time, package_name, platform, tag + )) + dbDisconnect(con) + } } - unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag)) - cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.") - unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) - total_build_time <- round(Sys.time() - t1, 2) - - s3 <- paws.storage::s3(config = list( - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region = "eu-central-003" - )) - db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite") - writeBin(db_local$Body, "/tmp/metadata.sqlite") - - con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite") - dbExecute(con, "BEGIN TRANSACTION") - # Create an index on the package_name and tag columns - dbExecute(con, sprintf( - "UPDATE metadata set build_time = '%s' where package_name = '%s' and platform = '%s' and tag = '%s'", - total_build_time, package_name, platform, tag - )) - dbExecute(con, "COMMIT") - dbDisconnect(con) - - invisible(s3$put_object( - Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite", - Body = "/tmp/metadata.sqlite" - )) + return(invisible(TRUE)) } diff --git a/R/upload.R b/R/upload.R new file mode 100644 index 0000000..32821f7 --- /dev/null +++ b/R/upload.R @@ -0,0 +1,61 @@ +#' @importFrom paws.storage s3 +upload_single_binary_to_s3 <- function( + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region = "eu-central-003", + local_build_root = "/root", codename = NULL, + package_name, tag, + r_version_minor = NULL, build_for_minor = TRUE, + force = FALSE, + debug = FALSE) { + + codename <- set_codename(codename) + + if (is.null(r_version_minor)) { + r_version_minor <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string) + } + + # create directory structure + if (is.null(r_version_minor) && !build_for_minor) { + dir_out_bin <- sprintf( + "%s/__linux__/%s/latest/src/contrib", + local_build_root, codename + ) + } else { + dir_out_bin <- sprintf( + "%s/__linux__/%s/%s/latest/src/contrib", + local_build_root, codename, r_version_minor + ) + } + + if (debug) { + cli::cli_alert_danger("DEBUG: dir_out_bin: {dir_out_bin}, codename: {codename}") + cli::cli_alert_danger("DEBUG: Body: {sprintf('%s/%s', dir_out_bin, tarball_name)}") + } + + s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"), + aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + ) + + tarball_name <- sprintf("%s_%s.tar.gz", package_name, tag) + exists <- s3fs::s3_file_exists(sprintf("devxy-arm64-r-binaries/__linux__/%s/%s/latest/src/contrib/%s", codename, r_version_minor, tarball_name)) + + if (!exists && !force) { + cli::cli_alert_info("{.fun build_single_tag}: 5. Uploading {.pkg {package_name}} {.strong {tag}} to S3.") + s3fs::s3_file_upload( + sprintf("%s/%s", dir_out_bin, tarball_name), + sprintf("devxy-arm64-r-binaries/__linux__/%s/%s/latest/src/contrib/%s", codename, r_version_minor, tarball_name) + ) + } else if (exists && force) { + cli::cli_alert_info("{.fun upload_to_s3}: Force uploading package {.pkg {package_name}} {.path {tag}} because force = TRUE was set.") + s3fs::s3_file_upload( + sprintf("%s/%s", dir_out_bin, tarball_name), + sprintf("devxy-arm64-r-binaries/__linux__/%s/%s/latest/src/contrib/%s", codename, r_version_minor, tarball_name), + overwrite = TRUE + ) + } else if (exists && !force) { + cli::cli_alert_info("{.fun upload_to_s3}: Package {.pkg {package_name}} {.path {tag}} already exists in S3. Skipping upload.") + } +}