add upload function

R/build_binaries.R

@@ -61,8 +61,6 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
 
   cli::cli_alert_info("Building binaries for {.pkg {package_name[[1]]}} with tags {.strong {tag}}.")
 
-  # devtools::install(quiet = TRUE)
-
   # Set up the progress handler
   progressr::handlers(global = TRUE)
   progressr::handlers("cli", "debug")
@@ -73,11 +71,11 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
       tryCatch(
         {
           p()
-          build_single_tag(x, y, dir_out_bin, local_clone_dir, platform = platform)
           store_build_metadata(x, y, platform, FALSE)
+          build_single_tag(x, y, dir_out_bin, local_clone_dir, platform = platform)
         },
         error = function(e) {
-          message("Error in processing package ", x, " with tag ", y, ": ")
+          message(sprintf("Error in processing package %s with tag %s: %s", x, y, e))
           local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y)
           unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
           store_build_metadata(x, y, platform, TRUE)
@@ -91,6 +89,22 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
   total_build_time <- round(Sys.time() - t1, 2)
   cli::cli_alert_info("Execution time for {.pkg {package_name[[1]]}} ({length(tag)} tags): {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.")
 
+  cli::cli_h2("Uploading")
+  out <- progressr::with_progress({
+    p <- progressr::progressor(along = tag)
+    future_mapply(function(x, y) {
+      tryCatch(
+        {
+          p()
+          upload_single_binary_to_s3(package_name = x, tag = y, debug = F)
+        },
+        error = function(e) {
+          message("Error in uploading package ", x, " with tag ", y, ": ")
+        }
+      )
+    }, package_name, tag, future.seed = TRUE)
+  })
+
   return(invisible(TRUE))
 }
 
@@ -137,51 +151,49 @@ build_single_tag <- function(
     sprintf("https://github.com/cran/%s", package_name), local_clone_dir_single
   ))
 
-  cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.strong {tag}}.")
+  if (file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
-
+    cli::cli_alert_info("{.fun build_single_tag}: 2. Tarball for package {.pkg {package_name}} with tag {.strong {tag}} already exists. Skipping build.")
-  t1 <- Sys.time()
-  pkgbuild::build(
-    path = sprintf("%s", local_clone_dir_single),
-    binary = TRUE, vignettes = FALSE,
-    dest_path = dir_out_bin
-  )
-
-  if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
-    cli::cli_alert_info('{.fun build_single_tag}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
-    # remove _aarch64-unknown-linux-gnu part in filename
-    system2("rsync", args = c(
-      sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag),
-      sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)
-    ))
   } else {
-    cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.')
+    cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.strong {tag}}.")
+
+    t1 <- Sys.time()
+    pkgbuild::build(
+      path = sprintf("%s", local_clone_dir_single),
+      binary = TRUE, vignettes = FALSE,
+      dest_path = dir_out_bin
+    )
+
+    if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
+      cli::cli_alert_info('{.fun build_single_tag}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
+      # remove _aarch64-unknown-linux-gnu part in filename
+      system2("rsync", args = c(
+        sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag),
+        sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)
+      ))
+    } else {
+      cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.')
+    }
+    unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag))
+
+    cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.")
+    unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
+
+    total_build_time <- round(Sys.time() - t1, 2)
+
+    # skip DB connection if package already exists
+    if (file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
+      con <- DBI::dbConnect(RPostgres::Postgres(),
+        dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io",
+        port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS")
+      )
+      dbExecute(con, sprintf(
+        "UPDATE single_builds set build_duration = %s WHERE package_name = '%s' and platform = '%s' and tag = '%s'",
+        total_build_time, package_name, platform, tag
+      ))
+      dbDisconnect(con)
+    }
   }
-  unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag))
 
-  cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.")
-  unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
 
-  total_build_time <- round(Sys.time() - t1, 2)
+  return(invisible(TRUE))
-
-  s3 <- paws.storage::s3(config = list(
-    endpoint = "https://s3.eu-central-003.backblazeb2.com",
-    region = "eu-central-003"
-  ))
-  db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
-  writeBin(db_local$Body, "/tmp/metadata.sqlite")
-
-  con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
-  dbExecute(con, "BEGIN TRANSACTION")
-  # Create an index on the package_name and tag columns
-  dbExecute(con, sprintf(
-    "UPDATE metadata set build_time = '%s' where package_name = '%s' and platform = '%s' and tag = '%s'",
-    total_build_time, package_name, platform, tag
-  ))
-  dbExecute(con, "COMMIT")
-  dbDisconnect(con)
-
-  invisible(s3$put_object(
-    Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
-    Body = "/tmp/metadata.sqlite"
-  ))
 }

R/upload.R

@@ -0,0 +1,61 @@
+#' @importFrom paws.storage s3
+upload_single_binary_to_s3 <- function(
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    region = "eu-central-003",
+    local_build_root = "/root", codename = NULL,
+    package_name, tag,
+    r_version_minor = NULL, build_for_minor = TRUE,
+    force = FALSE,
+    debug = FALSE) {
+
+  codename <- set_codename(codename)
+
+  if (is.null(r_version_minor)) {
+    r_version_minor <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
+  }
+
+  # create directory structure
+  if (is.null(r_version_minor) && !build_for_minor) {
+    dir_out_bin <- sprintf(
+      "%s/__linux__/%s/latest/src/contrib",
+      local_build_root, codename
+    )
+  } else {
+    dir_out_bin <- sprintf(
+      "%s/__linux__/%s/%s/latest/src/contrib",
+      local_build_root, codename, r_version_minor
+    )
+  }
+
+  if (debug) {
+    cli::cli_alert_danger("DEBUG: dir_out_bin: {dir_out_bin}, codename: {codename}")
+    cli::cli_alert_danger("DEBUG: Body: {sprintf('%s/%s', dir_out_bin, tarball_name)}")
+  }
+
+  s3fs::s3_file_system(
+    aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
+    aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    region_name = "eu-central-003",
+  )
+
+  tarball_name <- sprintf("%s_%s.tar.gz", package_name, tag)
+  exists <- s3fs::s3_file_exists(sprintf("devxy-arm64-r-binaries/__linux__/%s/%s/latest/src/contrib/%s", codename, r_version_minor, tarball_name))
+
+  if (!exists && !force) {
+    cli::cli_alert_info("{.fun build_single_tag}: 5. Uploading {.pkg {package_name}} {.strong {tag}} to S3.")
+    s3fs::s3_file_upload(
+      sprintf("%s/%s", dir_out_bin, tarball_name),
+      sprintf("devxy-arm64-r-binaries/__linux__/%s/%s/latest/src/contrib/%s", codename, r_version_minor, tarball_name)
+    )
+  } else if (exists && force) {
+    cli::cli_alert_info("{.fun upload_to_s3}: Force uploading package {.pkg {package_name}} {.path {tag}} because force = TRUE was set.")
+    s3fs::s3_file_upload(
+      sprintf("%s/%s", dir_out_bin, tarball_name),
+      sprintf("devxy-arm64-r-binaries/__linux__/%s/%s/latest/src/contrib/%s", codename, r_version_minor, tarball_name),
+      overwrite = TRUE
+    )
+  } else if (exists && !force) {
+    cli::cli_alert_info("{.fun upload_to_s3}: Package {.pkg {package_name}} {.path {tag}} already exists in S3. Skipping upload.")
+  }
+}