add upload function
This commit is contained in:
parent
d983f391fb
commit
2aed996280
GPG key ID:
3C6318841EF78925
2 changed files with 119 additions and 46 deletions
|
|
@ -61,8 +61,6 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
|||
|
||||
cli::cli_alert_info("Building binaries for {.pkg {package_name[[1]]}} with tags {.strong {tag}}.")
|
||||
|
||||
# devtools::install(quiet = TRUE)
|
||||
|
||||
# Set up the progress handler
|
||||
progressr::handlers(global = TRUE)
|
||||
progressr::handlers("cli", "debug")
|
||||
|
|
@ -73,11 +71,11 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
|||
tryCatch(
|
||||
{
|
||||
p()
|
||||
build_single_tag(x, y, dir_out_bin, local_clone_dir, platform = platform)
|
||||
store_build_metadata(x, y, platform, FALSE)
|
||||
build_single_tag(x, y, dir_out_bin, local_clone_dir, platform = platform)
|
||||
},
|
||||
error = function(e) {
|
||||
message("Error in processing package ", x, " with tag ", y, ": ")
|
||||
message(sprintf("Error in processing package %s with tag %s: %s", x, y, e))
|
||||
local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y)
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
store_build_metadata(x, y, platform, TRUE)
|
||||
|
|
@ -91,6 +89,22 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
|||
total_build_time <- round(Sys.time() - t1, 2)
|
||||
cli::cli_alert_info("Execution time for {.pkg {package_name[[1]]}} ({length(tag)} tags): {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.")
|
||||
|
||||
cli::cli_h2("Uploading")
|
||||
out <- progressr::with_progress({
|
||||
p <- progressr::progressor(along = tag)
|
||||
future_mapply(function(x, y) {
|
||||
tryCatch(
|
||||
{
|
||||
p()
|
||||
upload_single_binary_to_s3(package_name = x, tag = y, debug = F)
|
||||
},
|
||||
error = function(e) {
|
||||
message("Error in uploading package ", x, " with tag ", y, ": ")
|
||||
}
|
||||
)
|
||||
}, package_name, tag, future.seed = TRUE)
|
||||
})
|
||||
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
||||
|
|
@ -137,51 +151,49 @@ build_single_tag <- function(
|
|||
sprintf("https://github.com/cran/%s", package_name), local_clone_dir_single
|
||||
))
|
||||
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.strong {tag}}.")
|
||||
|
||||
t1 <- Sys.time()
|
||||
pkgbuild::build(
|
||||
path = sprintf("%s", local_clone_dir_single),
|
||||
binary = TRUE, vignettes = FALSE,
|
||||
dest_path = dir_out_bin
|
||||
)
|
||||
|
||||
if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
|
||||
cli::cli_alert_info('{.fun build_single_tag}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
|
||||
# remove _aarch64-unknown-linux-gnu part in filename
|
||||
system2("rsync", args = c(
|
||||
sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag),
|
||||
sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)
|
||||
))
|
||||
if (file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 2. Tarball for package {.pkg {package_name}} with tag {.strong {tag}} already exists. Skipping build.")
|
||||
} else {
|
||||
cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.')
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.strong {tag}}.")
|
||||
|
||||
t1 <- Sys.time()
|
||||
pkgbuild::build(
|
||||
path = sprintf("%s", local_clone_dir_single),
|
||||
binary = TRUE, vignettes = FALSE,
|
||||
dest_path = dir_out_bin
|
||||
)
|
||||
|
||||
if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
|
||||
cli::cli_alert_info('{.fun build_single_tag}: 3. Moving package from {.path {sprintf("%s/%s*.tar.gz", dir_out_bin, package_name)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
|
||||
# remove _aarch64-unknown-linux-gnu part in filename
|
||||
system2("rsync", args = c(
|
||||
sprintf("%s/%s_%s*.tar.gz", dir_out_bin, package_name, tag),
|
||||
sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)
|
||||
))
|
||||
} else {
|
||||
cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.')
|
||||
}
|
||||
unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag))
|
||||
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.")
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
|
||||
total_build_time <- round(Sys.time() - t1, 2)
|
||||
|
||||
# skip DB connection if package already exists
|
||||
if (file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io",
|
||||
port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS")
|
||||
)
|
||||
dbExecute(con, sprintf(
|
||||
"UPDATE single_builds set build_duration = %s WHERE package_name = '%s' and platform = '%s' and tag = '%s'",
|
||||
total_build_time, package_name, platform, tag
|
||||
))
|
||||
dbDisconnect(con)
|
||||
}
|
||||
}
|
||||
unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag))
|
||||
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.")
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
|
||||
total_build_time <- round(Sys.time() - t1, 2)
|
||||
|
||||
s3 <- paws.storage::s3(config = list(
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003"
|
||||
))
|
||||
db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
|
||||
writeBin(db_local$Body, "/tmp/metadata.sqlite")
|
||||
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
|
||||
dbExecute(con, "BEGIN TRANSACTION")
|
||||
# Create an index on the package_name and tag columns
|
||||
dbExecute(con, sprintf(
|
||||
"UPDATE metadata set build_time = '%s' where package_name = '%s' and platform = '%s' and tag = '%s'",
|
||||
total_build_time, package_name, platform, tag
|
||||
))
|
||||
dbExecute(con, "COMMIT")
|
||||
dbDisconnect(con)
|
||||
|
||||
invisible(s3$put_object(
|
||||
Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
|
||||
Body = "/tmp/metadata.sqlite"
|
||||
))
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
|
|
|||
61
R/upload.R
Normal file
61
R/upload.R
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
#' @importFrom paws.storage s3
|
||||
upload_single_binary_to_s3 <- function(
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
local_build_root = "/root", codename = NULL,
|
||||
package_name, tag,
|
||||
r_version_minor = NULL, build_for_minor = TRUE,
|
||||
force = FALSE,
|
||||
debug = FALSE) {
|
||||
|
||||
codename <- set_codename(codename)
|
||||
|
||||
if (is.null(r_version_minor)) {
|
||||
r_version_minor <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
|
||||
}
|
||||
|
||||
# create directory structure
|
||||
if (is.null(r_version_minor) && !build_for_minor) {
|
||||
dir_out_bin <- sprintf(
|
||||
"%s/__linux__/%s/latest/src/contrib",
|
||||
local_build_root, codename
|
||||
)
|
||||
} else {
|
||||
dir_out_bin <- sprintf(
|
||||
"%s/__linux__/%s/%s/latest/src/contrib",
|
||||
local_build_root, codename, r_version_minor
|
||||
)
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
cli::cli_alert_danger("DEBUG: dir_out_bin: {dir_out_bin}, codename: {codename}")
|
||||
cli::cli_alert_danger("DEBUG: Body: {sprintf('%s/%s', dir_out_bin, tarball_name)}")
|
||||
}
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region_name = "eu-central-003",
|
||||
)
|
||||
|
||||
tarball_name <- sprintf("%s_%s.tar.gz", package_name, tag)
|
||||
exists <- s3fs::s3_file_exists(sprintf("devxy-arm64-r-binaries/__linux__/%s/%s/latest/src/contrib/%s", codename, r_version_minor, tarball_name))
|
||||
|
||||
if (!exists && !force) {
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 5. Uploading {.pkg {package_name}} {.strong {tag}} to S3.")
|
||||
s3fs::s3_file_upload(
|
||||
sprintf("%s/%s", dir_out_bin, tarball_name),
|
||||
sprintf("devxy-arm64-r-binaries/__linux__/%s/%s/latest/src/contrib/%s", codename, r_version_minor, tarball_name)
|
||||
)
|
||||
} else if (exists && force) {
|
||||
cli::cli_alert_info("{.fun upload_to_s3}: Force uploading package {.pkg {package_name}} {.path {tag}} because force = TRUE was set.")
|
||||
s3fs::s3_file_upload(
|
||||
sprintf("%s/%s", dir_out_bin, tarball_name),
|
||||
sprintf("devxy-arm64-r-binaries/__linux__/%s/%s/latest/src/contrib/%s", codename, r_version_minor, tarball_name),
|
||||
overwrite = TRUE
|
||||
)
|
||||
} else if (exists && !force) {
|
||||
cli::cli_alert_info("{.fun upload_to_s3}: Package {.pkg {package_name}} {.path {tag}} already exists in S3. Skipping upload.")
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue