diff --git a/R/build_binaries.R b/R/build_binaries.R index 02cfd98..3a87a55 100644 --- a/R/build_binaries.R +++ b/R/build_binaries.R @@ -12,6 +12,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, local_build_root = "/root", local_clone_dir = "/tmp", platform = NULL, + arch = NULL, install_system_dependencies = TRUE, deps_verbose = FALSE, debug = FALSE, @@ -46,6 +47,14 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, local_bin_path <- set_bin_path(r_minor_version, build_for_minor, local_build_root = local_build_root, codename) + # set arch + local_arch <- Sys.info()[["machine"]] + if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) { + arch <- "arm64" + } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) { + arch <- "amd64" + } + if (debug) { cli::cli_alert_warning("DEBUG: dir_out_bin {dir_out_bin}.") } @@ -96,7 +105,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, tryCatch( { dump <- build_single_tag(x, y, dir_out_bin, local_clone_dir, - platform = platform, debug = debug, force = force, + platform = platform, arch = arch, debug = debug, force = force, install_system_dependencies = install_system_dependencies, deps_verbose = deps_verbose ) @@ -107,7 +116,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, cli::cli_alert_success("Successfully built package {.pkg {x}} with tag {.field {y}}.") } else { cli::cli_alert_warning("Error in building package {.pkg {x}} with tag {.field {y}}: Uncommon/unspecific error during build.") - store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE, error = "Uncommon/unspecific error during build") + store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE, arch = arch, error = "Uncommon/unspecific error during build") } }, error = function(e) { @@ -115,7 +124,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y) unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) # only stderr contains the important information why the build failed - store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE, error = e$stderr) + store_build_metadata(x, y, platform, error_occurred = TRUE, arch = arch, force = TRUE, error = e$stderr) } ) } @@ -154,8 +163,10 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, #' @importFrom fs file_size file_move #' @export build_single_tag <- function( - package_name, tag = NULL, + package_name, + tag = NULL, platform, + arch, dir_out_bin, local_clone_dir, debug = FALSE, @@ -181,7 +192,7 @@ build_single_tag <- function( # NB: here we need to use conditionMessage() to extract the actual error - as opposed to using $stderr for errors within the tryCatch used in the future* calls error = function(e) { cli::cli_alert_warning("Error in installing dependencies for package {.pkg {package_name[1]}} with tag {.field {tag[1]}}: {e}") - store_build_metadata(package_name[1], tag[1], platform, error_occurred = TRUE, force = TRUE, error = conditionMessage(e)) + store_build_metadata(package_name[1], tag[1], platform, arch = arch, error_occurred = TRUE, force = TRUE, error = conditionMessage(e)) return(TRUE) } ) @@ -217,7 +228,7 @@ build_single_tag <- function( cli::cli_alert_warning("Error in starting build command for package {.pkg {package_name}} with tag {.field {tag}}: {e}") local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag) unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) - store_build_metadata(package_name, tag, platform, error_occurred = TRUE, force = TRUE, error = sprintf("Error trying to initiate pkgbuild - likely a non-valid R package structure. Full error: %s", e)) + store_build_metadata(package_name, tag, platform, arch = arch, error_occurred = TRUE, force = TRUE, error = sprintf("Error trying to initiate pkgbuild - likely a non-valid R package structure. Full error: %s", e)) return(invisible(TRUE)) } ) @@ -228,28 +239,27 @@ build_single_tag <- function( linux_suffix <- "gnu" } + # set tarball id for arch local_arch <- Sys.info()[["machine"]] if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) { - arch <- "aarch64" - whatever_id <- "unknown" + tarball_id <- "unknown" } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) { - arch <- "x86_64" - whatever_id <- "pc" + tarball_id <- "pc" } if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) { if (debug) { - cli::cli_alert_info('{.fun build_single_tag}: DEBUG: Moving package from {.path {sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, whatever_id, linux_suffix)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}') + cli::cli_alert_info('{.fun build_single_tag}: DEBUG: Moving package from {.path {sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, tarball_id, linux_suffix)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}') } # double-check that file exists (some packages like https://github.com/cran/BACCO/tree/1.0-14 don't include R/ and hence don't procude a valid binary) - if (fs::file_exists(sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, whatever_id, linux_suffix))) { + if (fs::file_exists(sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, tarball_id, linux_suffix))) { # remove _aarch64-unknown-linux-gnu/musl part in filename fs::file_move( - sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, whatever_id, linux_suffix), + sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, tarball_id, linux_suffix), sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag) ) } else { - cli::cli_alert_info('{.fun build_single_tag}: File for package {.pkg {package_name}} {.field {tag}} at {.path {sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, whatever_id, linux_suffix)}} does not exist - skipping.') + cli::cli_alert_info('{.fun build_single_tag}: File for package {.pkg {package_name}} {.field {tag}} at {.path {sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, tarball_id, linux_suffix)}} does not exist - skipping.') if (debug) { message(sprintf("DEBUG: Listing dir 'dir_out_bin': %s", dir_out_bin)) message(fs::dir_ls(dir_out_bin)) @@ -275,7 +285,7 @@ build_single_tag <- function( tarball_name <- sprintf("%s_%s.tar.gz", package_name, tag) if (fs::file_exists(sprintf("%s/%s", dir_out_bin, tarball_name))) { - store_build_metadata(package_name, tag, platform, error_occurred = FALSE, force = force, build_duration = total_build_time, size = file_size) + store_build_metadata(package_name, tag, platform, arch = arch, error_occurred = FALSE, force = force, build_duration = total_build_time, size = file_size) } } diff --git a/R/metadata-build.R b/R/metadata-build.R index 6a4eed5..25b525b 100644 --- a/R/metadata-build.R +++ b/R/metadata-build.R @@ -1,7 +1,7 @@ #' Stores build metadata of single binary builds in a Postgres DB #' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute store_build_metadata <- function( - package_name, tag, platform, error_occurred, + package_name, tag, platform, error_occurred, arch, force = FALSE, error = NA, build_duration = NA, size = NA) { con <- DBI::dbConnect(RPostgres::Postgres(), dbname = "build_metadata", host = "r-binaries.devxy.io", @@ -11,7 +11,7 @@ store_build_metadata <- function( # Check if an entry with the same package_name and tag already exists existing_entries <- dbGetQuery(con, paste0( "SELECT * FROM single_builds WHERE name = '", - package_name, "' AND tag = '", tag, "' AND platform = '", platform, "'" + package_name, "' AND tag = '", tag, "' AND platform = '", arch, "' AND arch = '", platform, "'" )) if (nrow(existing_entries) >= 1 && !force) { @@ -29,6 +29,7 @@ store_build_metadata <- function( name = package_name, tag = tag, platform = platform, + arch = arch, error_occurred = error_occurred, error_text = error, size = size, @@ -90,7 +91,9 @@ build_metadata_summary <- function(package_name, platform) { ", percentage_successful_builds = ", summary$percentage_successful_builds, ", average_build_time_per_tag = '", summary$average_build_time_per_tag, "', package_name = '", package_name, - "', platform = '", platform, "'" + "', platform = '", platform, "'", + "', arch = '", platform, "'", + )) } else { dbWriteTable(con, "metadata_summary", summary) diff --git a/man/archive_package.Rd b/man/archive_package.Rd index fe58177..da3a68e 100644 --- a/man/archive_package.Rd +++ b/man/archive_package.Rd @@ -13,7 +13,9 @@ archive_package( endpoint = "https://s3.eu-central-003.backblazeb2.com", region = "eu-central-003", bucket = "devxy-arm64-r-binaries", - pause = NULL + pause = NULL, + arch = NULL, + debug = FALSE ) } \description{ @@ -21,5 +23,6 @@ Archive packages in CRAN-like repositories } \examples{ archive_package("AATtools", codename = "rhel9") +archive_package("adw", codename = "rhel8", arch = "amd64") } diff --git a/man/build_binary_package.Rd b/man/build_binary_package.Rd index 798c5cc..8ed2cd9 100644 --- a/man/build_binary_package.Rd +++ b/man/build_binary_package.Rd @@ -13,11 +13,14 @@ build_binary_package( local_build_root = "/root", local_clone_dir = "/tmp", platform = NULL, + arch = NULL, install_system_dependencies = TRUE, deps_verbose = FALSE, debug = FALSE, force = FALSE, - archive = TRUE + archive = TRUE, + future_strategy = "multisession", + future_workers = 2 ) } \description{ diff --git a/man/store_build_metadata.Rd b/man/store_build_metadata.Rd index 25ec899..740205d 100644 --- a/man/store_build_metadata.Rd +++ b/man/store_build_metadata.Rd @@ -9,6 +9,7 @@ store_build_metadata( tag, platform, error_occurred, + arch, force = FALSE, error = NA, build_duration = NA,