diff --git a/NAMESPACE b/NAMESPACE index 5a340a2..8b4073d 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -4,6 +4,7 @@ export(build_binary_package) export(build_single_tag) export(dbcon) export(install_package_system_dependencies) +export(list_metadata_tables) export(set_bin_path) export(set_codename) export(upload_single_binary_to_s3) @@ -17,13 +18,11 @@ importFrom(DBI,dbWriteTable) importFrom(RPostgres,Postgres) importFrom(cli,cli_alert) importFrom(cranlike,add_PACKAGES) -importFrom(dplyr,all_vars) importFrom(dplyr,filter) -importFrom(dplyr,filter_at) importFrom(dplyr,group_by) importFrom(dplyr,n) importFrom(dplyr,summarise) -importFrom(dplyr,vars) +importFrom(dplyr,tbl) importFrom(future,plan) importFrom(future.apply,future_mapply) importFrom(gert,git_config_set) diff --git a/R/build-metadata.R b/R/build-metadata.R deleted file mode 100644 index 4e08791..0000000 --- a/R/build-metadata.R +++ /dev/null @@ -1,179 +0,0 @@ -#' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute -store_build_metadata <- function( - package_name, tag, platform, - error_occurred, error, force = FALSE) { - # s3 <- paws.storage::s3(config = list( - # endpoint = "https://s3.eu-central-003.backblazeb2.com", - # region = "eu-central-003" - # )) - # db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite") - # writeBin(db_local$Body, "/tmp/metadata.sqlite") - - # Create a new SQLite connection - con <- DBI::dbConnect(RPostgres::Postgres(), - dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io", - port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS") - ) - - # Create an index on the package_name and tag columns - dbExecute(con, "CREATE INDEX IF NOT EXISTS idx_metadata_package_tag ON single_builds(package_name, platform)") - - # Check if an entry with the same package_name and tag already exists - existing_entries <- dbGetQuery(con, paste0( - "SELECT * FROM single_builds WHERE package_name = '", - package_name, "' AND tag = '", tag, "'" - )) - - if (nrow(existing_entries) >= 1 && !force) { - cli::cli_alert("{.fun store_build_metadata}: Build metadata for {.field {.pkg package_name}} {.field {tag}} already exists.") - } else if (nrow(existing_entries) >= 1 && force) { - cli::cli_alert_info("{.fun store_build_metadata}: Force overwriting build metadata for {.pkg {package_name}} {.field {tag}} because {.code force = TRUE} was set.") - - dbExecute(con, sprintf("UPDATE single_builds SET build_timestamp = '%s' WHERE package_name = '%s' and tag = '%s'", format(Sys.time(), "%Y-%m-%d %H:%M:%S"), package_name, tag)) - } else if (nrow(existing_entries) == 0) { - cli::cli_alert("{.fun store_build_metadata}: Storing build metadata for {.pkg {package_name}} {.field {tag}}.") - # Create a data frame with the metadata - metadata <- data.frame( - package_name = package_name, - tag = tag, - platform = platform, - error_occurred = error_occurred, - build_timestamp = format(Sys.time(), "%Y-%m-%d %H:%M:%S"), - build_duration = NA - ) - # Write the data frame to the SQLite database - dbWriteTable(con, "single_builds", metadata, append = TRUE) - } - - # Close the SQLite connection - dbDisconnect(con) - - return(invisible(TRUE)) -} - -#' @importFrom DBI dbConnect dbGetQuery -#' @importFrom dplyr filter_at vars all_vars -query_build_metadata <- function(table = "single_builds", ...) { - # s3 <- paws.storage::s3(config = list( - # endpoint = "https://s3.eu-central-003.backblazeb2.com", - # region = "eu-central-003" - # )) - # db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite") - # writeBin(db_local$Body, "/tmp/metadata.sqlite") - - con <- dbcon_mem() - - # Read the entire table - metadata <- dbGetQuery(con, sprintf("SELECT * FROM %s", table)) - - # Close the SQLite connection - dbDisconnect(con) - - # Filter the data frame based on the given parameters - filter_conditions <- list(...) - - # Remove NULL elements from the list - filter_conditions <- filter_conditions[!sapply(filter_conditions, is.null)] - - # Apply the filter conditions if any exist - if (length(filter_conditions) > 0) { - metadata <- dplyr::filter_at( - metadata, dplyr::vars(names(filter_conditions)), - dplyr::all_vars(. == filter_conditions[[.]]) - ) - } - - return(metadata) -} - - -#' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute dbListTables -#' @importFrom dplyr group_by summarise filter n -build_metadata_summary <- function(package_name, platform) { - con <- dbcon_mem() - - metadata <- dbGetQuery(con, sprintf( - "SELECT * FROM single_builds WHERE package_name = '%s' AND platform = '%s'", package_name, platform - )) - - cli::cli_alert("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.") - - # add average_build_time_per_tag to metadata summary table - - # If the entry does not exist or is not valid, entry will be NULL or NA - # entry_exists <- !is.null(entry$average_build_time_per_tag) && !is.na(entry$average_build_time_per_tag) - - # If the entry does not exist, update the average_build_time_per_tag - # if (!entry_exists) { - # dbExecute(con, paste0( - # "UPDATE metadata_summary SET average_build_time_per_tag = ", average_build_time_per_tag, - # " WHERE package_name = '", package_name[[1]], "' AND platform = '", platform, "'" - # )) - # } - - # Calculate the summary - summary <- metadata %>% - group_by(package_name, platform) %>% - summarise( - successful_builds = sum(!error_occurred), - unsuccessful_builds = sum(error_occurred), - total_builds = n(), - percentage_successful_builds = round(successful_builds / total_builds * 100, 2), - average_build_time_per_tag = round(sum(as.numeric(build_duration)) / total_builds, 2) - ) - - # Check if the metadata_summary table exists - if ("metadata_summary" %in% dbListTables(con)) { - dbExecute(con, paste0( - "UPDATE metadata_summary SET successful_builds = ", summary$successful_builds, - ", unsuccessful_builds = ", summary$unsuccessful_builds, - ", total_builds = ", summary$total_builds, - ", percentage_successful_builds = ", summary$percentage_successful_builds, - ", average_build_time_per_tag = '", summary$average_build_time_per_tag, - "', package_name = '", package_name, - "', platform = '", platform, "'" - )) - } else { - dbWriteTable(con, "metadata_summary", summary) - } - - # Close the SQLite connection - dbDisconnect(con) - - # s3 <- paws.storage::s3(config = list( - # endpoint = "https://s3.eu-central-003.backblazeb2.com", - # region = "eu-central-003" - # )) - # invisible(s3$put_object( - # Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite", - # Body = "/tmp/metadata.sqlite" - # )) -} - -query_metadata_summary <- function(package_name = NULL, platform = NULL) { - # s3 <- paws.storage::s3(config = list( - # endpoint = "https://s3.eu-central-003.backblazeb2.com", - # region = "eu-central-003" - # )) - # db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite") - # writeBin(db_local$Body, "/tmp/metadata.sqlite") - - # Create a new SQLite connection - con <- dbcon_mem() - - # If both package_name and platform are NULL, return the full table - if (is.null(package_name) && is.null(platform)) { - summary <- dbGetQuery(con, "SELECT * FROM metadata_summary") - } else { - # Query the metadata_summary table for the specified package and platform - summary <- dbGetQuery(con, paste0( - "SELECT * FROM metadata_summary WHERE package_name = '", - package_name, "' AND platform = '", platform, "'" - )) - } - - # Close the SQLite connection - dbDisconnect(con) - - return(summary) -} diff --git a/R/build_binaries.R b/R/build_binaries.R index bc4ceb1..419d7bb 100644 --- a/R/build_binaries.R +++ b/R/build_binaries.R @@ -107,7 +107,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y) unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) # NB: this does not always work, which is why we are using the lack of build_duration as a secondary factor in store_metadata() - store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE) + store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE, error = e) } ) }, package_name, tag, future.seed = TRUE) diff --git a/R/metadata-build.R b/R/metadata-build.R index f4e1a3b..5ee3621 100644 --- a/R/metadata-build.R +++ b/R/metadata-build.R @@ -1,7 +1,7 @@ #' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute store_build_metadata <- function( package_name, tag, platform, - error_occurred, error, force = FALSE) { + error_occurred, error, force = FALSE, error = NULL) { con <- DBI::dbConnect(RPostgres::Postgres(), dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io", port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS") @@ -18,7 +18,7 @@ store_build_metadata <- function( } else if (nrow(existing_entries) >= 1 && force) { cli::cli_alert_info("{.fun store_build_metadata}: Force overwriting build metadata for {.pkg {package_name}} {.field {tag}} because {.code force = TRUE} was set.") - dbExecute(con, sprintf("UPDATE single_builds SET build_timestamp = '%s', error_occurred = %s WHERE package_name = '%s' and tag = '%s'", format(Sys.time(), "%Y-%m-%d %H:%M:%S"), error_occurred, package_name, tag)) + dbExecute(con, sprintf("UPDATE single_builds SET build_timestamp = '%s', error_occurred = %s, error = '%s' WHERE package_name = '%s' and tag = '%s'", format(Sys.time(), "%Y-%m-%d %H:%M:%S"), error_occurred, error, package_name, tag)) } else if (nrow(existing_entries) == 0) { cli::cli_alert("{.fun store_build_metadata}: Storing build metadata for {.pkg {package_name}} {.field {tag}}.") # Create a data frame with the metadata @@ -27,6 +27,7 @@ store_build_metadata <- function( tag = tag, platform = platform, error_occurred = error_occurred, + error = error, build_timestamp = format(Sys.time(), "%Y-%m-%d %H:%M:%S"), build_duration = NA )