add error to metadata table
This commit is contained in:
parent
d1e4c48104
commit
687b2cea36
GPG key ID:
3C6318841EF78925
3 changed files with 6 additions and 185 deletions
|
|
@ -1,179 +0,0 @@
|
|||
#' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute
|
||||
store_build_metadata <- function(
|
||||
package_name, tag, platform,
|
||||
error_occurred, error, force = FALSE) {
|
||||
# s3 <- paws.storage::s3(config = list(
|
||||
# endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
# region = "eu-central-003"
|
||||
# ))
|
||||
# db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
|
||||
# writeBin(db_local$Body, "/tmp/metadata.sqlite")
|
||||
|
||||
# Create a new SQLite connection
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io",
|
||||
port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS")
|
||||
)
|
||||
|
||||
# Create an index on the package_name and tag columns
|
||||
dbExecute(con, "CREATE INDEX IF NOT EXISTS idx_metadata_package_tag ON single_builds(package_name, platform)")
|
||||
|
||||
# Check if an entry with the same package_name and tag already exists
|
||||
existing_entries <- dbGetQuery(con, paste0(
|
||||
"SELECT * FROM single_builds WHERE package_name = '",
|
||||
package_name, "' AND tag = '", tag, "'"
|
||||
))
|
||||
|
||||
if (nrow(existing_entries) >= 1 && !force) {
|
||||
cli::cli_alert("{.fun store_build_metadata}: Build metadata for {.field {.pkg package_name}} {.field {tag}} already exists.")
|
||||
} else if (nrow(existing_entries) >= 1 && force) {
|
||||
cli::cli_alert_info("{.fun store_build_metadata}: Force overwriting build metadata for {.pkg {package_name}} {.field {tag}} because {.code force = TRUE} was set.")
|
||||
|
||||
dbExecute(con, sprintf("UPDATE single_builds SET build_timestamp = '%s' WHERE package_name = '%s' and tag = '%s'", format(Sys.time(), "%Y-%m-%d %H:%M:%S"), package_name, tag))
|
||||
} else if (nrow(existing_entries) == 0) {
|
||||
cli::cli_alert("{.fun store_build_metadata}: Storing build metadata for {.pkg {package_name}} {.field {tag}}.")
|
||||
# Create a data frame with the metadata
|
||||
metadata <- data.frame(
|
||||
package_name = package_name,
|
||||
tag = tag,
|
||||
platform = platform,
|
||||
error_occurred = error_occurred,
|
||||
build_timestamp = format(Sys.time(), "%Y-%m-%d %H:%M:%S"),
|
||||
build_duration = NA
|
||||
)
|
||||
# Write the data frame to the SQLite database
|
||||
dbWriteTable(con, "single_builds", metadata, append = TRUE)
|
||||
}
|
||||
|
||||
# Close the SQLite connection
|
||||
dbDisconnect(con)
|
||||
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
||||
#' @importFrom DBI dbConnect dbGetQuery
|
||||
#' @importFrom dplyr filter_at vars all_vars
|
||||
query_build_metadata <- function(table = "single_builds", ...) {
|
||||
# s3 <- paws.storage::s3(config = list(
|
||||
# endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
# region = "eu-central-003"
|
||||
# ))
|
||||
# db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
|
||||
# writeBin(db_local$Body, "/tmp/metadata.sqlite")
|
||||
|
||||
con <- dbcon_mem()
|
||||
|
||||
# Read the entire table
|
||||
metadata <- dbGetQuery(con, sprintf("SELECT * FROM %s", table))
|
||||
|
||||
# Close the SQLite connection
|
||||
dbDisconnect(con)
|
||||
|
||||
# Filter the data frame based on the given parameters
|
||||
filter_conditions <- list(...)
|
||||
|
||||
# Remove NULL elements from the list
|
||||
filter_conditions <- filter_conditions[!sapply(filter_conditions, is.null)]
|
||||
|
||||
# Apply the filter conditions if any exist
|
||||
if (length(filter_conditions) > 0) {
|
||||
metadata <- dplyr::filter_at(
|
||||
metadata, dplyr::vars(names(filter_conditions)),
|
||||
dplyr::all_vars(. == filter_conditions[[.]])
|
||||
)
|
||||
}
|
||||
|
||||
return(metadata)
|
||||
}
|
||||
|
||||
|
||||
#' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute dbListTables
|
||||
#' @importFrom dplyr group_by summarise filter n
|
||||
build_metadata_summary <- function(package_name, platform) {
|
||||
con <- dbcon_mem()
|
||||
|
||||
metadata <- dbGetQuery(con, sprintf(
|
||||
"SELECT * FROM single_builds WHERE package_name = '%s' AND platform = '%s'", package_name, platform
|
||||
))
|
||||
|
||||
cli::cli_alert("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.")
|
||||
|
||||
# add average_build_time_per_tag to metadata summary table
|
||||
|
||||
# If the entry does not exist or is not valid, entry will be NULL or NA
|
||||
# entry_exists <- !is.null(entry$average_build_time_per_tag) && !is.na(entry$average_build_time_per_tag)
|
||||
|
||||
# If the entry does not exist, update the average_build_time_per_tag
|
||||
# if (!entry_exists) {
|
||||
# dbExecute(con, paste0(
|
||||
# "UPDATE metadata_summary SET average_build_time_per_tag = ", average_build_time_per_tag,
|
||||
# " WHERE package_name = '", package_name[[1]], "' AND platform = '", platform, "'"
|
||||
# ))
|
||||
# }
|
||||
|
||||
# Calculate the summary
|
||||
summary <- metadata %>%
|
||||
group_by(package_name, platform) %>%
|
||||
summarise(
|
||||
successful_builds = sum(!error_occurred),
|
||||
unsuccessful_builds = sum(error_occurred),
|
||||
total_builds = n(),
|
||||
percentage_successful_builds = round(successful_builds / total_builds * 100, 2),
|
||||
average_build_time_per_tag = round(sum(as.numeric(build_duration)) / total_builds, 2)
|
||||
)
|
||||
|
||||
# Check if the metadata_summary table exists
|
||||
if ("metadata_summary" %in% dbListTables(con)) {
|
||||
dbExecute(con, paste0(
|
||||
"UPDATE metadata_summary SET successful_builds = ", summary$successful_builds,
|
||||
", unsuccessful_builds = ", summary$unsuccessful_builds,
|
||||
", total_builds = ", summary$total_builds,
|
||||
", percentage_successful_builds = ", summary$percentage_successful_builds,
|
||||
", average_build_time_per_tag = '", summary$average_build_time_per_tag,
|
||||
"', package_name = '", package_name,
|
||||
"', platform = '", platform, "'"
|
||||
))
|
||||
} else {
|
||||
dbWriteTable(con, "metadata_summary", summary)
|
||||
}
|
||||
|
||||
# Close the SQLite connection
|
||||
dbDisconnect(con)
|
||||
|
||||
# s3 <- paws.storage::s3(config = list(
|
||||
# endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
# region = "eu-central-003"
|
||||
# ))
|
||||
# invisible(s3$put_object(
|
||||
# Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
|
||||
# Body = "/tmp/metadata.sqlite"
|
||||
# ))
|
||||
}
|
||||
|
||||
query_metadata_summary <- function(package_name = NULL, platform = NULL) {
|
||||
# s3 <- paws.storage::s3(config = list(
|
||||
# endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
# region = "eu-central-003"
|
||||
# ))
|
||||
# db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
|
||||
# writeBin(db_local$Body, "/tmp/metadata.sqlite")
|
||||
|
||||
# Create a new SQLite connection
|
||||
con <- dbcon_mem()
|
||||
|
||||
# If both package_name and platform are NULL, return the full table
|
||||
if (is.null(package_name) && is.null(platform)) {
|
||||
summary <- dbGetQuery(con, "SELECT * FROM metadata_summary")
|
||||
} else {
|
||||
# Query the metadata_summary table for the specified package and platform
|
||||
summary <- dbGetQuery(con, paste0(
|
||||
"SELECT * FROM metadata_summary WHERE package_name = '",
|
||||
package_name, "' AND platform = '", platform, "'"
|
||||
))
|
||||
}
|
||||
|
||||
# Close the SQLite connection
|
||||
dbDisconnect(con)
|
||||
|
||||
return(summary)
|
||||
}
|
||||
|
|
@ -107,7 +107,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
|||
local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y)
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
# NB: this does not always work, which is why we are using the lack of build_duration as a secondary factor in store_metadata()
|
||||
store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE)
|
||||
store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE, error = e)
|
||||
}
|
||||
)
|
||||
}, package_name, tag, future.seed = TRUE)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute
|
||||
store_build_metadata <- function(
|
||||
package_name, tag, platform,
|
||||
error_occurred, error, force = FALSE) {
|
||||
error_occurred, error, force = FALSE, error = NULL) {
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io",
|
||||
port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS")
|
||||
|
|
@ -18,7 +18,7 @@ store_build_metadata <- function(
|
|||
} else if (nrow(existing_entries) >= 1 && force) {
|
||||
cli::cli_alert_info("{.fun store_build_metadata}: Force overwriting build metadata for {.pkg {package_name}} {.field {tag}} because {.code force = TRUE} was set.")
|
||||
|
||||
dbExecute(con, sprintf("UPDATE single_builds SET build_timestamp = '%s', error_occurred = %s WHERE package_name = '%s' and tag = '%s'", format(Sys.time(), "%Y-%m-%d %H:%M:%S"), error_occurred, package_name, tag))
|
||||
dbExecute(con, sprintf("UPDATE single_builds SET build_timestamp = '%s', error_occurred = %s, error = '%s' WHERE package_name = '%s' and tag = '%s'", format(Sys.time(), "%Y-%m-%d %H:%M:%S"), error_occurred, error, package_name, tag))
|
||||
} else if (nrow(existing_entries) == 0) {
|
||||
cli::cli_alert("{.fun store_build_metadata}: Storing build metadata for {.pkg {package_name}} {.field {tag}}.")
|
||||
# Create a data frame with the metadata
|
||||
|
|
@ -27,6 +27,7 @@ store_build_metadata <- function(
|
|||
tag = tag,
|
||||
platform = platform,
|
||||
error_occurred = error_occurred,
|
||||
error = error,
|
||||
build_timestamp = format(Sys.time(), "%Y-%m-%d %H:%M:%S"),
|
||||
build_duration = NA
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue