add error to metadata table

NAMESPACE

@@ -4,6 +4,7 @@ export(build_binary_package)
 export(build_single_tag)
 export(dbcon)
 export(install_package_system_dependencies)
+export(list_metadata_tables)
 export(set_bin_path)
 export(set_codename)
 export(upload_single_binary_to_s3)
@@ -17,13 +18,11 @@ importFrom(DBI,dbWriteTable)
 importFrom(RPostgres,Postgres)
 importFrom(cli,cli_alert)
 importFrom(cranlike,add_PACKAGES)
-importFrom(dplyr,all_vars)
 importFrom(dplyr,filter)
-importFrom(dplyr,filter_at)
 importFrom(dplyr,group_by)
 importFrom(dplyr,n)
 importFrom(dplyr,summarise)
-importFrom(dplyr,vars)
+importFrom(dplyr,tbl)
 importFrom(future,plan)
 importFrom(future.apply,future_mapply)
 importFrom(gert,git_config_set)

R/build-metadata.R

@@ -1,179 +0,0 @@
-#' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute
-store_build_metadata <- function(
-    package_name, tag, platform,
-    error_occurred, error, force = FALSE) {
-  # s3 <- paws.storage::s3(config = list(
-  #   endpoint = "https://s3.eu-central-003.backblazeb2.com",
-  #   region = "eu-central-003"
-  # ))
-  # db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
-  # writeBin(db_local$Body, "/tmp/metadata.sqlite")
-
-  # Create a new SQLite connection
-  con <- DBI::dbConnect(RPostgres::Postgres(),
-    dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io",
-    port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS")
-  )
-
-  # Create an index on the package_name and tag columns
-  dbExecute(con, "CREATE INDEX IF NOT EXISTS idx_metadata_package_tag ON single_builds(package_name, platform)")
-
-  # Check if an entry with the same package_name and tag already exists
-  existing_entries <- dbGetQuery(con, paste0(
-    "SELECT * FROM single_builds WHERE package_name = '",
-    package_name, "' AND tag = '", tag, "'"
-  ))
-
-  if (nrow(existing_entries) >= 1 && !force) {
-    cli::cli_alert("{.fun store_build_metadata}: Build metadata for {.field {.pkg package_name}} {.field {tag}} already exists.")
-  } else if (nrow(existing_entries) >= 1 && force) {
-    cli::cli_alert_info("{.fun store_build_metadata}: Force overwriting build metadata for {.pkg {package_name}} {.field {tag}} because {.code force = TRUE} was set.")
-
-    dbExecute(con, sprintf("UPDATE single_builds SET build_timestamp = '%s' WHERE package_name = '%s' and tag = '%s'", format(Sys.time(), "%Y-%m-%d %H:%M:%S"), package_name, tag))
-  } else if (nrow(existing_entries) == 0) {
-    cli::cli_alert("{.fun store_build_metadata}: Storing build metadata for {.pkg {package_name}} {.field {tag}}.")
-    # Create a data frame with the metadata
-    metadata <- data.frame(
-      package_name = package_name,
-      tag = tag,
-      platform = platform,
-      error_occurred = error_occurred,
-      build_timestamp = format(Sys.time(), "%Y-%m-%d %H:%M:%S"),
-      build_duration = NA
-    )
-    # Write the data frame to the SQLite database
-    dbWriteTable(con, "single_builds", metadata, append = TRUE)
-  }
-
-  # Close the SQLite connection
-  dbDisconnect(con)
-
-  return(invisible(TRUE))
-}
-
-#' @importFrom DBI dbConnect dbGetQuery
-#' @importFrom dplyr filter_at vars all_vars
-query_build_metadata <- function(table = "single_builds", ...) {
-  # s3 <- paws.storage::s3(config = list(
-  #   endpoint = "https://s3.eu-central-003.backblazeb2.com",
-  #   region = "eu-central-003"
-  # ))
-  # db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
-  # writeBin(db_local$Body, "/tmp/metadata.sqlite")
-
-  con <- dbcon_mem()
-
-  # Read the entire table
-  metadata <- dbGetQuery(con, sprintf("SELECT * FROM %s", table))
-
-  # Close the SQLite connection
-  dbDisconnect(con)
-
-  # Filter the data frame based on the given parameters
-  filter_conditions <- list(...)
-
-  # Remove NULL elements from the list
-  filter_conditions <- filter_conditions[!sapply(filter_conditions, is.null)]
-
-  # Apply the filter conditions if any exist
-  if (length(filter_conditions) > 0) {
-    metadata <- dplyr::filter_at(
-      metadata, dplyr::vars(names(filter_conditions)),
-      dplyr::all_vars(. == filter_conditions[[.]])
-    )
-  }
-
-  return(metadata)
-}
-
-
-#' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute dbListTables
-#' @importFrom dplyr group_by summarise filter n
-build_metadata_summary <- function(package_name, platform) {
-  con <- dbcon_mem()
-
-  metadata <- dbGetQuery(con, sprintf(
-    "SELECT * FROM single_builds WHERE package_name = '%s' AND platform = '%s'", package_name, platform
-  ))
-
-  cli::cli_alert("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.")
-
-  # add average_build_time_per_tag to metadata summary table
-
-  # If the entry does not exist or is not valid, entry will be NULL or NA
-  # entry_exists <- !is.null(entry$average_build_time_per_tag) && !is.na(entry$average_build_time_per_tag)
-
-  # If the entry does not exist, update the average_build_time_per_tag
-  # if (!entry_exists) {
-  #   dbExecute(con, paste0(
-  #     "UPDATE metadata_summary SET average_build_time_per_tag = ", average_build_time_per_tag,
-  #     " WHERE package_name = '", package_name[[1]], "' AND platform = '", platform, "'"
-  #   ))
-  # }
-
-  # Calculate the summary
-  summary <- metadata %>%
-    group_by(package_name, platform) %>%
-    summarise(
-      successful_builds = sum(!error_occurred),
-      unsuccessful_builds = sum(error_occurred),
-      total_builds = n(),
-      percentage_successful_builds = round(successful_builds / total_builds * 100, 2),
-      average_build_time_per_tag = round(sum(as.numeric(build_duration)) / total_builds, 2)
-    )
-
-  # Check if the metadata_summary table exists
-  if ("metadata_summary" %in% dbListTables(con)) {
-    dbExecute(con, paste0(
-      "UPDATE metadata_summary SET successful_builds = ", summary$successful_builds,
-      ", unsuccessful_builds = ", summary$unsuccessful_builds,
-      ", total_builds = ", summary$total_builds,
-      ", percentage_successful_builds = ", summary$percentage_successful_builds,
-      ", average_build_time_per_tag = '", summary$average_build_time_per_tag,
-      "', package_name = '", package_name,
-      "', platform = '", platform, "'"
-    ))
-  } else {
-    dbWriteTable(con, "metadata_summary", summary)
-  }
-
-  # Close the SQLite connection
-  dbDisconnect(con)
-
-  # s3 <- paws.storage::s3(config = list(
-  #   endpoint = "https://s3.eu-central-003.backblazeb2.com",
-  #   region = "eu-central-003"
-  # ))
-  # invisible(s3$put_object(
-  #   Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
-  #   Body = "/tmp/metadata.sqlite"
-  # ))
-}
-
-query_metadata_summary <- function(package_name = NULL, platform = NULL) {
-  # s3 <- paws.storage::s3(config = list(
-  #   endpoint = "https://s3.eu-central-003.backblazeb2.com",
-  #   region = "eu-central-003"
-  # ))
-  # db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
-  # writeBin(db_local$Body, "/tmp/metadata.sqlite")
-
-  # Create a new SQLite connection
-  con <- dbcon_mem()
-
-  # If both package_name and platform are NULL, return the full table
-  if (is.null(package_name) && is.null(platform)) {
-    summary <- dbGetQuery(con, "SELECT * FROM metadata_summary")
-  } else {
-    # Query the metadata_summary table for the specified package and platform
-    summary <- dbGetQuery(con, paste0(
-      "SELECT * FROM metadata_summary WHERE package_name = '",
-      package_name, "' AND platform = '", platform, "'"
-    ))
-  }
-
-  # Close the SQLite connection
-  dbDisconnect(con)
-
-  return(summary)
-}

R/build_binaries.R

@@ -107,7 +107,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
           local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y)
           unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
           # NB: this does not always work, which is why we are using the lack of build_duration as a secondary factor in store_metadata()
-          store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE)
+          store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE, error = e)
         }
       )
     }, package_name, tag, future.seed = TRUE)

R/metadata-build.R

@@ -1,7 +1,7 @@
 #' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute
 store_build_metadata <- function(
     package_name, tag, platform,
-    error_occurred, error, force = FALSE) {
+    error_occurred, error, force = FALSE, error = NULL) {
   con <- DBI::dbConnect(RPostgres::Postgres(),
     dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io",
     port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS")
@@ -18,7 +18,7 @@ store_build_metadata <- function(
   } else if (nrow(existing_entries) >= 1 && force) {
     cli::cli_alert_info("{.fun store_build_metadata}: Force overwriting build metadata for {.pkg {package_name}} {.field {tag}} because {.code force = TRUE} was set.")
 
-    dbExecute(con, sprintf("UPDATE single_builds SET build_timestamp = '%s', error_occurred = %s WHERE package_name = '%s' and tag = '%s'", format(Sys.time(), "%Y-%m-%d %H:%M:%S"), error_occurred, package_name, tag))
+    dbExecute(con, sprintf("UPDATE single_builds SET build_timestamp = '%s', error_occurred = %s, error = '%s' WHERE package_name = '%s' and tag = '%s'", format(Sys.time(), "%Y-%m-%d %H:%M:%S"), error_occurred, error, package_name, tag))
   } else if (nrow(existing_entries) == 0) {
     cli::cli_alert("{.fun store_build_metadata}: Storing build metadata for {.pkg {package_name}} {.field {tag}}.")
     # Create a data frame with the metadata
@@ -27,6 +27,7 @@ store_build_metadata <- function(
       tag = tag,
       platform = platform,
       error_occurred = error_occurred,
+      error = error,
       build_timestamp = format(Sys.time(), "%Y-%m-%d %H:%M:%S"),
       build_duration = NA
     )