move sqlite to s3, mature metadata summary functions

.gitignore

@@ -2,3 +2,4 @@
 .Rhistory
 .RData
 .Ruserdata
+.Renviron

DESCRIPTION

@@ -9,11 +9,13 @@ License: `use_mit_license()`, `use_gpl3_license()` or friends to pick a
     license
 Encoding: UTF-8
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.3.1
+RoxygenNote: 7.3.2
 Imports:
     cli,
     DBI,
     dplyr,
+    future,
+    future.apply,
     gert,
     lubridate,
     magrittr,

R/build-metadata.R

@@ -3,8 +3,23 @@
 store_build_metadata <- function(
     package_name, tag, platform,
     error_occurred, error) {
+  # s3 <- paws.storage::s3(config = list(
+  #   endpoint = "https://s3.eu-central-003.backblazeb2.com",
+  #   region = "eu-central-003"
+  # ))
+  # db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
+  # writeBin(db_local$Body, "/tmp/metadata.sqlite")
+
   # Create a new SQLite connection
-  con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db")
+  con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
+
+  # NB: helper fun which should only be commented in if a new column in the DB is needed!
+  # columns_to_add <- list(
+  #   c("average_build_time_per_tag", "REAL")
+  # )
+  # for (column in columns_to_add) {
+  #   add_column_if_not_exists("/tmp/metadata.sqlite", "metadata_summary", column[1], column[2])
+  # }
 
   dbExecute(con, "BEGIN TRANSACTION")
 
@@ -14,9 +29,12 @@ store_build_metadata <- function(
     tag TEXT,
     platform TEXT,
     error_occurred BOOLEAN,
-    build_timestamp TEXT
+    build_timestamp TEXT,
+    build_time REAL
   )")
 
+  # columns added after initial DB creation
+
   # Create an index on the package_name and tag columns
   dbExecute(con, "CREATE INDEX IF NOT EXISTS idx_metadata_package_tag ON metadata(package_name, platform)")
 
@@ -36,7 +54,8 @@ store_build_metadata <- function(
       tag = tag,
       platform = platform,
       error_occurred = error_occurred,
-      build_timestamp = format(Sys.time(), "%Y-%m-%d")
+      build_timestamp = format(Sys.time(), "%Y-%m-%d"),
+      build_time = NA
     )
     # Write the data frame to the SQLite database
     dbWriteTable(con, "metadata", metadata, append = TRUE)
@@ -46,28 +65,39 @@ store_build_metadata <- function(
 
   # Close the SQLite connection
   dbDisconnect(con)
+
+  s3 <- paws.storage::s3(config = list(
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    region = "eu-central-003"
+  ))
+  s3$put_object(
+    Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
+    Body = "/tmp/metadata.sqlite"
+  )
 }
 
 #' @importFrom DBI dbConnect dbGetQuery
 #' @importFrom RSQLite SQLite
 #' @importFrom dplyr filter_at vars all_vars
-query_build_metadata <- function(package_name = NULL, tag = NULL, platform = NULL,
+query_build_metadata <- function(table = "metadata", ...) {
-                                 error_occurred = NULL, error = NULL, build_timestamp = NULL) {
+  s3 <- paws.storage::s3(config = list(
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    region = "eu-central-003"
+  ))
+  db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
+  writeBin(db_local$Body, "/tmp/metadata.sqlite")
+
   # Create a new SQLite connection
-  con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db")
+  con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
 
   # Read the entire table
-  metadata <- dbGetQuery(con, "SELECT * FROM metadata")
+  metadata <- dbGetQuery(con, sprintf("SELECT * FROM %s", table))
 
   # Close the SQLite connection
   dbDisconnect(con)
 
   # Filter the data frame based on the given parameters
-  filter_conditions <- list(
+  filter_conditions <- list(...)
-    package_name = package_name, tag = tag, platform = platform,
-    error_occurred = error_occurred, error = error,
-    build_timestamp = build_timestamp
-  )
 
   # Remove NULL elements from the list
   filter_conditions <- filter_conditions[!sapply(filter_conditions, is.null)]
@@ -88,13 +118,34 @@ query_build_metadata <- function(package_name = NULL, tag = NULL, platform = NUL
 #' @importFrom RSQLite SQLite
 #' @importFrom dplyr group_by summarise filter n
 build_metadata_summary <- function(package_name, platform) {
-  # Create a new SQLite connection
+  s3 <- paws.storage::s3(config = list(
-  con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db")
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
-
+    region = "eu-central-003"
-  metadata <- dbGetQuery(con, paste0(
-    "SELECT * FROM metadata WHERE package_name = '", package_name,
-    "' AND platform = '", platform, "'"
   ))
+  db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
+  writeBin(db_local$Body, "/tmp/metadata.sqlite")
+
+  # Create a new SQLite connection
+  con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
+
+  metadata <- dbGetQuery(con, sprintf(
+    "SELECT * FROM metadata WHERE package_name = '%s' AND platform = '%s'", package_name, platform
+  ))
+
+  cli::cli_alert_info("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.")
+
+  # add average_build_time_per_tag to metadata summary table
+
+  # If the entry does not exist or is not valid, entry will be NULL or NA
+  # entry_exists <- !is.null(entry$average_build_time_per_tag) && !is.na(entry$average_build_time_per_tag)
+
+  # If the entry does not exist, update the average_build_time_per_tag
+  # if (!entry_exists) {
+  #   dbExecute(con, paste0(
+  #     "UPDATE metadata_summary SET average_build_time_per_tag = ", average_build_time_per_tag,
+  #     " WHERE package_name = '", package_name[[1]], "' AND platform = '", platform, "'"
+  #   ))
+  # }
 
   # Calculate the summary
   summary <- metadata %>%
@@ -103,33 +154,49 @@ build_metadata_summary <- function(package_name, platform) {
       successful_builds = sum(!error_occurred),
       unsuccessful_builds = sum(error_occurred),
       total_builds = n(),
-      percentage_successful_builds = round(successful_builds / total_builds * 100, 2)
+      percentage_successful_builds = round(successful_builds / total_builds * 100, 2),
+      average_build_time_per_tag = round(sum(as.numeric(build_time)) / total_builds, 2)
     )
 
   # Check if the metadata_summary table exists
   if ("metadata_summary" %in% dbListTables(con)) {
-    # Update the row for the specified package
     dbExecute(con, paste0(
       "UPDATE metadata_summary SET successful_builds = ", summary$successful_builds,
       ", unsuccessful_builds = ", summary$unsuccessful_builds,
       ", total_builds = ", summary$total_builds,
       ", percentage_successful_builds = ", summary$percentage_successful_builds,
-      " WHERE package_name = '", package_name,
+      ", average_build_time_per_tag = '", summary$average_build_time_per_tag,
-      "' AND platform = '", platform, "'"
+      "', package_name = '", package_name,
+      "', platform = '", platform, "'"
     ))
   } else {
-    # Write the summary to a new table in the SQLite database
     dbWriteTable(con, "metadata_summary", summary)
   }
 
   # Close the SQLite connection
   dbDisconnect(con)
+
+  s3 <- paws.storage::s3(config = list(
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    region = "eu-central-003"
+  ))
+  invisible(s3$put_object(
+    Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
+    Body = "/tmp/metadata.sqlite"
+  ))
 }
 
 #' @importFrom RSQLite SQLite
 query_metadata_summary <- function(package_name = NULL, platform = NULL) {
+  s3 <- paws.storage::s3(config = list(
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    region = "eu-central-003"
+  ))
+  db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
+  writeBin(db_local$Body, "/tmp/metadata.sqlite")
+
   # Create a new SQLite connection
-  con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db")
+  con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
 
   # If both package_name and platform are NULL, return the full table
   if (is.null(package_name) && is.null(platform)) {

R/build_binaries.R

@@ -73,11 +73,12 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
       tryCatch(
         {
           p()
-          build_single_tag(x, y, dir_out_bin, local_clone_dir)
+          build_single_tag(x, y, dir_out_bin, local_clone_dir, platform = platform)
           store_build_metadata(x, y, platform, FALSE)
         },
         error = function(e) {
           message("Error in processing package ", x, " with tag ", y, ": ")
+          local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y)
           unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
           store_build_metadata(x, y, platform, TRUE)
         }
@@ -90,29 +91,6 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
   total_build_time <- round(Sys.time() - t1, 2)
   cli::cli_alert_info("Execution time for {.pkg {package_name[[1]]}} ({length(tag)} tags): {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.")
 
-  # add average_build_time_per_tag to metadata summary table
-  average_build_time_per_tag <- round(total_build_time / length(tag), 2)
-  con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db")
-  # Check if the entry exists in the metadata_summary table
-  # Check if a valid entry exists in the metadata_summary table
-  entry <- dbGetQuery(con, paste0(
-    "SELECT average_build_time_per_tag FROM metadata_summary WHERE package_name = '", package_name[[1]],
-    "' AND platform = '", platform, "'"
-  ))
-  # If the entry does not exist or is not valid, entry will be NULL or NA
-  entry_exists <- !is.null(entry$average_build_time_per_tag) && !is.na(entry$average_build_time_per_tag)
-
-  # If the entry does not exist, update the average_build_time_per_tag
-  if (!entry_exists) {
-    dbExecute(con, paste0(
-      "UPDATE metadata_summary SET average_build_time_per_tag = ", average_build_time_per_tag,
-      " WHERE package_name = '", package_name[[1]], "' AND platform = '", platform, "'"
-    ))
-  }
-
-  cli::cli_alert_info("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.")
-  build_metadata_summary(package_name[[1]], platform)
-
   return(invisible(TRUE))
 }
 
@@ -144,7 +122,9 @@ install_package_system_dependencies <- function(package_name,
 
 #' @importFrom cli cli_alert_info
 #' @importFrom pkgbuild build
-build_single_tag <- function(package_name, tag = NULL,
+build_single_tag <- function(
+    package_name, tag = NULL,
+    platform,
     dir_out_bin,
     local_clone_dir) {
   cli::cli_alert_info("{.fun build_single_tag}: 1. Cloning package {.pkg {package_name}} with tag {.strong {tag}}.")
@@ -159,6 +139,7 @@ build_single_tag <- function(package_name, tag = NULL,
 
   cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.strong {tag}}.")
 
+  t1 <- Sys.time()
   pkgbuild::build(
     path = sprintf("%s", local_clone_dir_single),
     binary = TRUE, vignettes = FALSE,
@@ -179,4 +160,28 @@ build_single_tag <- function(package_name, tag = NULL,
 
   cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.")
   unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
+
+  total_build_time <- round(Sys.time() - t1, 2)
+
+  s3 <- paws.storage::s3(config = list(
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    region = "eu-central-003"
+  ))
+  db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
+  writeBin(db_local$Body, "/tmp/metadata.sqlite")
+
+  con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
+  dbExecute(con, "BEGIN TRANSACTION")
+  # Create an index on the package_name and tag columns
+  dbExecute(con, sprintf(
+    "UPDATE metadata set build_time = '%s' where package_name = '%s' and platform = '%s' and tag = '%s'",
+    total_build_time, package_name, platform, tag
+  ))
+  dbExecute(con, "COMMIT")
+  dbDisconnect(con)
+
+  invisible(s3$put_object(
+    Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
+    Body = "/tmp/metadata.sqlite"
+  ))
 }

R/helpers.R

@@ -40,3 +40,25 @@ set_bin_path <- function(r_version_minor, build_for_minor, local_build_root, cod
 
   return(path)
 }
+
+add_column_if_not_exists <- function(db_name, table_name, column_name, column_type) {
+  # Connect to the database
+  conn <- dbConnect(SQLite(), dbname = db_name)
+
+  # Get the list of columns in the table
+  columns <- dbGetQuery(conn, paste0("PRAGMA table_info(", table_name, ");"))
+
+  # Check if the column already exists
+  column_exists <- any(columns$name == column_name)
+
+  # Add the column if it doesn't exist
+  if (!column_exists) {
+    dbExecute(conn, paste0("ALTER TABLE ", table_name, " ADD COLUMN ", column_name, " ", column_type, ";"))
+    message(paste("Column '", column_name, "' added to table '", table_name, "'.", sep = ""))
+  } else {
+    message(paste("Column '", column_name, "' already exists in table '", table_name, "'.", sep = ""))
+  }
+
+  # Disconnect from the database
+  dbDisconnect(conn)
+}