diff --git a/.gitignore b/.gitignore index 5b6a065..402911f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .Rhistory .RData .Ruserdata +.Renviron \ No newline at end of file diff --git a/DESCRIPTION b/DESCRIPTION index 35ddde3..8995337 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -9,11 +9,13 @@ License: `use_mit_license()`, `use_gpl3_license()` or friends to pick a license Encoding: UTF-8 Roxygen: list(markdown = TRUE) -RoxygenNote: 7.3.1 +RoxygenNote: 7.3.2 Imports: cli, DBI, dplyr, + future, + future.apply, gert, lubridate, magrittr, diff --git a/R/build-metadata.R b/R/build-metadata.R index bae4aa9..7b47ac1 100644 --- a/R/build-metadata.R +++ b/R/build-metadata.R @@ -3,8 +3,23 @@ store_build_metadata <- function( package_name, tag, platform, error_occurred, error) { + # s3 <- paws.storage::s3(config = list( + # endpoint = "https://s3.eu-central-003.backblazeb2.com", + # region = "eu-central-003" + # )) + # db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite") + # writeBin(db_local$Body, "/tmp/metadata.sqlite") + # Create a new SQLite connection - con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db") + con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite") + + # NB: helper fun which should only be commented in if a new column in the DB is needed! + # columns_to_add <- list( + # c("average_build_time_per_tag", "REAL") + # ) + # for (column in columns_to_add) { + # add_column_if_not_exists("/tmp/metadata.sqlite", "metadata_summary", column[1], column[2]) + # } dbExecute(con, "BEGIN TRANSACTION") @@ -14,9 +29,12 @@ store_build_metadata <- function( tag TEXT, platform TEXT, error_occurred BOOLEAN, - build_timestamp TEXT + build_timestamp TEXT, + build_time REAL )") + # columns added after initial DB creation + # Create an index on the package_name and tag columns dbExecute(con, "CREATE INDEX IF NOT EXISTS idx_metadata_package_tag ON metadata(package_name, platform)") @@ -36,7 +54,8 @@ store_build_metadata <- function( tag = tag, platform = platform, error_occurred = error_occurred, - build_timestamp = format(Sys.time(), "%Y-%m-%d") + build_timestamp = format(Sys.time(), "%Y-%m-%d"), + build_time = NA ) # Write the data frame to the SQLite database dbWriteTable(con, "metadata", metadata, append = TRUE) @@ -46,28 +65,39 @@ store_build_metadata <- function( # Close the SQLite connection dbDisconnect(con) + + s3 <- paws.storage::s3(config = list( + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region = "eu-central-003" + )) + s3$put_object( + Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite", + Body = "/tmp/metadata.sqlite" + ) } #' @importFrom DBI dbConnect dbGetQuery #' @importFrom RSQLite SQLite #' @importFrom dplyr filter_at vars all_vars -query_build_metadata <- function(package_name = NULL, tag = NULL, platform = NULL, - error_occurred = NULL, error = NULL, build_timestamp = NULL) { +query_build_metadata <- function(table = "metadata", ...) { + s3 <- paws.storage::s3(config = list( + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region = "eu-central-003" + )) + db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite") + writeBin(db_local$Body, "/tmp/metadata.sqlite") + # Create a new SQLite connection - con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db") + con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite") # Read the entire table - metadata <- dbGetQuery(con, "SELECT * FROM metadata") + metadata <- dbGetQuery(con, sprintf("SELECT * FROM %s", table)) # Close the SQLite connection dbDisconnect(con) # Filter the data frame based on the given parameters - filter_conditions <- list( - package_name = package_name, tag = tag, platform = platform, - error_occurred = error_occurred, error = error, - build_timestamp = build_timestamp - ) + filter_conditions <- list(...) # Remove NULL elements from the list filter_conditions <- filter_conditions[!sapply(filter_conditions, is.null)] @@ -88,13 +118,34 @@ query_build_metadata <- function(package_name = NULL, tag = NULL, platform = NUL #' @importFrom RSQLite SQLite #' @importFrom dplyr group_by summarise filter n build_metadata_summary <- function(package_name, platform) { - # Create a new SQLite connection - con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db") - - metadata <- dbGetQuery(con, paste0( - "SELECT * FROM metadata WHERE package_name = '", package_name, - "' AND platform = '", platform, "'" + s3 <- paws.storage::s3(config = list( + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region = "eu-central-003" )) + db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite") + writeBin(db_local$Body, "/tmp/metadata.sqlite") + + # Create a new SQLite connection + con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite") + + metadata <- dbGetQuery(con, sprintf( + "SELECT * FROM metadata WHERE package_name = '%s' AND platform = '%s'", package_name, platform + )) + + cli::cli_alert_info("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.") + + # add average_build_time_per_tag to metadata summary table + + # If the entry does not exist or is not valid, entry will be NULL or NA + # entry_exists <- !is.null(entry$average_build_time_per_tag) && !is.na(entry$average_build_time_per_tag) + + # If the entry does not exist, update the average_build_time_per_tag + # if (!entry_exists) { + # dbExecute(con, paste0( + # "UPDATE metadata_summary SET average_build_time_per_tag = ", average_build_time_per_tag, + # " WHERE package_name = '", package_name[[1]], "' AND platform = '", platform, "'" + # )) + # } # Calculate the summary summary <- metadata %>% @@ -103,33 +154,49 @@ build_metadata_summary <- function(package_name, platform) { successful_builds = sum(!error_occurred), unsuccessful_builds = sum(error_occurred), total_builds = n(), - percentage_successful_builds = round(successful_builds / total_builds * 100, 2) + percentage_successful_builds = round(successful_builds / total_builds * 100, 2), + average_build_time_per_tag = round(sum(as.numeric(build_time)) / total_builds, 2) ) # Check if the metadata_summary table exists if ("metadata_summary" %in% dbListTables(con)) { - # Update the row for the specified package dbExecute(con, paste0( "UPDATE metadata_summary SET successful_builds = ", summary$successful_builds, ", unsuccessful_builds = ", summary$unsuccessful_builds, ", total_builds = ", summary$total_builds, ", percentage_successful_builds = ", summary$percentage_successful_builds, - " WHERE package_name = '", package_name, - "' AND platform = '", platform, "'" + ", average_build_time_per_tag = '", summary$average_build_time_per_tag, + "', package_name = '", package_name, + "', platform = '", platform, "'" )) } else { - # Write the summary to a new table in the SQLite database dbWriteTable(con, "metadata_summary", summary) } # Close the SQLite connection dbDisconnect(con) + + s3 <- paws.storage::s3(config = list( + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region = "eu-central-003" + )) + invisible(s3$put_object( + Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite", + Body = "/tmp/metadata.sqlite" + )) } #' @importFrom RSQLite SQLite query_metadata_summary <- function(package_name = NULL, platform = NULL) { + s3 <- paws.storage::s3(config = list( + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region = "eu-central-003" + )) + db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite") + writeBin(db_local$Body, "/tmp/metadata.sqlite") + # Create a new SQLite connection - con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db") + con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite") # If both package_name and platform are NULL, return the full table if (is.null(package_name) && is.null(platform)) { diff --git a/R/build_binaries.R b/R/build_binaries.R index da59b11..2cf7b59 100644 --- a/R/build_binaries.R +++ b/R/build_binaries.R @@ -73,11 +73,12 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, tryCatch( { p() - build_single_tag(x, y, dir_out_bin, local_clone_dir) + build_single_tag(x, y, dir_out_bin, local_clone_dir, platform = platform) store_build_metadata(x, y, platform, FALSE) }, error = function(e) { message("Error in processing package ", x, " with tag ", y, ": ") + local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y) unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) store_build_metadata(x, y, platform, TRUE) } @@ -90,29 +91,6 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, total_build_time <- round(Sys.time() - t1, 2) cli::cli_alert_info("Execution time for {.pkg {package_name[[1]]}} ({length(tag)} tags): {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.") - # add average_build_time_per_tag to metadata summary table - average_build_time_per_tag <- round(total_build_time / length(tag), 2) - con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db") - # Check if the entry exists in the metadata_summary table - # Check if a valid entry exists in the metadata_summary table - entry <- dbGetQuery(con, paste0( - "SELECT average_build_time_per_tag FROM metadata_summary WHERE package_name = '", package_name[[1]], - "' AND platform = '", platform, "'" - )) - # If the entry does not exist or is not valid, entry will be NULL or NA - entry_exists <- !is.null(entry$average_build_time_per_tag) && !is.na(entry$average_build_time_per_tag) - - # If the entry does not exist, update the average_build_time_per_tag - if (!entry_exists) { - dbExecute(con, paste0( - "UPDATE metadata_summary SET average_build_time_per_tag = ", average_build_time_per_tag, - " WHERE package_name = '", package_name[[1]], "' AND platform = '", platform, "'" - )) - } - - cli::cli_alert_info("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.") - build_metadata_summary(package_name[[1]], platform) - return(invisible(TRUE)) } @@ -144,9 +122,11 @@ install_package_system_dependencies <- function(package_name, #' @importFrom cli cli_alert_info #' @importFrom pkgbuild build -build_single_tag <- function(package_name, tag = NULL, - dir_out_bin, - local_clone_dir) { +build_single_tag <- function( + package_name, tag = NULL, + platform, + dir_out_bin, + local_clone_dir) { cli::cli_alert_info("{.fun build_single_tag}: 1. Cloning package {.pkg {package_name}} with tag {.strong {tag}}.") local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag) @@ -159,6 +139,7 @@ build_single_tag <- function(package_name, tag = NULL, cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.strong {tag}}.") + t1 <- Sys.time() pkgbuild::build( path = sprintf("%s", local_clone_dir_single), binary = TRUE, vignettes = FALSE, @@ -179,4 +160,28 @@ build_single_tag <- function(package_name, tag = NULL, cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.") unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) + + total_build_time <- round(Sys.time() - t1, 2) + + s3 <- paws.storage::s3(config = list( + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region = "eu-central-003" + )) + db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite") + writeBin(db_local$Body, "/tmp/metadata.sqlite") + + con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite") + dbExecute(con, "BEGIN TRANSACTION") + # Create an index on the package_name and tag columns + dbExecute(con, sprintf( + "UPDATE metadata set build_time = '%s' where package_name = '%s' and platform = '%s' and tag = '%s'", + total_build_time, package_name, platform, tag + )) + dbExecute(con, "COMMIT") + dbDisconnect(con) + + invisible(s3$put_object( + Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite", + Body = "/tmp/metadata.sqlite" + )) } diff --git a/R/helpers.R b/R/helpers.R index 04104e9..aa3a962 100644 --- a/R/helpers.R +++ b/R/helpers.R @@ -40,3 +40,25 @@ set_bin_path <- function(r_version_minor, build_for_minor, local_build_root, cod return(path) } + +add_column_if_not_exists <- function(db_name, table_name, column_name, column_type) { + # Connect to the database + conn <- dbConnect(SQLite(), dbname = db_name) + + # Get the list of columns in the table + columns <- dbGetQuery(conn, paste0("PRAGMA table_info(", table_name, ");")) + + # Check if the column already exists + column_exists <- any(columns$name == column_name) + + # Add the column if it doesn't exist + if (!column_exists) { + dbExecute(conn, paste0("ALTER TABLE ", table_name, " ADD COLUMN ", column_name, " ", column_type, ";")) + message(paste("Column '", column_name, "' added to table '", table_name, "'.", sep = "")) + } else { + message(paste("Column '", column_name, "' already exists in table '", table_name, "'.", sep = "")) + } + + # Disconnect from the database + dbDisconnect(conn) +}