move sqlite to s3, mature metadata summary functions
This commit is contained in:
parent
63a742d775
commit
d983f391fb
GPG key ID:
3C6318841EF78925
3 changed files with 147 additions and 50 deletions
|
|
@ -3,8 +3,23 @@
|
|||
store_build_metadata <- function(
|
||||
package_name, tag, platform,
|
||||
error_occurred, error) {
|
||||
# s3 <- paws.storage::s3(config = list(
|
||||
# endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
# region = "eu-central-003"
|
||||
# ))
|
||||
# db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
|
||||
# writeBin(db_local$Body, "/tmp/metadata.sqlite")
|
||||
|
||||
# Create a new SQLite connection
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db")
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
|
||||
|
||||
# NB: helper fun which should only be commented in if a new column in the DB is needed!
|
||||
# columns_to_add <- list(
|
||||
# c("average_build_time_per_tag", "REAL")
|
||||
# )
|
||||
# for (column in columns_to_add) {
|
||||
# add_column_if_not_exists("/tmp/metadata.sqlite", "metadata_summary", column[1], column[2])
|
||||
# }
|
||||
|
||||
dbExecute(con, "BEGIN TRANSACTION")
|
||||
|
||||
|
|
@ -14,9 +29,12 @@ store_build_metadata <- function(
|
|||
tag TEXT,
|
||||
platform TEXT,
|
||||
error_occurred BOOLEAN,
|
||||
build_timestamp TEXT
|
||||
build_timestamp TEXT,
|
||||
build_time REAL
|
||||
)")
|
||||
|
||||
# columns added after initial DB creation
|
||||
|
||||
# Create an index on the package_name and tag columns
|
||||
dbExecute(con, "CREATE INDEX IF NOT EXISTS idx_metadata_package_tag ON metadata(package_name, platform)")
|
||||
|
||||
|
|
@ -36,7 +54,8 @@ store_build_metadata <- function(
|
|||
tag = tag,
|
||||
platform = platform,
|
||||
error_occurred = error_occurred,
|
||||
build_timestamp = format(Sys.time(), "%Y-%m-%d")
|
||||
build_timestamp = format(Sys.time(), "%Y-%m-%d"),
|
||||
build_time = NA
|
||||
)
|
||||
# Write the data frame to the SQLite database
|
||||
dbWriteTable(con, "metadata", metadata, append = TRUE)
|
||||
|
|
@ -46,28 +65,39 @@ store_build_metadata <- function(
|
|||
|
||||
# Close the SQLite connection
|
||||
dbDisconnect(con)
|
||||
|
||||
s3 <- paws.storage::s3(config = list(
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003"
|
||||
))
|
||||
s3$put_object(
|
||||
Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
|
||||
Body = "/tmp/metadata.sqlite"
|
||||
)
|
||||
}
|
||||
|
||||
#' @importFrom DBI dbConnect dbGetQuery
|
||||
#' @importFrom RSQLite SQLite
|
||||
#' @importFrom dplyr filter_at vars all_vars
|
||||
query_build_metadata <- function(package_name = NULL, tag = NULL, platform = NULL,
|
||||
error_occurred = NULL, error = NULL, build_timestamp = NULL) {
|
||||
query_build_metadata <- function(table = "metadata", ...) {
|
||||
s3 <- paws.storage::s3(config = list(
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003"
|
||||
))
|
||||
db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
|
||||
writeBin(db_local$Body, "/tmp/metadata.sqlite")
|
||||
|
||||
# Create a new SQLite connection
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db")
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
|
||||
|
||||
# Read the entire table
|
||||
metadata <- dbGetQuery(con, "SELECT * FROM metadata")
|
||||
metadata <- dbGetQuery(con, sprintf("SELECT * FROM %s", table))
|
||||
|
||||
# Close the SQLite connection
|
||||
dbDisconnect(con)
|
||||
|
||||
# Filter the data frame based on the given parameters
|
||||
filter_conditions <- list(
|
||||
package_name = package_name, tag = tag, platform = platform,
|
||||
error_occurred = error_occurred, error = error,
|
||||
build_timestamp = build_timestamp
|
||||
)
|
||||
filter_conditions <- list(...)
|
||||
|
||||
# Remove NULL elements from the list
|
||||
filter_conditions <- filter_conditions[!sapply(filter_conditions, is.null)]
|
||||
|
|
@ -88,13 +118,34 @@ query_build_metadata <- function(package_name = NULL, tag = NULL, platform = NUL
|
|||
#' @importFrom RSQLite SQLite
|
||||
#' @importFrom dplyr group_by summarise filter n
|
||||
build_metadata_summary <- function(package_name, platform) {
|
||||
# Create a new SQLite connection
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db")
|
||||
|
||||
metadata <- dbGetQuery(con, paste0(
|
||||
"SELECT * FROM metadata WHERE package_name = '", package_name,
|
||||
"' AND platform = '", platform, "'"
|
||||
s3 <- paws.storage::s3(config = list(
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003"
|
||||
))
|
||||
db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
|
||||
writeBin(db_local$Body, "/tmp/metadata.sqlite")
|
||||
|
||||
# Create a new SQLite connection
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
|
||||
|
||||
metadata <- dbGetQuery(con, sprintf(
|
||||
"SELECT * FROM metadata WHERE package_name = '%s' AND platform = '%s'", package_name, platform
|
||||
))
|
||||
|
||||
cli::cli_alert_info("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.")
|
||||
|
||||
# add average_build_time_per_tag to metadata summary table
|
||||
|
||||
# If the entry does not exist or is not valid, entry will be NULL or NA
|
||||
# entry_exists <- !is.null(entry$average_build_time_per_tag) && !is.na(entry$average_build_time_per_tag)
|
||||
|
||||
# If the entry does not exist, update the average_build_time_per_tag
|
||||
# if (!entry_exists) {
|
||||
# dbExecute(con, paste0(
|
||||
# "UPDATE metadata_summary SET average_build_time_per_tag = ", average_build_time_per_tag,
|
||||
# " WHERE package_name = '", package_name[[1]], "' AND platform = '", platform, "'"
|
||||
# ))
|
||||
# }
|
||||
|
||||
# Calculate the summary
|
||||
summary <- metadata %>%
|
||||
|
|
@ -103,33 +154,49 @@ build_metadata_summary <- function(package_name, platform) {
|
|||
successful_builds = sum(!error_occurred),
|
||||
unsuccessful_builds = sum(error_occurred),
|
||||
total_builds = n(),
|
||||
percentage_successful_builds = round(successful_builds / total_builds * 100, 2)
|
||||
percentage_successful_builds = round(successful_builds / total_builds * 100, 2),
|
||||
average_build_time_per_tag = round(sum(as.numeric(build_time)) / total_builds, 2)
|
||||
)
|
||||
|
||||
# Check if the metadata_summary table exists
|
||||
if ("metadata_summary" %in% dbListTables(con)) {
|
||||
# Update the row for the specified package
|
||||
dbExecute(con, paste0(
|
||||
"UPDATE metadata_summary SET successful_builds = ", summary$successful_builds,
|
||||
", unsuccessful_builds = ", summary$unsuccessful_builds,
|
||||
", total_builds = ", summary$total_builds,
|
||||
", percentage_successful_builds = ", summary$percentage_successful_builds,
|
||||
" WHERE package_name = '", package_name,
|
||||
"' AND platform = '", platform, "'"
|
||||
", average_build_time_per_tag = '", summary$average_build_time_per_tag,
|
||||
"', package_name = '", package_name,
|
||||
"', platform = '", platform, "'"
|
||||
))
|
||||
} else {
|
||||
# Write the summary to a new table in the SQLite database
|
||||
dbWriteTable(con, "metadata_summary", summary)
|
||||
}
|
||||
|
||||
# Close the SQLite connection
|
||||
dbDisconnect(con)
|
||||
|
||||
s3 <- paws.storage::s3(config = list(
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003"
|
||||
))
|
||||
invisible(s3$put_object(
|
||||
Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
|
||||
Body = "/tmp/metadata.sqlite"
|
||||
))
|
||||
}
|
||||
|
||||
#' @importFrom RSQLite SQLite
|
||||
query_metadata_summary <- function(package_name = NULL, platform = NULL) {
|
||||
s3 <- paws.storage::s3(config = list(
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003"
|
||||
))
|
||||
db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
|
||||
writeBin(db_local$Body, "/tmp/metadata.sqlite")
|
||||
|
||||
# Create a new SQLite connection
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db")
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
|
||||
|
||||
# If both package_name and platform are NULL, return the full table
|
||||
if (is.null(package_name) && is.null(platform)) {
|
||||
|
|
|
|||
|
|
@ -73,11 +73,12 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
|||
tryCatch(
|
||||
{
|
||||
p()
|
||||
build_single_tag(x, y, dir_out_bin, local_clone_dir)
|
||||
build_single_tag(x, y, dir_out_bin, local_clone_dir, platform = platform)
|
||||
store_build_metadata(x, y, platform, FALSE)
|
||||
},
|
||||
error = function(e) {
|
||||
message("Error in processing package ", x, " with tag ", y, ": ")
|
||||
local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y)
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
store_build_metadata(x, y, platform, TRUE)
|
||||
}
|
||||
|
|
@ -90,29 +91,6 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
|||
total_build_time <- round(Sys.time() - t1, 2)
|
||||
cli::cli_alert_info("Execution time for {.pkg {package_name[[1]]}} ({length(tag)} tags): {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.")
|
||||
|
||||
# add average_build_time_per_tag to metadata summary table
|
||||
average_build_time_per_tag <- round(total_build_time / length(tag), 2)
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "metadata.db")
|
||||
# Check if the entry exists in the metadata_summary table
|
||||
# Check if a valid entry exists in the metadata_summary table
|
||||
entry <- dbGetQuery(con, paste0(
|
||||
"SELECT average_build_time_per_tag FROM metadata_summary WHERE package_name = '", package_name[[1]],
|
||||
"' AND platform = '", platform, "'"
|
||||
))
|
||||
# If the entry does not exist or is not valid, entry will be NULL or NA
|
||||
entry_exists <- !is.null(entry$average_build_time_per_tag) && !is.na(entry$average_build_time_per_tag)
|
||||
|
||||
# If the entry does not exist, update the average_build_time_per_tag
|
||||
if (!entry_exists) {
|
||||
dbExecute(con, paste0(
|
||||
"UPDATE metadata_summary SET average_build_time_per_tag = ", average_build_time_per_tag,
|
||||
" WHERE package_name = '", package_name[[1]], "' AND platform = '", platform, "'"
|
||||
))
|
||||
}
|
||||
|
||||
cli::cli_alert_info("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.")
|
||||
build_metadata_summary(package_name[[1]], platform)
|
||||
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
||||
|
|
@ -144,9 +122,11 @@ install_package_system_dependencies <- function(package_name,
|
|||
|
||||
#' @importFrom cli cli_alert_info
|
||||
#' @importFrom pkgbuild build
|
||||
build_single_tag <- function(package_name, tag = NULL,
|
||||
dir_out_bin,
|
||||
local_clone_dir) {
|
||||
build_single_tag <- function(
|
||||
package_name, tag = NULL,
|
||||
platform,
|
||||
dir_out_bin,
|
||||
local_clone_dir) {
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 1. Cloning package {.pkg {package_name}} with tag {.strong {tag}}.")
|
||||
|
||||
local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag)
|
||||
|
|
@ -159,6 +139,7 @@ build_single_tag <- function(package_name, tag = NULL,
|
|||
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 2. Building package {.pkg {package_name}} with tag {.strong {tag}}.")
|
||||
|
||||
t1 <- Sys.time()
|
||||
pkgbuild::build(
|
||||
path = sprintf("%s", local_clone_dir_single),
|
||||
binary = TRUE, vignettes = FALSE,
|
||||
|
|
@ -179,4 +160,28 @@ build_single_tag <- function(package_name, tag = NULL,
|
|||
|
||||
cli::cli_alert_info("{.fun build_single_tag}: 4. Removing {.path {local_clone_dir_single}}.")
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
|
||||
total_build_time <- round(Sys.time() - t1, 2)
|
||||
|
||||
s3 <- paws.storage::s3(config = list(
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003"
|
||||
))
|
||||
db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
|
||||
writeBin(db_local$Body, "/tmp/metadata.sqlite")
|
||||
|
||||
con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
|
||||
dbExecute(con, "BEGIN TRANSACTION")
|
||||
# Create an index on the package_name and tag columns
|
||||
dbExecute(con, sprintf(
|
||||
"UPDATE metadata set build_time = '%s' where package_name = '%s' and platform = '%s' and tag = '%s'",
|
||||
total_build_time, package_name, platform, tag
|
||||
))
|
||||
dbExecute(con, "COMMIT")
|
||||
dbDisconnect(con)
|
||||
|
||||
invisible(s3$put_object(
|
||||
Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
|
||||
Body = "/tmp/metadata.sqlite"
|
||||
))
|
||||
}
|
||||
|
|
|
|||
22
R/helpers.R
22
R/helpers.R
|
|
@ -40,3 +40,25 @@ set_bin_path <- function(r_version_minor, build_for_minor, local_build_root, cod
|
|||
|
||||
return(path)
|
||||
}
|
||||
|
||||
add_column_if_not_exists <- function(db_name, table_name, column_name, column_type) {
|
||||
# Connect to the database
|
||||
conn <- dbConnect(SQLite(), dbname = db_name)
|
||||
|
||||
# Get the list of columns in the table
|
||||
columns <- dbGetQuery(conn, paste0("PRAGMA table_info(", table_name, ");"))
|
||||
|
||||
# Check if the column already exists
|
||||
column_exists <- any(columns$name == column_name)
|
||||
|
||||
# Add the column if it doesn't exist
|
||||
if (!column_exists) {
|
||||
dbExecute(conn, paste0("ALTER TABLE ", table_name, " ADD COLUMN ", column_name, " ", column_type, ";"))
|
||||
message(paste("Column '", column_name, "' added to table '", table_name, "'.", sep = ""))
|
||||
} else {
|
||||
message(paste("Column '", column_name, "' already exists in table '", table_name, "'.", sep = ""))
|
||||
}
|
||||
|
||||
# Disconnect from the database
|
||||
dbDisconnect(conn)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue