devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 10 Pull requests Releases Activity

update db connection objects and use postgres (fix #2, fix #3)

Browse source
This commit is contained in:
Patrick Schratz 2024-07-23 21:46:21 +02:00
commit 71449937da
Signed by: pat-s
GPG key ID: 3C6318841EF78925
1 changed files with 53 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

112
R/build-metadata.R
View file

@ -11,7 +11,10 @@ store_build_metadata <- function(
# writeBin(db_local$Body, "/tmp/metadata.sqlite")
# Create a new SQLite connection
con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
con <- DBI::dbConnect(RPostgres::Postgres(),
dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io",
port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS")
)
# NB: helper fun which should only be commented in if a new column in the DB is needed!
# columns_to_add <- list(
@ -21,26 +24,26 @@ store_build_metadata <- function(
# add_column_if_not_exists("/tmp/metadata.sqlite", "metadata_summary", column[1], column[2])
# }
dbExecute(con, "BEGIN TRANSACTION")
# dbExecute(con, "BEGIN TRANSACTION")
# Create the "metadata" table if it does not exist
dbExecute(con, "CREATE TABLE IF NOT EXISTS metadata (
package_name TEXT,
tag TEXT,
platform TEXT,
error_occurred BOOLEAN,
build_timestamp TEXT,
build_time REAL
)")
# dbExecute(con, "CREATE TABLE IF NOT EXISTS single_builds (
# package_name TEXT,
# tag TEXT,
# platform TEXT,
# error_occurred BOOLEAN,
# build_timestamp TEXT,
# build_duration REAL
# )")
# columns added after initial DB creation
# Create an index on the package_name and tag columns
dbExecute(con, "CREATE INDEX IF NOT EXISTS idx_metadata_package_tag ON metadata(package_name, platform)")
dbExecute(con, "CREATE INDEX IF NOT EXISTS idx_metadata_package_tag ON single_builds(package_name, platform)")
# Check if an entry with the same package_name and tag already exists
existing_entries <- dbGetQuery(con, paste0(
"SELECT * FROM metadata WHERE package_name = '",
"SELECT * FROM single_builds WHERE package_name = '",
package_name, "' AND tag = '", tag, "'"
))
@ -54,41 +57,40 @@ store_build_metadata <- function(
tag = tag,
platform = platform,
error_occurred = error_occurred,
build_timestamp = format(Sys.time(), "%Y-%m-%d"),
build_time = NA
build_timestamp = format(Sys.time(), "%Y-%m-%d %H:%M:%S"),
build_duration = NA
)
# Write the data frame to the SQLite database
dbWriteTable(con, "metadata", metadata, append = TRUE)
dbWriteTable(con, "single_builds", metadata, append = TRUE)
}
dbExecute(con, "COMMIT")
# dbExecute(con, "COMMIT")
# Close the SQLite connection
dbDisconnect(con)
s3 <- paws.storage::s3(config = list(
endpoint = "https://s3.eu-central-003.backblazeb2.com",
region = "eu-central-003"
))
s3$put_object(
Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
Body = "/tmp/metadata.sqlite"
)
# s3 <- paws.storage::s3(config = list(
# endpoint = "https://s3.eu-central-003.backblazeb2.com",
# region = "eu-central-003"
# ))
# s3$put_object(
# Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
# Body = "/tmp/metadata.sqlite"
# )
}
#' @importFrom DBI dbConnect dbGetQuery
#' @importFrom RSQLite SQLite
#' @importFrom dplyr filter_at vars all_vars
query_build_metadata <- function(table = "metadata", ...) {
s3 <- paws.storage::s3(config = list(
endpoint = "https://s3.eu-central-003.backblazeb2.com",
region = "eu-central-003"
))
db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
writeBin(db_local$Body, "/tmp/metadata.sqlite")
query_build_metadata <- function(table = "single_builds", ...) {
# s3 <- paws.storage::s3(config = list(
# endpoint = "https://s3.eu-central-003.backblazeb2.com",
# region = "eu-central-003"
# ))
# db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
# writeBin(db_local$Body, "/tmp/metadata.sqlite")
# Create a new SQLite connection
con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
con <- dbcon_mem()
# Read the entire table
metadata <- dbGetQuery(con, sprintf("SELECT * FROM %s", table))
@ -118,18 +120,10 @@ query_build_metadata <- function(table = "metadata", ...) {
#' @importFrom RSQLite SQLite
#' @importFrom dplyr group_by summarise filter n
build_metadata_summary <- function(package_name, platform) {
s3 <- paws.storage::s3(config = list(
endpoint = "https://s3.eu-central-003.backblazeb2.com",
region = "eu-central-003"
))
db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
writeBin(db_local$Body, "/tmp/metadata.sqlite")
# Create a new SQLite connection
con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
con <- dbcon_mem()
metadata <- dbGetQuery(con, sprintf(
"SELECT * FROM metadata WHERE package_name = '%s' AND platform = '%s'", package_name, platform
"SELECT * FROM single_builds WHERE package_name = '%s' AND platform = '%s'", package_name, platform
))
cli::cli_alert_info("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.")
@ -155,7 +149,7 @@ build_metadata_summary <- function(package_name, platform) {
unsuccessful_builds = sum(error_occurred),
total_builds = n(),
percentage_successful_builds = round(successful_builds / total_builds * 100, 2),
average_build_time_per_tag = round(sum(as.numeric(build_time)) / total_builds, 2)
average_build_time_per_tag = round(sum(as.numeric(build_duration)) / total_builds, 2)
)
# Check if the metadata_summary table exists
@ -176,27 +170,27 @@ build_metadata_summary <- function(package_name, platform) {
# Close the SQLite connection
dbDisconnect(con)
s3 <- paws.storage::s3(config = list(
endpoint = "https://s3.eu-central-003.backblazeb2.com",
region = "eu-central-003"
))
invisible(s3$put_object(
Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
Body = "/tmp/metadata.sqlite"
))
# s3 <- paws.storage::s3(config = list(
# endpoint = "https://s3.eu-central-003.backblazeb2.com",
# region = "eu-central-003"
# ))
# invisible(s3$put_object(
# Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite",
# Body = "/tmp/metadata.sqlite"
# ))
}
#' @importFrom RSQLite SQLite
query_metadata_summary <- function(package_name = NULL, platform = NULL) {
s3 <- paws.storage::s3(config = list(
endpoint = "https://s3.eu-central-003.backblazeb2.com",
region = "eu-central-003"
))
db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
writeBin(db_local$Body, "/tmp/metadata.sqlite")
# s3 <- paws.storage::s3(config = list(
# endpoint = "https://s3.eu-central-003.backblazeb2.com",
# region = "eu-central-003"
# ))
# db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
# writeBin(db_local$Body, "/tmp/metadata.sqlite")
# Create a new SQLite connection
con <- dbConnect(RSQLite::SQLite(), dbname = "/tmp/metadata.sqlite")
con <- dbcon_mem()
# If both package_name and platform are NULL, return the full table
if (is.null(package_name) && is.null(platform)) {