library(bincraft, quietly = TRUE) suppressPackageStartupMessages(library(dplyr)) library(DBI, quietly = TRUE) library(purrr, quietly = TRUE) library(progressr, quietly = TRUE) suppressPackageStartupMessages(library(data.table)) # Sys.setenv("OS" = "alpine") # Sys.setenv("OS_VERSION" = "3.22") # Sys.setenv("ARCH" = "arm64") arch = Sys.getenv("ARCH") if (Sys.getenv("OS") == "alpine") { # target: alpine-322 platform = paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-") } con <- DBI::dbConnect( RPostgres::Postgres(), dbname = "build_metadata", host = "r-binaries.devxy.io", port = 15432, user = "rpkgs", password = Sys.getenv("PGPASS"), sslmode = "require" ) cran_archive = tools::CRAN_archive_db() cran_release = tools::CRAN_package_db() # Subset cran_archive to only those packages cran_archive_in_release <- cran_archive[names(cran_archive) %in% cran_release$Package] archive_versions <- rbindlist( lapply(names(cran_archive), function(pkg) { df <- as.data.table(cran_archive_in_release[[pkg]]) file_names <- rownames(cran_archive_in_release[[pkg]]) # Get row names from the original data.frame! versions <- sub(".*_(.*)\\.tar\\.gz$", "\\1", file_names) data.table( Package = pkg, Version = versions, mtime = df$mtime ) }), fill = TRUE ) # Select the 9 most recent by mtime for each package archive_versions <- archive_versions[ order(Package, -as.numeric(mtime)) ][, head(.SD, 9), by = Package ][, .(Package, Version) ] # Now get release versions (assuming cran_release has Package and Version columns) release_versions <- data.table( Package = cran_release$Package, Version = as.character(cran_release$Version) ) pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE)) setorder(pkgs_to_build, Package, Version) ### Get all packages in S3 s3fs::s3_file_system( aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), endpoint = "https://s3.eu-central-003.backblazeb2.com", region_name = "eu-central-003", refresh = TRUE ) s3_pkgs = s3fs::s3_dir_ls( sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, sub("-", "", platform)), recurse = TRUE ) file_names <- basename(s3_pkgs) matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) parts <- regmatches(file_names, matches) parts <- parts[sapply(parts, length) == 3] s3_dt <- data.table( Package = sapply(parts, `[`, 2), Version = sapply(parts, `[`, 3) ) ### Get all packages with build errors sql_query <- paste0( # nolint "SELECT error_occurred FROM ", "single_builds", " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4" ) # Function to query for a single package-version query_error <- function(pkg, ver) { purrr::insistently( ~ DBI::dbGetQuery( con, sql_query, params = list(pkg, ver, platform, arch) ), rate = purrr::rate_backoff( pause_base = 1L, pause_cap = 60L, pause_min = 1L, max_times = 10L, jitter = FALSE ), quiet = FALSE )() } # Fetch all relevant columns from the database errored_pkgs <- DBI::dbGetQuery( con, sprintf("SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", platform, arch) ) errored_pkgs <- as.data.table(errored_pkgs) setkey(pkgs_to_build, Package, Version) setnames(errored_pkgs, c("Package","Version")) setkey(errored_pkgs, Package, Version) ### Final subsetting pkgs_no_error <- pkgs_to_build[!errored_pkgs] # only return package names pkgs <- unique(as.character(pkgs_no_error[!s3_dt, Package]))