From 251bba8f23381cb30ed3fd198910713eaed4654c Mon Sep 17 00:00:00 2001 From: pat-s Date: Sat, 13 Jun 2026 21:23:37 +0200 Subject: [PATCH] style: apply air and prettier formatting repo-wide --- .crow/archive-missed-packages.yaml | 1 - .crow/build-all-versions-amd64.yaml | 4 +- .crow/build-all-versions-arm64.yaml | 2 +- local/archive-missed-pkgs.R | 26 ++- local/check-NA.R | 9 +- local/debug-packages-writing.R | 70 ++++--- local/detect-duplicates.R | 7 +- local/excluded-packages.json | 150 +++++++------- local/fetch-rebuild-packages-from-issue.R | 52 +++-- local/label-removed-cran-packages.R | 14 +- local/last-processed-by-platform.R | 2 +- local/missing-cran-packages-db.R | 48 +++-- local/missing-cran-packages.R | 48 +++-- local/missing-packages-in-index.R | 117 ++++++++--- local/packages-without-any-binary.R | 25 ++- local/weekly-missing-binaries-audit.R | 229 ++++++++++++++++------ renovate.json | 8 +- 17 files changed, 554 insertions(+), 258 deletions(-) diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index bb9ecb4..88c199e 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -6,7 +6,6 @@ when: - event: cron cron: archive-missed-packages - matrix: include: - CODENAME: jammy diff --git a/.crow/build-all-versions-amd64.yaml b/.crow/build-all-versions-amd64.yaml index 796c812..30def98 100644 --- a/.crow/build-all-versions-amd64.yaml +++ b/.crow/build-all-versions-amd64.yaml @@ -39,7 +39,7 @@ depends_on: steps: - name: 'Build binaries' - image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: RED_HAT_DEV_PW: @@ -57,7 +57,7 @@ steps: # normal env vars GIT_USER: pat-s # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: "" + R_PKG_CACHE_DIR: '' R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache NCPUS: 2 diff --git a/.crow/build-all-versions-arm64.yaml b/.crow/build-all-versions-arm64.yaml index 37992df..c3a2977 100644 --- a/.crow/build-all-versions-arm64.yaml +++ b/.crow/build-all-versions-arm64.yaml @@ -54,7 +54,7 @@ depends_on: steps: - name: 'Build binaries' - image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: RED_HAT_DEV_PW: diff --git a/local/archive-missed-pkgs.R b/local/archive-missed-pkgs.R index 57e92d7..f817c3e 100644 --- a/local/archive-missed-pkgs.R +++ b/local/archive-missed-pkgs.R @@ -11,16 +11,34 @@ s3fs::s3_file_system( arch = "arm64" os = "alpine321" -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", arch, os)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", + arch, + os +)) length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1]))) -non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) x[1])[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))]) +non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) { + x[1] +})[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))]) # RInno is not built because it only exists for Windows non_archived = setdiff(non_archived, "RInno") # future::plan("sequential", workers = 8) future::plan("sequential", workers = 1) # future.apply::future_lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch)) -lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch, s3_region = "hel1", s3_endpoint = "https://hel1.your-objectstorage.com", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S", s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S")))) +lapply(non_archived, function(x) { + archive_package( + x, + codename = os, + arch = arch, + s3_region = "hel1", + s3_endpoint = "https://hel1.your-objectstorage.com", + s3_bucket = "devxy-r-package-binaries-hel1", + s3_access_key_id = Sys.getenv( + "HETZNER_S3_ACCESS_KEY_K3S", + s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S") + ) + ) +}) # archive_package() - # grep("duckdb_", files, value = T) diff --git a/local/check-NA.R b/local/check-NA.R index 7405db4..95c158e 100644 --- a/local/check-NA.R +++ b/local/check-NA.R @@ -1,4 +1,3 @@ - ### Scope: Check whether there are any incomplete entries in PACKAGES.rds. These will result in NA when calling available.packages() foo = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib") sum((is.na(foo[, "Version"]))) @@ -17,7 +16,9 @@ foo = available.packages("https://cran.devxy.io/amd64/rhel9/latest/src/contrib") sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) -foo = available.packages("https://cran.devxy.io/amd64/alpine320/latest/src/contrib") +foo = available.packages( + "https://cran.devxy.io/amd64/alpine320/latest/src/contrib" +) sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) @@ -40,6 +41,8 @@ foo = available.packages("https://cran.devxy.io/arm64/rhel9/latest/src/contrib") sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) -foo = available.packages("https://cran.devxy.io/arm64/alpine320/latest/src/contrib") +foo = available.packages( + "https://cran.devxy.io/arm64/alpine320/latest/src/contrib" +) sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) diff --git a/local/debug-packages-writing.R b/local/debug-packages-writing.R index 5a0fd24..533ae2b 100644 --- a/local/debug-packages-writing.R +++ b/local/debug-packages-writing.R @@ -2,42 +2,58 @@ library(future) future::plan(multisession) future::plan(sequential) time = Sys.time() -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive"), recurse = T) +files <- s3fs::s3_dir_ls( + sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive" + ), + recurse = T +) Sys.time() - time - archive <- llply(dirs, function(dir) { - files <- list.files(dir, recursive = FALSE, full.names = TRUE, pattern = "*.tar.gz") + files <- list.files( + dir, + recursive = FALSE, + full.names = TRUE, + pattern = "*.tar.gz" + ) if (length(files) == 0) { print(paste0("Error: Empty directory: ", dir)) return(NULL) } info <- file.info(files) - tryCatch({ - rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) - }, error = function(e) { - print(paste0("Error: Exception catched for Archived directory: ", dir)) - print(e) - return(NULL) - }) + tryCatch( + { + rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) + }, + error = function(e) { + print(paste0("Error: Exception catched for Archived directory: ", dir)) + print(e) + return(NULL) + } + ) info }) -tryCatch({ - rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) -}, error = function(e) { - print(paste0("Error: Exception catched for Archived directory: ", dir)) - print(e) - return(NULL) -}) +tryCatch( + { + rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) + }, + error = function(e) { + print(paste0("Error: Exception catched for Archived directory: ", dir)) + print(e) + return(NULL) + } +) - - -curl::curl_download("https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db") +curl::curl_download( + "https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", + "PACKAGES.db" +) con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db") df = DBI::dbReadTable(con, "packages") @@ -50,6 +66,14 @@ which(grepl("digest", df$Package)) system("cat /tmp/PACKAGES | grep '^Package: digest' | wc -l") -curl::curl_fetch_memory("https://cloud.r-project.org/src/contrib/Meta/archive.rds") -remotes = readRDS(url("https://cloud.r-project.org/src/contrib/Meta/archive.rds", "rb")) -packages = readRDS(url("https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds", "rb")) +curl::curl_fetch_memory( + "https://cloud.r-project.org/src/contrib/Meta/archive.rds" +) +remotes = readRDS(url( + "https://cloud.r-project.org/src/contrib/Meta/archive.rds", + "rb" +)) +packages = readRDS(url( + "https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds", + "rb" +)) diff --git a/local/detect-duplicates.R b/local/detect-duplicates.R index 076499d..f7e2c18 100644 --- a/local/detect-duplicates.R +++ b/local/detect-duplicates.R @@ -37,10 +37,13 @@ future.apply::future_lapply( files <- s3fs::s3_dir_ls( sprintf( "devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", - arch, codename + arch, + codename ) ) - if (length(files) == 0) return(NULL) + if (length(files) == 0) { + return(NULL) + } files_b <- basename(files) pkg_names <- sapply(strsplit(files_b, "_"), `[`, 1) dupes <- unique(pkg_names[duplicated(pkg_names)]) diff --git a/local/excluded-packages.json b/local/excluded-packages.json index dccfc0b..6489998 100644 --- a/local/excluded-packages.json +++ b/local/excluded-packages.json @@ -1,77 +1,77 @@ [ - {"package": "RInno", "reason": "windows-only"}, - {"package": "KeyboardSimulator", "reason": "windows-only"}, - {"package": "R2PPT", "reason": "windows-only"}, - {"package": "RWinEdt", "reason": "windows-only"}, - {"package": "blatr", "reason": "windows-only"}, - {"package": "excel.link", "reason": "windows-only"}, - {"package": "spectrino", "reason": "windows-only"}, - {"package": "taskscheduleR", "reason": "windows-only"}, - {"package": "MDSGUI", "reason": "windows-only"}, - {"package": "BiplotGUI", "reason": "windows-only"}, - {"package": "R2wd", "reason": "windows-only"}, - {"package": "rFUSION", "reason": "windows-only"}, - {"package": "MediaNews", "reason": "windows-only"}, - {"package": "doBy", "reason": "hang"}, - {"package": "IDPmisc", "reason": "hang"}, - {"package": "frailtypack", "reason": "hang"}, - {"package": "afex", "reason": "hang"}, - {"package": "FrF2", "reason": "hang"}, - {"package": "DoE.base", "reason": "hang"}, - {"package": "agricolae", "reason": "hang"}, - {"package": "doFuture", "reason": "hang"}, - {"package": "fscaret", "reason": "hang"}, - {"package": "PHYLOGR", "reason": "hang"}, - {"package": "seewave", "reason": "hang"}, - {"package": "pls", "reason": "hang"}, - {"package": "relaimpo", "reason": "hang"}, - {"package": "geepack", "reason": "hang"}, - {"package": "gggenes", "reason": "hang"}, - {"package": "NPCirc", "reason": "hang"}, - {"package": "repmis", "reason": "hang"}, - {"package": "PNDSIBGE", "reason": "hang"}, - {"package": "lidR", "reason": "hang"}, - {"package": "poismf", "reason": "hang"}, - {"package": "neonstore", "reason": "hang"}, - {"package": "MachineShop", "reason": "hang"}, - {"package": "mvst", "reason": "hang"}, - {"package": "MacBehaviour", "reason": "hang"}, - {"package": "mcmcderive", "reason": "hang"}, - {"package": "RGIFT", "reason": "hang"}, - {"package": "KnowBR", "reason": "hang"}, - {"package": "netmeta", "reason": "hang"}, - {"package": "spdep", "reason": "hang"}, - {"package": "Rfast", "reason": "hang"}, - {"package": "compareGroups", "reason": "hang"}, - {"package": "ff", "reason": "hang"}, - {"package": "GsymPoint", "reason": "hang"}, - {"package": "RcppDynProg", "reason": "hang"}, - {"package": "comtradr", "reason": "hang"}, - {"package": "FD", "reason": "hang"}, - {"package": "PearsonDS", "reason": "hang"}, - {"package": "DCluster", "reason": "hang"}, - {"package": "gRc", "reason": "hang"}, - {"package": "mixlm", "reason": "hang"}, - {"package": "geospt", "reason": "hang"}, - {"package": "fdth", "reason": "hang"}, - {"package": "ffmanova", "reason": "hang"}, - {"package": "fiery", "reason": "hang"}, - {"package": "ffscrapr", "reason": "hang"}, - {"package": "cold", "reason": "hang"}, - {"package": "RcmdrPlugin.DoE", "reason": "hang"}, - {"package": "RcmdrPlugin.NMBU", "reason": "hang"}, - {"package": "RcmdrPlugin.RiskDemo", "reason": "hang"}, - {"package": "RcmdrPlugin.ROC", "reason": "hang"}, - {"package": "RcmdrPlugin.TeachStat", "reason": "hang"}, - {"package": "RcmdrPlugin.TeachingDemos", "reason": "hang"}, - {"package": "RcmdrPlugin.UCA", "reason": "hang"}, - {"package": "RcmdrPlugin.WorldFlora", "reason": "hang"}, - {"package": "RcmdrPlugin.aRnova", "reason": "hang"}, - {"package": "RcmdrPlugin.depthTools", "reason": "hang"}, - {"package": "RcmdrPlugin.orloca", "reason": "hang"}, - {"package": "RcmdrPlugin.sos", "reason": "hang"}, - {"package": "RcmdrPlugin.survival", "reason": "hang"}, - {"package": "RcmdrPlugin.temis", "reason": "hang"}, - {"package": "GWlasso", "reason": "hang"}, - {"package": "GWmodelVis", "reason": "hang"} + { "package": "RInno", "reason": "windows-only" }, + { "package": "KeyboardSimulator", "reason": "windows-only" }, + { "package": "R2PPT", "reason": "windows-only" }, + { "package": "RWinEdt", "reason": "windows-only" }, + { "package": "blatr", "reason": "windows-only" }, + { "package": "excel.link", "reason": "windows-only" }, + { "package": "spectrino", "reason": "windows-only" }, + { "package": "taskscheduleR", "reason": "windows-only" }, + { "package": "MDSGUI", "reason": "windows-only" }, + { "package": "BiplotGUI", "reason": "windows-only" }, + { "package": "R2wd", "reason": "windows-only" }, + { "package": "rFUSION", "reason": "windows-only" }, + { "package": "MediaNews", "reason": "windows-only" }, + { "package": "doBy", "reason": "hang" }, + { "package": "IDPmisc", "reason": "hang" }, + { "package": "frailtypack", "reason": "hang" }, + { "package": "afex", "reason": "hang" }, + { "package": "FrF2", "reason": "hang" }, + { "package": "DoE.base", "reason": "hang" }, + { "package": "agricolae", "reason": "hang" }, + { "package": "doFuture", "reason": "hang" }, + { "package": "fscaret", "reason": "hang" }, + { "package": "PHYLOGR", "reason": "hang" }, + { "package": "seewave", "reason": "hang" }, + { "package": "pls", "reason": "hang" }, + { "package": "relaimpo", "reason": "hang" }, + { "package": "geepack", "reason": "hang" }, + { "package": "gggenes", "reason": "hang" }, + { "package": "NPCirc", "reason": "hang" }, + { "package": "repmis", "reason": "hang" }, + { "package": "PNDSIBGE", "reason": "hang" }, + { "package": "lidR", "reason": "hang" }, + { "package": "poismf", "reason": "hang" }, + { "package": "neonstore", "reason": "hang" }, + { "package": "MachineShop", "reason": "hang" }, + { "package": "mvst", "reason": "hang" }, + { "package": "MacBehaviour", "reason": "hang" }, + { "package": "mcmcderive", "reason": "hang" }, + { "package": "RGIFT", "reason": "hang" }, + { "package": "KnowBR", "reason": "hang" }, + { "package": "netmeta", "reason": "hang" }, + { "package": "spdep", "reason": "hang" }, + { "package": "Rfast", "reason": "hang" }, + { "package": "compareGroups", "reason": "hang" }, + { "package": "ff", "reason": "hang" }, + { "package": "GsymPoint", "reason": "hang" }, + { "package": "RcppDynProg", "reason": "hang" }, + { "package": "comtradr", "reason": "hang" }, + { "package": "FD", "reason": "hang" }, + { "package": "PearsonDS", "reason": "hang" }, + { "package": "DCluster", "reason": "hang" }, + { "package": "gRc", "reason": "hang" }, + { "package": "mixlm", "reason": "hang" }, + { "package": "geospt", "reason": "hang" }, + { "package": "fdth", "reason": "hang" }, + { "package": "ffmanova", "reason": "hang" }, + { "package": "fiery", "reason": "hang" }, + { "package": "ffscrapr", "reason": "hang" }, + { "package": "cold", "reason": "hang" }, + { "package": "RcmdrPlugin.DoE", "reason": "hang" }, + { "package": "RcmdrPlugin.NMBU", "reason": "hang" }, + { "package": "RcmdrPlugin.RiskDemo", "reason": "hang" }, + { "package": "RcmdrPlugin.ROC", "reason": "hang" }, + { "package": "RcmdrPlugin.TeachStat", "reason": "hang" }, + { "package": "RcmdrPlugin.TeachingDemos", "reason": "hang" }, + { "package": "RcmdrPlugin.UCA", "reason": "hang" }, + { "package": "RcmdrPlugin.WorldFlora", "reason": "hang" }, + { "package": "RcmdrPlugin.aRnova", "reason": "hang" }, + { "package": "RcmdrPlugin.depthTools", "reason": "hang" }, + { "package": "RcmdrPlugin.orloca", "reason": "hang" }, + { "package": "RcmdrPlugin.sos", "reason": "hang" }, + { "package": "RcmdrPlugin.survival", "reason": "hang" }, + { "package": "RcmdrPlugin.temis", "reason": "hang" }, + { "package": "GWlasso", "reason": "hang" }, + { "package": "GWmodelVis", "reason": "hang" } ] diff --git a/local/fetch-rebuild-packages-from-issue.R b/local/fetch-rebuild-packages-from-issue.R index d944dd0..26e544d 100644 --- a/local/fetch-rebuild-packages-from-issue.R +++ b/local/fetch-rebuild-packages-from-issue.R @@ -1,15 +1,21 @@ library(httr2, quietly = TRUE) forgejo_base <- "https://git.devxy.io/api/v1" -repo <- "devxy/build-cran-binaries" -platform <- Sys.getenv("PLATFORM") -arch <- Sys.getenv("ARCH") -token <- Sys.getenv("FORGEJO_TOKEN") -output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") +repo <- "devxy/build-cran-binaries" +platform <- Sys.getenv("PLATFORM") +arch <- Sys.getenv("ARCH") +token <- Sys.getenv("FORGEJO_TOKEN") +output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") -if (nchar(platform) == 0) stop("PLATFORM env var is not set") -if (nchar(arch) == 0) stop("ARCH env var is not set") -if (nchar(token) == 0) stop("FORGEJO_TOKEN env var is not set") +if (nchar(platform) == 0) { + stop("PLATFORM env var is not set") +} +if (nchar(arch) == 0) { + stop("ARCH env var is not set") +} +if (nchar(token) == 0) { + stop("FORGEJO_TOKEN env var is not set") +} os_family <- if (grepl("^ubuntu", platform)) { "Ubuntu" @@ -21,12 +27,16 @@ os_family <- if (grepl("^ubuntu", platform)) { platform } -issue_title <- sprintf("Missing package binaries for latest version (%s)", os_family) +issue_title <- sprintf( + "Missing package binaries for latest version (%s)", + os_family +) cat(sprintf("Searching for issue: %s\n", issue_title)) search_url <- sprintf( "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", - forgejo_base, repo, + forgejo_base, + repo, utils::URLencode(issue_title, reserved = TRUE) ) search_resp <- request(search_url) |> @@ -34,7 +44,9 @@ search_resp <- request(search_url) |> req_perform() issues <- resp_body_json(search_resp, simplifyVector = FALSE) -match_idx <- which(vapply(issues, function(x) x$title, character(1)) == issue_title) +match_idx <- which( + vapply(issues, function(x) x$title, character(1)) == issue_title +) if (length(match_idx) == 0) { cat("No matching issue found - nothing to rebuild\n") @@ -55,7 +67,10 @@ lines <- strsplit(body, "\n", fixed = TRUE)[[1]] plat_header <- sprintf("## %s", platform) plat_idx <- which(lines == plat_header) if (length(plat_idx) == 0) { - cat(sprintf("No section found for platform %s - nothing to rebuild\n", platform)) + cat(sprintf( + "No section found for platform %s - nothing to rebuild\n", + platform + )) writeLines(character(0), output_file) q("no") } @@ -64,7 +79,11 @@ if (length(plat_idx) == 0) { arch_pattern <- sprintf("^### %s ", arch) arch_idx <- which(grepl(arch_pattern, lines) & seq_along(lines) > plat_idx[1]) if (length(arch_idx) == 0) { - cat(sprintf("No section found for arch %s under %s - nothing to rebuild\n", arch, platform)) + cat(sprintf( + "No section found for arch %s under %s - nothing to rebuild\n", + arch, + platform + )) writeLines(character(0), output_file) q("no") } @@ -82,6 +101,11 @@ pkg_lines <- section_lines[grepl("^- ", section_lines)] pkgs <- sub("^- ([^ ]+) \\(.*\\)$", "\\1", pkg_lines) pkgs <- pkgs[nchar(pkgs) > 0 & pkgs != "_None_"] -cat(sprintf("Found %d rebuildable packages for %s/%s\n", length(pkgs), platform, arch)) +cat(sprintf( + "Found %d rebuildable packages for %s/%s\n", + length(pkgs), + platform, + arch +)) writeLines(pkgs, output_file) cat(sprintf("Wrote package list to %s\n", output_file)) diff --git a/local/label-removed-cran-packages.R b/local/label-removed-cran-packages.R index 92cd940..12dff9d 100644 --- a/local/label-removed-cran-packages.R +++ b/local/label-removed-cran-packages.R @@ -6,9 +6,13 @@ archived <- quickcode::archivedPkg() |> pull(name) # Query all distinct pkgs in the DB -con <- DBI::dbConnect(RPostgres::Postgres(), - dbname = "build_metadata", host = "r-binaries.devxy.io", - port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "r_binaries", + password = Sys.getenv("PGPASS"), sslmode = "require" ) @@ -22,7 +26,9 @@ to_process <- pkgs_db[pkgs_db %in% archived] # for all matches, set 'removed = TRUE' sapply(to_process, function(.x) { - DBI::dbExecute(con, "UPDATE single_builds SET removed = 'TRUE' where name = $1", + DBI::dbExecute( + con, + "UPDATE single_builds SET removed = 'TRUE' where name = $1", params = list(.x) ) }) diff --git a/local/last-processed-by-platform.R b/local/last-processed-by-platform.R index f913610..398b6c1 100644 --- a/local/last-processed-by-platform.R +++ b/local/last-processed-by-platform.R @@ -7,4 +7,4 @@ query_metadata_table() |> group_by(platform, arch) |> arrange(desc(timestamp)) |> select(name, timestamp) |> - filter(row_number()==1) + filter(row_number() == 1) diff --git a/local/missing-cran-packages-db.R b/local/missing-cran-packages-db.R index 56b3d94..882bca4 100644 --- a/local/missing-cran-packages-db.R +++ b/local/missing-cran-packages-db.R @@ -3,22 +3,34 @@ library(bincraft) library(dplyr) library(DBI) -con <- DBI::dbConnect(RPostgres::Postgres(), - dbname = "build_metadata", host = "r-binaries.devxy.io", - port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "r_binaries", + password = Sys.getenv("PGPASS"), sslmode = "require" ) `%nin%` <- Negate(`%in%`) -new_packages <- get_new_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name -removed_pkgs <- get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name +new_packages <- get_new_cran_packages(lubridate::interval( + lubridate::today(), + lubridate::today() - 2 +))$name +removed_pkgs <- get_removed_cran_packages(lubridate::interval( + lubridate::today(), + lubridate::today() - 2 +))$name # filter windows packages # filter new packages from the last X days -cran_pkgs <- unique(tools::CRAN_package_db() |> - filter(`OS_type` != "windows" | is.na(`OS_type`)) |> - filter(`Package` %nin% new_packages) |> - pull(Package)) +cran_pkgs <- unique( + tools::CRAN_package_db() |> + filter(`OS_type` != "windows" | is.na(`OS_type`)) |> + filter(`Package` %nin% new_packages) |> + pull(Package) +) arch <- "arm64" platform <- "redhat-8" @@ -28,13 +40,24 @@ platform <- "alpine-321" platform <- "ubuntu-2204" platform <- "ubuntu-2404" -data <- dbGetQuery(con, "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", params = list(platform, arch)) -removed_pkgs = get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name +data <- dbGetQuery( + con, + "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", + params = list(platform, arch) +) +removed_pkgs = get_removed_cran_packages(lubridate::interval( + lubridate::today(), + lubridate::today() - 2 +))$name pkgs_db <- unique(data$name) pkgs_db = setdiff(pkgs_db, removed_pkgs) pkgs <- setdiff(cran_pkgs, pkgs_db) -formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)]))))) +formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)])))) +) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") cat(formatted_pkgs_one_line, "\n", sep = "") @@ -44,7 +67,6 @@ clipr::write_clip(formatted_pkgs_one_line) # sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs))) - s3fs::s3_file_system( aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"), diff --git a/local/missing-cran-packages.R b/local/missing-cran-packages.R index ab71b5b..ca5cad8 100644 --- a/local/missing-cran-packages.R +++ b/local/missing-cran-packages.R @@ -10,19 +10,39 @@ s3fs::s3_file_system( ) s3fs::s3_dir_ls("s3://devxy-r-package-binaries") -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib" +)) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib" +)) -files_b=basename(files) +files_b = basename(files) files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) cran_pkgs <- tools::CRAN_package_db() |> @@ -30,7 +50,11 @@ cran_pkgs <- tools::CRAN_package_db() |> pull(Package) pkgs = setdiff(cran_pkgs, files_b_split) -formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900]))))) +formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs[1:900])))) +) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") cat(formatted_pkgs_one_line, "\n", sep = "") diff --git a/local/missing-packages-in-index.R b/local/missing-packages-in-index.R index 810c89d..6453888 100644 --- a/local/missing-packages-in-index.R +++ b/local/missing-packages-in-index.R @@ -7,42 +7,109 @@ s3fs::s3_file_system( region_name = "hel1", ) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib" +)) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib" +)) -files_b=basename(files) +files_b = basename(files) files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) -files_b_split = setdiff(files_b_split, c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive")) +files_b_split = setdiff( + files_b_split, + c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive") +) # windows-only -files_b_split = setdiff(files_b_split, c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive", "RInno", "KeyboardSimulator", "R2PPT", "RWinEdt", "blatr", "excel.link", "spectrino", "taskscheduleR", "MDSGUI", "BiplotGUI", "R2wd")) -pkgs_index = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib")[, "Package"] -pkgs_index_b=basename(pkgs_index) +files_b_split = setdiff( + files_b_split, + c( + "PACKAGES", + "PACKAGES.gz", + "PACKAGES.rds", + "PACKAGES.db", + "Archive", + "RInno", + "KeyboardSimulator", + "R2PPT", + "RWinEdt", + "blatr", + "excel.link", + "spectrino", + "taskscheduleR", + "MDSGUI", + "BiplotGUI", + "R2wd" + ) +) +pkgs_index = available.packages( + "https://cran.devxy.io/amd64/jammy/latest/src/contrib" +)[, "Package"] +pkgs_index_b = basename(pkgs_index) if (length(files_b_split) != length(pkgs_index_b)) { pkgs_missing = setdiff(files_b_split, pkgs_index_b) message("Packages missing in index but present in S3:") pkgs_missing - formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs_missing[1:900]))))) + formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs_missing[1:900])))) + ) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") cat(formatted_pkgs_one_line, "\n", sep = "") } -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib" +)) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib" +)) diff --git a/local/packages-without-any-binary.R b/local/packages-without-any-binary.R index a6994ad..f42196f 100644 --- a/local/packages-without-any-binary.R +++ b/local/packages-without-any-binary.R @@ -1,17 +1,24 @@ ### Lists packages without any successful binaries, i.e. pkgs for which all builds errored (per platform & arch) library(DBI) library(dplyr) -con <- DBI::dbConnect(RPostgres::Postgres(), - dbname = "build_metadata", host = "r-binaries.devxy.io", - port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "r_binaries", + password = Sys.getenv("PGPASS"), sslmode = "require" ) -cran_pkgs <- tools::CRAN_package_db() |> - filter(`Date/Publication` <= "2024-12-02") |> +cran_pkgs <- tools::CRAN_package_db() |> + filter(`Date/Publication` <= "2024-12-02") |> pull(Package) -data <- dbGetQuery(con, "SELECT name,platform,arch,removed,error_occurred FROM single_builds;") +data <- dbGetQuery( + con, + "SELECT name,platform,arch,removed,error_occurred FROM single_builds;" +) pkgs <- data |> filter(platform == "alpine-320", arch == "arm64") |> @@ -22,7 +29,11 @@ pkgs <- data |> pkgs = setdiff(cran_pkgs, pkgs) # Format, remove line breaks, and print as a single line -formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900]))))) +formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs[1:900])))) +) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R index c6b67e5..6b32037 100644 --- a/local/weekly-missing-binaries-audit.R +++ b/local/weekly-missing-binaries-audit.R @@ -15,10 +15,14 @@ library(httr2, quietly = TRUE) # Environment / config # --------------------------------------------------------------------------- platform <- Sys.getenv("PLATFORM") -arch <- Sys.getenv("ARCH") +arch <- Sys.getenv("ARCH") -if (nchar(platform) == 0) stop("PLATFORM env var is not set") -if (nchar(arch) == 0) stop("ARCH env var is not set") +if (nchar(platform) == 0) { + stop("PLATFORM env var is not set") +} +if (nchar(arch) == 0) { + stop("ARCH env var is not set") +} # Map platform names to S3 codenames s3_codename <- gsub("-", "", platform) @@ -37,8 +41,13 @@ os_family <- if (grepl("^ubuntu", platform)) { platform } -cat(sprintf("Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n", - platform, arch, s3_codename, os_family)) +cat(sprintf( + "Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n", + platform, + arch, + s3_codename, + os_family +)) # --------------------------------------------------------------------------- # 1. Query PostgreSQL for known build failures (before s3fs init to avoid @@ -47,19 +56,20 @@ cat(sprintf("Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n", cat("Connecting to PostgreSQL...\n") con <- DBI::dbConnect( RPostgres::Postgres(), - dbname = "build_metadata", - host = "r-binaries.devxy.io", - port = 15432, - user = "rpkgs", + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", password = Sys.getenv("PGPASS"), - sslmode = "require" + sslmode = "require" ) errored_pkgs <- DBI::dbGetQuery( con, sprintf( "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE AND platform = '%s' AND arch = '%s'", - platform, arch + platform, + arch ) ) DBI::dbDisconnect(con) @@ -69,7 +79,12 @@ if (nrow(errored_dt) > 0) { setnames(errored_dt, c("Package", "Version")) setkey(errored_dt, Package, Version) } -cat(sprintf("Found %d known build failures for %s/%s\n", nrow(errored_dt), platform, arch)) +cat(sprintf( + "Found %d known build failures for %s/%s\n", + nrow(errored_dt), + platform, + arch +)) # --------------------------------------------------------------------------- # 2. CRAN release packages (uses curl internally - must come after PG) @@ -86,28 +101,36 @@ cran_dt <- data.table( # --------------------------------------------------------------------------- cat("Connecting to S3...\n") s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE ) -s3_path <- sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, s3_codename) +s3_path <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", + arch, + s3_codename +) cat(sprintf("Listing S3 path: %s\n", s3_path)) s3_pkgs <- tryCatch( s3fs::s3_dir_ls(s3_path, recurse = FALSE), error = function(e) { - cat(sprintf("WARNING: Could not list S3 path %s: %s\n", s3_path, conditionMessage(e))) + cat(sprintf( + "WARNING: Could not list S3 path %s: %s\n", + s3_path, + conditionMessage(e) + )) character(0) } ) file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) -parts <- parts[sapply(parts, length) == 3] +matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) +parts <- regmatches(file_names, matches) +parts <- parts[sapply(parts, length) == 3] if (length(parts) == 0) { s3_dt <- data.table(Package = character(0), Version = character(0)) } else { @@ -117,13 +140,18 @@ if (length(parts) == 0) { ) } -cat(sprintf("S3 contains %d tarballs for %s/%s\n", nrow(s3_dt), arch, s3_codename)) +cat(sprintf( + "S3 contains %d tarballs for %s/%s\n", + nrow(s3_dt), + arch, + s3_codename +)) # --------------------------------------------------------------------------- # 4. Find missing packages (CRAN release version not in S3) # --------------------------------------------------------------------------- setkey(cran_dt, Package, Version) -setkey(s3_dt, Package, Version) +setkey(s3_dt, Package, Version) missing_dt <- cran_dt[!s3_dt] cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) @@ -132,9 +160,15 @@ cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) # --------------------------------------------------------------------------- script_dir <- tryCatch( dirname(normalizePath( - if (exists("ofile", envir = sys.frame(1), inherits = FALSE)) sys.frame(1)$ofile - else commandArgs(trailingOnly = FALSE)[grepl("--file=", commandArgs(trailingOnly = FALSE))] |> - sub("--file=", "", x = _), + if (exists("ofile", envir = sys.frame(1), inherits = FALSE)) { + sys.frame(1)$ofile + } else { + commandArgs(trailingOnly = FALSE)[grepl( + "--file=", + commandArgs(trailingOnly = FALSE) + )] |> + sub("--file=", "", x = _) + }, mustWork = FALSE )), error = function(e) "local" @@ -143,13 +177,23 @@ excluded_path <- file.path(script_dir, "excluded-packages.json") excluded_pkgs <- data.table(package = character(0), reason = character(0)) if (file.exists(excluded_path)) { - excluded_raw <- jsonlite::fromJSON(excluded_path) + excluded_raw <- jsonlite::fromJSON(excluded_path) excluded_pkgs <- as.data.table(excluded_raw) - cat(sprintf("Loaded %d excluded packages from %s\n", nrow(excluded_pkgs), excluded_path)) + cat(sprintf( + "Loaded %d excluded packages from %s\n", + nrow(excluded_pkgs), + excluded_path + )) missing_dt <- missing_dt[!Package %in% excluded_pkgs$package] - cat(sprintf("%d packages remain after removing exclusions\n", nrow(missing_dt))) + cat(sprintf( + "%d packages remain after removing exclusions\n", + nrow(missing_dt) + )) } else { - cat(sprintf("No excluded-packages.json found at %s -- skipping exclusion step\n", excluded_path)) + cat(sprintf( + "No excluded-packages.json found at %s -- skipping exclusion step\n", + excluded_path + )) } # --------------------------------------------------------------------------- @@ -157,24 +201,34 @@ if (file.exists(excluded_path)) { # --------------------------------------------------------------------------- if (nrow(errored_dt) > 0) { known_failures_dt <- missing_dt[errored_dt, nomatch = 0] - rebuildable_dt <- missing_dt[!errored_dt] + rebuildable_dt <- missing_dt[!errored_dt] } else { known_failures_dt <- missing_dt[0] - rebuildable_dt <- missing_dt + rebuildable_dt <- missing_dt } -cat(sprintf("Rebuildable: %d | Known failures: %d\n", - nrow(rebuildable_dt), nrow(known_failures_dt))) +cat(sprintf( + "Rebuildable: %d | Known failures: %d\n", + nrow(rebuildable_dt), + nrow(known_failures_dt) +)) # --------------------------------------------------------------------------- # 7. Write rebuildable package names to cache RDS # --------------------------------------------------------------------------- -cache_dir <- "/mnt/cache/packages" -cache_file <- file.path(cache_dir, sprintf("weekly_rebuild_%s_%s.rds", platform, arch)) +cache_dir <- "/mnt/cache/packages" +cache_file <- file.path( + cache_dir, + sprintf("weekly_rebuild_%s_%s.rds", platform, arch) +) if (dir.exists(cache_dir)) { saveRDS(rebuildable_dt$Package, cache_file) - cat(sprintf("Wrote %d rebuildable packages to %s\n", nrow(rebuildable_dt), cache_file)) + cat(sprintf( + "Wrote %d rebuildable packages to %s\n", + nrow(rebuildable_dt), + cache_file + )) } else { cat(sprintf("Cache dir %s does not exist -- skipping RDS write\n", cache_dir)) } @@ -186,29 +240,47 @@ forgejo_token <- Sys.getenv("FORGEJO_TOKEN") if (nchar(forgejo_token) == 0) { cat("FORGEJO_TOKEN not set -- skipping issue update\n") } else { - issue_title <- sprintf("Missing package binaries for latest version (%s)", os_family) + issue_title <- sprintf( + "Missing package binaries for latest version (%s)", + os_family + ) - n_missing <- nrow(missing_dt) - n_rebuild <- nrow(rebuildable_dt) + n_missing <- nrow(missing_dt) + n_rebuild <- nrow(rebuildable_dt) - arch_lines <- sprintf("### %s (%d missing, %d to rebuild)", arch, n_missing, n_rebuild) + arch_lines <- sprintf( + "### %s (%d missing, %d to rebuild)", + arch, + n_missing, + n_rebuild + ) if (nrow(rebuildable_dt) > 0) { - arch_lines <- c(arch_lines, + arch_lines <- c( + arch_lines, paste0("- ", rebuildable_dt$Package, " (", rebuildable_dt$Version, ")") ) } else { arch_lines <- c(arch_lines, "_None_") } if (nrow(known_failures_dt) > 0) { - arch_lines <- c(arch_lines, + arch_lines <- c( + arch_lines, "", "#### Known build failures", - paste0("- ", known_failures_dt$Package, " (", known_failures_dt$Version, ")") + paste0( + "- ", + known_failures_dt$Package, + " (", + known_failures_dt$Version, + ")" + ) ) } build_excluded_footer <- function() { - if (nrow(excluded_pkgs) == 0) return(character(0)) + if (nrow(excluded_pkgs) == 0) { + return(character(0)) + } entries <- paste( paste0(excluded_pkgs$package, " (", excluded_pkgs$reason, ")"), collapse = ", " @@ -217,11 +289,12 @@ if (nchar(forgejo_token) == 0) { } forgejo_base <- "https://git.devxy.io/api/v1" - repo <- "devxy/build-cran-binaries" + repo <- "devxy/build-cran-binaries" search_url <- sprintf( "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", - forgejo_base, repo, + forgejo_base, + repo, utils::URLencode(issue_title, reserved = TRUE) ) search_resp <- httr2::request(search_url) |> @@ -230,15 +303,19 @@ if (nchar(forgejo_token) == 0) { existing_issues <- httr2::resp_body_json(search_resp, simplifyVector = FALSE) - match_idx <- which(sapply(existing_issues, function(x) x$title) == issue_title) + match_idx <- which( + sapply(existing_issues, function(x) x$title) == issue_title + ) today_str <- format(Sys.Date(), "%Y-%m-%d") if (length(match_idx) > 0) { # ---- Update existing issue ---- issue_number <- existing_issues[[match_idx[1]]]$number - old_body <- existing_issues[[match_idx[1]]]$body - if (is.null(old_body)) old_body <- "" + old_body <- existing_issues[[match_idx[1]]]$body + if (is.null(old_body)) { + old_body <- "" + } lines <- strsplit(old_body, "\n", fixed = TRUE)[[1]] @@ -258,7 +335,11 @@ if (nchar(forgejo_token) == 0) { if (length(plat_idx) == 0) { # Platform section missing -- insert before --- footer footer_idx <- which(lines == "---") - insert_at <- if (length(footer_idx) > 0) footer_idx[length(footer_idx)] else length(lines) + 1 + insert_at <- if (length(footer_idx) > 0) { + footer_idx[length(footer_idx)] + } else { + length(lines) + 1 + } new_plat_block <- c(sprintf("## %s", platform), "", arch_lines, "") lines <- c( @@ -271,10 +352,14 @@ if (nchar(forgejo_token) == 0) { # End of platform section: next ## or --- at a higher level, or EOF next_section <- which(grepl("^## |^---", lines) & seq_along(lines) > pi) - plat_end <- if (length(next_section) > 0) next_section[1] - 1 else length(lines) + plat_end <- if (length(next_section) > 0) { + next_section[1] - 1 + } else { + length(lines) + } - plat_lines <- lines[seq(pi, plat_end)] - arch_local_idx <- which(plat_lines == arch_header) + plat_lines <- lines[seq(pi, plat_end)] + arch_local_idx <- which(plat_lines == arch_header) if (length(arch_local_idx) == 0) { # Append arch subsection at end of platform block @@ -285,11 +370,13 @@ if (nchar(forgejo_token) == 0) { lines[seq(plat_end + 1, length(lines))] ) } else { - ai <- pi + arch_local_idx[1] - 1 # absolute line index + ai <- pi + arch_local_idx[1] - 1 # absolute line index # End of arch subsection - next_arch <- which(grepl("^### |^## |^---", lines) & seq_along(lines) > ai) - arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end + next_arch <- which( + grepl("^### |^## |^---", lines) & seq_along(lines) > ai + ) + arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end lines <- c( lines[seq_len(ai - 1)], @@ -302,8 +389,12 @@ if (nchar(forgejo_token) == 0) { # Rebuild excluded footer excl_hdr_idx <- which(lines == "## Excluded packages") if (length(excl_hdr_idx) > 0) { - pre_dash <- which(lines == "---" & seq_along(lines) < excl_hdr_idx[1]) - remove_from <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_hdr_idx[1] + pre_dash <- which(lines == "---" & seq_along(lines) < excl_hdr_idx[1]) + remove_from <- if (length(pre_dash) > 0) { + pre_dash[length(pre_dash)] + } else { + excl_hdr_idx[1] + } lines <- lines[seq_len(remove_from - 1)] } footer <- build_excluded_footer() @@ -313,10 +404,15 @@ if (nchar(forgejo_token) == 0) { new_body <- paste(lines, collapse = "\n") - patch_url <- sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, issue_number) + patch_url <- sprintf( + "%s/repos/%s/issues/%d", + forgejo_base, + repo, + issue_number + ) httr2::request(patch_url) |> httr2::req_headers( - Authorization = paste("token", forgejo_token), + Authorization = paste("token", forgejo_token), `Content-Type` = "application/json" ) |> httr2::req_body_json(list(body = new_body)) |> @@ -324,7 +420,6 @@ if (nchar(forgejo_token) == 0) { httr2::req_perform() cat(sprintf("Updated Forgejo issue #%d: %s\n", issue_number, issue_title)) - } else { # ---- Create new issue ---- body_lines <- c( @@ -342,17 +437,21 @@ if (nchar(forgejo_token) == 0) { post_url <- sprintf("%s/repos/%s/issues", forgejo_base, repo) create_resp <- httr2::request(post_url) |> httr2::req_headers( - Authorization = paste("token", forgejo_token), + Authorization = paste("token", forgejo_token), `Content-Type` = "application/json" ) |> httr2::req_body_json(list( title = issue_title, - body = paste(body_lines, collapse = "\n") + body = paste(body_lines, collapse = "\n") )) |> httr2::req_perform() new_issue <- httr2::resp_body_json(create_resp) - cat(sprintf("Created Forgejo issue #%d: %s\n", new_issue$number, issue_title)) + cat(sprintf( + "Created Forgejo issue #%d: %s\n", + new_issue$number, + issue_title + )) } } diff --git a/renovate.json b/renovate.json index 97ca6ad..53c7306 100644 --- a/renovate.json +++ b/renovate.json @@ -1,11 +1,7 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "local>devxy/renovate-config" - ], - "ignorePaths": [ - "docker/**" - ], + "extends": ["local>devxy/renovate-config"], + "ignorePaths": ["docker/**"], "customManagers": [ { "customType": "regex",