fix(ci): pass the metadata DB to per-minor reindexes and fail on errors (#234)

Step 1 of rpkgs/bincraft#211: pass the metadata database explicitly so bincraft can drop its implicit production defaults.

## Changes

- Pass `metadata_db_*` explicitly to every per-minor `upload_package_index()` call in `reindex.yaml`, `process-updates.yaml`, `weekly-rebuild-reindex.yaml`, `build-all-versions.yaml` and `prune-stale-flat-binaries.yaml`.
- Replace `|| true` in those per-minor loops with a list of failed minors: every minor is still attempted, then the step exits 1 naming the ones that failed.
- Give the "Upload package indexes" step of `build-all-versions.yaml` the `PGPASS` secret. It never had it, so its per-minor reindexes always aborted and `|| true` hid that.
- Run the CDN purge in `process-updates.yaml` and `prune-stale-flat-binaries.yaml` even when a reindex failed, as `reindex.yaml` and `weekly-rebuild-reindex.yaml` already do, because the generic index was still replaced.
- `local/repair-built-stamp.R` passes the database too, and fails once all minors are tried instead of only printing the error.

The generic (non-per-minor) calls are unchanged: they never read the database.

## Behaviour change

A per-minor reindex that fails now turns the pipeline red instead of passing silently.
Expect build-all-versions runs to start reindexing per-minor slots for real, since they now have the password.

## Ordering

Merge this before rpkgs/bincraft's companion PR is released: `local/install-bincraft.R` installs the latest bincraft tag, and the explicit arguments work with both the current and the next release.

Reviewed-on: #234

local/backfill-abi-classification.R

@@ -47,7 +47,7 @@ db <- list(
   metadata_db_port = 15432L,
   metadata_db_user = "rpkgs",
   metadata_db_password = Sys.getenv("PGPASS"),
-  metadata_db_sslmode = "require"
+  metadata_db_sslmode = "verify-full"
 )
 
 read_index <- function(index_url) {

local/build-all.R

@@ -152,7 +152,8 @@ con <- DBI::dbConnect(
   port = 15432,
   user = "rpkgs",
   password = Sys.getenv("PGPASS"),
-  sslmode = "require"
+  sslmode = "verify-full",
+  sslrootcert = bincraft::postgres_sslrootcert("verify-full")
 )
 # Scope the skip to the R minor this pass is running under. `single_builds`
 # records `r_version` per attempt, but querying without it made a non-primary
@@ -250,7 +251,7 @@ mapply(
       metadata_db_table = "single_builds",
       metadata_db_user = "rpkgs",
       metadata_db_password = Sys.getenv("PGPASS"),
-      metadata_db_sslmode = "require",
+      metadata_db_sslmode = "verify-full",
       metadata_db_port = 15432,
       archive = TRUE,
       patches = "local/patches",

local/build-newer-r.R

@@ -48,7 +48,7 @@ if ("--dry-run" %in% args) {
         metadata_db_name = "build_metadata",
         metadata_db_user = "rpkgs",
         metadata_db_password = Sys.getenv("PGPASS"),
-        metadata_db_sslmode = "require",
+        metadata_db_sslmode = "verify-full",
         metadata_db_port = 15432
       ),
       s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
@@ -61,7 +61,7 @@ if ("--dry-run" %in% args) {
       metadata_db_table = "single_builds",
       metadata_db_user = "rpkgs",
       metadata_db_password = Sys.getenv("PGPASS"),
-      metadata_db_sslmode = "require",
+      metadata_db_sslmode = "verify-full",
       metadata_db_port = 15432,
       archive = TRUE,
       patches = "local/patches",

local/build-one.R

@@ -115,7 +115,7 @@ for (ver in versions) {
     metadata_db_table = "single_builds",
     metadata_db_user = "rpkgs",
     metadata_db_password = Sys.getenv("PGPASS"),
-    metadata_db_sslmode = "require",
+    metadata_db_sslmode = "verify-full",
     metadata_db_port = 15432
   )
   if (sensitive) touched_minor <- TRUE else touched_generic <- TRUE

local/failing-builds-report.R

@@ -69,7 +69,8 @@ con <- DBI::dbConnect(
   port = 15432,
   user = "rpkgs",
   password = Sys.getenv("PGPASS"),
-  sslmode = "require"
+  sslmode = "verify-full",
+  sslrootcert = bincraft::postgres_sslrootcert("verify-full")
 )
 on.exit(DBI::dbDisconnect(con), add = TRUE)
 

local/label-removed-cran-packages.R

@@ -13,7 +13,8 @@ con <- DBI::dbConnect(
   port = 15432,
   user = "r_binaries",
   password = Sys.getenv("PGPASS"),
-  sslmode = "require"
+  sslmode = "verify-full",
+  sslrootcert = bincraft::postgres_sslrootcert("verify-full")
 )
 
 pkgs_db <- query_metadata_table() |>

local/list-unsafe-flat-objects.R

@@ -30,7 +30,8 @@ con <- DBI::dbConnect(
   port = 15432L,
   user = "rpkgs",
   password = Sys.getenv("PGPASS"),
-  sslmode = "require"
+  sslmode = "verify-full",
+  sslrootcert = bincraft::postgres_sslrootcert("verify-full")
 )
 on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE)
 

local/missing-cran-packages-db.R

@@ -10,7 +10,8 @@ con <- DBI::dbConnect(
   port = 15432,
   user = "r_binaries",
   password = Sys.getenv("PGPASS"),
-  sslmode = "require"
+  sslmode = "verify-full",
+  sslrootcert = bincraft::postgres_sslrootcert("verify-full")
 )
 
 `%nin%` <- Negate(`%in%`)
@@ -45,12 +46,12 @@ data <- dbGetQuery(
   "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;",
   params = list(platform, arch)
 )
-removed_pkgs = get_removed_cran_packages(lubridate::interval(
+removed_pkgs <- get_removed_cran_packages(lubridate::interval(
   lubridate::today(),
   lubridate::today() - 2
 ))$name
 pkgs_db <- unique(data$name)
-pkgs_db = setdiff(pkgs_db, removed_pkgs)
+pkgs_db <- setdiff(pkgs_db, removed_pkgs)
 
 pkgs <- setdiff(cran_pkgs, pkgs_db)
 formatted_pkgs <- gsub(

local/packages-to-build.R

@@ -32,7 +32,8 @@ con <- DBI::dbConnect(
   port = 15432,
   user = "rpkgs",
   password = Sys.getenv("PGPASS"),
-  sslmode = "require"
+  sslmode = "verify-full",
+  sslrootcert = bincraft::postgres_sslrootcert("verify-full")
 )
 
 cran_archive <- tools::CRAN_archive_db()

local/packages-without-any-binary.R

@@ -8,7 +8,8 @@ con <- DBI::dbConnect(
   port = 15432,
   user = "r_binaries",
   password = Sys.getenv("PGPASS"),
-  sslmode = "require"
+  sslmode = "verify-full",
+  sslrootcert = bincraft::postgres_sslrootcert("verify-full")
 )
 
 cran_pkgs <- tools::CRAN_package_db() |>
@@ -27,7 +28,7 @@ pkgs <- data |>
   distinct(name) |>
   pull(name)
 
-pkgs = setdiff(cran_pkgs, pkgs)
+pkgs <- setdiff(cran_pkgs, pkgs)
 # Format, remove line breaks, and print as a single line
 formatted_pkgs <- gsub(
   '"',

local/proposal-tracking.R

@@ -53,7 +53,8 @@ con <- DBI::dbConnect(
   port = 15432,
   user = "rpkgs",
   password = Sys.getenv("PGPASS"),
-  sslmode = "require"
+  sslmode = "verify-full",
+  sslrootcert = bincraft::postgres_sslrootcert("verify-full")
 )
 on.exit(DBI::dbDisconnect(con), add = TRUE)
 

local/propose-patches.R

@@ -87,7 +87,8 @@ con <- DBI::dbConnect(
   port = 15432,
   user = "rpkgs",
   password = Sys.getenv("PGPASS"),
-  sslmode = "require"
+  sslmode = "verify-full",
+  sslrootcert = bincraft::postgres_sslrootcert("verify-full")
 )
 on.exit(DBI::dbDisconnect(con), add = TRUE)
 

local/prune-stale-flat-binaries.R

@@ -86,7 +86,8 @@ if (length(explicit) > 0L) {
     port = 15432L,
     user = "rpkgs",
     password = Sys.getenv("PGPASS"),
-    sslmode = "require"
+    sslmode = "verify-full",
+    sslrootcert = bincraft::postgres_sslrootcert("verify-full")
   )
   on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE)
 

local/rebuild-missing.R

@@ -171,7 +171,7 @@ for (i in seq_along(mine)) {
       metadata_db_table = "single_builds",
       metadata_db_user = "rpkgs",
       metadata_db_password = Sys.getenv("PGPASS"),
-      metadata_db_sslmode = "require",
+      metadata_db_sslmode = "verify-full",
       metadata_db_port = 15432,
       archive = TRUE,
       upload = TRUE,

local/repair-built-stamp.R

@@ -152,18 +152,44 @@ bincraft::upload_package_index(
   s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
   s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY")
 )
+# A per-minor index reads the ABI classifier cache and the flat-object safety
+# table from the build database. A failed minor does not stop the others, but
+# it does fail the run once they are done.
+failed_minors <- character(0L)
 for (minor in r_minors) {
-  try(
-    bincraft::upload_package_index(
-      codename = codename,
-      r_minor = minor,
-      s3_endpoint = endpoint,
-      s3_region = region,
-      s3_bucket = bucket,
-      s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
-      s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY")
-    ),
-    silent = FALSE
+  ok <- tryCatch(
+    {
+      bincraft::upload_package_index(
+        codename = codename,
+        r_minor = minor,
+        s3_endpoint = endpoint,
+        s3_region = region,
+        s3_bucket = bucket,
+        s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
+        s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
+        metadata_db_host = "r-binaries.devxy.io",
+        metadata_db_name = "build_metadata",
+        metadata_db_port = 15432L,
+        metadata_db_user = "rpkgs",
+        metadata_db_password = Sys.getenv("PGPASS"),
+        metadata_db_sslmode = "verify-full"
+      )
+      TRUE
+    },
+    error = function(e) {
+      message(sprintf("Re-indexing %s failed: %s", minor, conditionMessage(e)))
+      FALSE
+    }
+  )
+  if (!ok) {
+    failed_minors <- c(failed_minors, minor)
+  }
+}
+if (length(failed_minors) > 0L) {
+  stop(
+    "Could not republish the per-minor index for: ",
+    paste(failed_minors, collapse = ", "),
+    call. = FALSE
   )
 }
 

local/weekly-missing-binaries-audit.R

@@ -60,7 +60,8 @@ con <- DBI::dbConnect(
   port = 15432,
   user = "rpkgs",
   password = Sys.getenv("PGPASS"),
-  sslmode = "require"
+  sslmode = "verify-full",
+  sslrootcert = bincraft::postgres_sslrootcert("verify-full")
 )
 
 errored_pkgs <- DBI::dbGetQuery(