fix(ci): pass the metadata DB to per-minor reindexes and fail on errors

- Pass `metadata_db_*` explicitly to every per-minor `upload_package_index()` call, so bincraft can drop its implicit production defaults (rpkgs/bincraft#211).
- Replace `|| true` in the per-minor reindex loops with a failure list, so every minor is still attempted but a failed reindex fails the step.
- Give the "Upload package indexes" step of build-all-versions the `PGPASS` secret, without which its per-minor reindexes always aborted.
- Run the CDN purge in process-updates and prune-stale-flat-binaries even when a reindex failed, as reindex.yaml already does.
- Fail local/repair-built-stamp.R when a per-minor reindex fails instead of only printing the error.

.crow/build-all-versions.yaml

@@ -193,6 +193,10 @@ steps:
         from_secret: B2_S3_ACCESS_KEY
       B2_S3_SECRET_KEY:
         from_secret: B2_S3_SECRET_KEY
+      # A per-minor index reads the ABI classifier cache and the flat-object
+      # safety table from the build database, and is not republished without it.
+      PGPASS:
+        from_secret: PGPASS
       R_LIBS_USER: /mnt/cache/R-pkgs
     volumes:
       - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
@@ -200,10 +204,15 @@ steps:
       - |
         CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e "cat(bincraft::set_codename(NULL))")
         /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))"
+        FAILED_MINORS=""
         for RBIN in /opt/R/[0-9]*/bin/R; do
           RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
-          /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
+          /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
         done
+        if [ -n "$FAILED_MINORS" ]; then
+          echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
+          exit 1
+        fi
     backend_options:
       docker:
         resources:

.crow/process-updates.yaml

@@ -212,10 +212,15 @@ steps:
         done
       - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
       - |
+        FAILED_MINORS=""
         for RBIN in /opt/R/[0-9]*/bin/R; do
           RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
-          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = '${OS_ID}', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
+          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = '${OS_ID}', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
         done
+        if [ -n "$FAILED_MINORS" ]; then
+          echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
+          exit 1
+        fi
     backend_options:
       docker:
         resources:
@@ -242,3 +247,7 @@ steps:
     commands:
       - apk add --no-cache -q bash curl
       - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3"
+    # Run it even when a per-minor re-index above failed: the generic index
+    # and the objects were still replaced, and a stale edge keeps them hidden.
+    when:
+      - status: [success, failure]

.crow/prune-stale-flat-binaries.yaml

@@ -209,10 +209,15 @@ steps:
           exit 0
         fi
         /opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
+        FAILED_MINORS=""
         for RBIN in /opt/R/[0-9]*/bin/R; do
           RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
-          /opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
+          /opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
         done
+        if [ -n "$FAILED_MINORS" ]; then
+          echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
+          exit 1
+        fi
     backend_options:
       docker:
         resources:
@@ -240,3 +245,7 @@ steps:
         fi
         apk add --no-cache -q bash curl jq
         bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
+    # Run it even when a per-minor re-index above failed: the generic index
+    # and the objects were still replaced, and a stale edge keeps them hidden.
+    when:
+      - status: [success, failure]

.crow/reindex.yaml

@@ -162,10 +162,15 @@ steps:
       # The codename is detected from the image's /etc/os-release.
       - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
       - |
+        FAILED_MINORS=""
         for RBIN in /opt/R/[0-9]*/bin/R; do
           RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
-          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
+          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
         done
+        if [ -n "$FAILED_MINORS" ]; then
+          echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
+          exit 1
+        fi
 
   - name: Purge CDN cache
     image: reg.devxy.io/docker.io/library/alpine:3.24

.crow/weekly-rebuild-reindex.yaml

@@ -165,10 +165,15 @@ steps:
       # The codename is detected from the image's /etc/os-release.
       - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
       - |
+        FAILED_MINORS=""
         for RBIN in /opt/R/[0-9]*/bin/R; do
           RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
-          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
+          /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
         done
+        if [ -n "$FAILED_MINORS" ]; then
+          echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
+          exit 1
+        fi
 
   - name: Purge CDN cache
     image: reg.devxy.io/docker.io/library/alpine:3.24

local/repair-built-stamp.R

@@ -152,18 +152,44 @@ bincraft::upload_package_index(
   s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
   s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY")
 )
+# A per-minor index reads the ABI classifier cache and the flat-object safety
+# table from the build database. A failed minor does not stop the others, but
+# it does fail the run once they are done.
+failed_minors <- character(0L)
 for (minor in r_minors) {
-  try(
+  ok <- tryCatch(
-    bincraft::upload_package_index(
+    {
-      codename = codename,
+      bincraft::upload_package_index(
-      r_minor = minor,
+        codename = codename,
-      s3_endpoint = endpoint,
+        r_minor = minor,
-      s3_region = region,
+        s3_endpoint = endpoint,
-      s3_bucket = bucket,
+        s3_region = region,
-      s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
+        s3_bucket = bucket,
-      s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY")
+        s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
-    ),
+        s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
-    silent = FALSE
+        metadata_db_host = "r-binaries.devxy.io",
+        metadata_db_name = "build_metadata",
+        metadata_db_port = 15432L,
+        metadata_db_user = "rpkgs",
+        metadata_db_password = Sys.getenv("PGPASS"),
+        metadata_db_sslmode = "require"
+      )
+      TRUE
+    },
+    error = function(e) {
+      message(sprintf("Re-indexing %s failed: %s", minor, conditionMessage(e)))
+      FALSE
+    }
+  )
+  if (!ok) {
+    failed_minors <- c(failed_minors, minor)
+  }
+}
+if (length(failed_minors) > 0L) {
+  stop(
+    "Could not republish the per-minor index for: ",
+    paste(failed_minors, collapse = ", "),
+    call. = FALSE
   )
 }