fix(ci): pass the metadata DB to per-minor reindexes and fail on errors (#234)
Some checks failed
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/cron/auto-apply-patches Pipeline failed
ci/crow/cron/weekly-audit-missing/1 Pipeline was successful
ci/crow/cron/weekly-audit-missing/2 Pipeline was successful
ci/crow/cron/weekly-audit-missing/4 Pipeline was successful
ci/crow/cron/weekly-audit-missing/5 Pipeline was successful
ci/crow/cron/weekly-audit-missing/6 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline failed
ci/crow/cron/process-updates/1 Pipeline failed
Some checks failed
ci/crow/cron/process-updates/2 Pipeline was successful
ci/crow/cron/process-updates/11 Pipeline was successful
ci/crow/cron/process-updates/9 Pipeline was successful
ci/crow/cron/process-updates/16 Pipeline was successful
ci/crow/cron/process-updates/15 Pipeline was successful
ci/crow/cron/process-updates/8 Pipeline was successful
ci/crow/cron/process-updates/13 Pipeline was successful
ci/crow/cron/process-updates/12 Pipeline was successful
ci/crow/cron/process-updates/10 Pipeline was successful
ci/crow/cron/process-updates/14 Pipeline was successful
ci/crow/cron/process-updates/4 Pipeline was successful
ci/crow/cron/auto-apply-patches Pipeline failed
ci/crow/cron/weekly-audit-missing/1 Pipeline was successful
ci/crow/cron/weekly-audit-missing/2 Pipeline was successful
ci/crow/cron/weekly-audit-missing/4 Pipeline was successful
ci/crow/cron/weekly-audit-missing/5 Pipeline was successful
ci/crow/cron/weekly-audit-missing/6 Pipeline was successful
ci/crow/cron/process-updates/3 Pipeline was successful
ci/crow/cron/process-updates/5 Pipeline was successful
ci/crow/cron/process-updates/7 Pipeline was successful
ci/crow/cron/process-updates/6 Pipeline failed
ci/crow/cron/process-updates/1 Pipeline failed
Step 1 of rpkgs/bincraft#211: pass the metadata database explicitly so bincraft can drop its implicit production defaults. ## Changes - Pass `metadata_db_*` explicitly to every per-minor `upload_package_index()` call in `reindex.yaml`, `process-updates.yaml`, `weekly-rebuild-reindex.yaml`, `build-all-versions.yaml` and `prune-stale-flat-binaries.yaml`. - Replace `|| true` in those per-minor loops with a list of failed minors: every minor is still attempted, then the step exits 1 naming the ones that failed. - Give the "Upload package indexes" step of `build-all-versions.yaml` the `PGPASS` secret. It never had it, so its per-minor reindexes always aborted and `|| true` hid that. - Run the CDN purge in `process-updates.yaml` and `prune-stale-flat-binaries.yaml` even when a reindex failed, as `reindex.yaml` and `weekly-rebuild-reindex.yaml` already do, because the generic index was still replaced. - `local/repair-built-stamp.R` passes the database too, and fails once all minors are tried instead of only printing the error. The generic (non-per-minor) calls are unchanged: they never read the database. ## Behaviour change A per-minor reindex that fails now turns the pipeline red instead of passing silently. Expect build-all-versions runs to start reindexing per-minor slots for real, since they now have the password. ## Ordering Merge this before rpkgs/bincraft's companion PR is released: `local/install-bincraft.R` installs the latest bincraft tag, and the explicit arguments work with both the current and the next release. Reviewed-on: #234
This commit is contained in:
parent
a310790ad8
commit
3618168572
22 changed files with 113 additions and 38 deletions
|
|
@ -193,6 +193,10 @@ steps:
|
||||||
from_secret: B2_S3_ACCESS_KEY
|
from_secret: B2_S3_ACCESS_KEY
|
||||||
B2_S3_SECRET_KEY:
|
B2_S3_SECRET_KEY:
|
||||||
from_secret: B2_S3_SECRET_KEY
|
from_secret: B2_S3_SECRET_KEY
|
||||||
|
# A per-minor index reads the ABI classifier cache and the flat-object
|
||||||
|
# safety table from the build database, and is not republished without it.
|
||||||
|
PGPASS:
|
||||||
|
from_secret: PGPASS
|
||||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||||
volumes:
|
volumes:
|
||||||
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
||||||
|
|
@ -200,10 +204,15 @@ steps:
|
||||||
- |
|
- |
|
||||||
CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e "cat(bincraft::set_codename(NULL))")
|
CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e "cat(bincraft::set_codename(NULL))")
|
||||||
/opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))"
|
/opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))"
|
||||||
|
FAILED_MINORS=""
|
||||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||||
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
||||||
/opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
|
/opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
|
||||||
done
|
done
|
||||||
|
if [ -n "$FAILED_MINORS" ]; then
|
||||||
|
echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
backend_options:
|
backend_options:
|
||||||
docker:
|
docker:
|
||||||
resources:
|
resources:
|
||||||
|
|
|
||||||
|
|
@ -194,7 +194,7 @@ steps:
|
||||||
# rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
|
# rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
|
||||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS="-a"; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||||
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
|
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
|
||||||
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"
|
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"
|
||||||
- |
|
- |
|
||||||
PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2)
|
PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2)
|
||||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||||
|
|
@ -206,16 +206,21 @@ steps:
|
||||||
mkdir -p "$LIB"
|
mkdir -p "$LIB"
|
||||||
R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
|
R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true
|
||||||
R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
|
R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true
|
||||||
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true
|
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true
|
||||||
# Also build current packages whose required dependency tree needs a newer R.
|
# Also build current packages whose required dependency tree needs a newer R.
|
||||||
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION"
|
R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/Rscript" local/build-newer-r.R "$R_VERSION"
|
||||||
done
|
done
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
||||||
- |
|
- |
|
||||||
|
FAILED_MINORS=""
|
||||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||||
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
||||||
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = '${OS_ID}', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
|
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = '${OS_ID}', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
|
||||||
done
|
done
|
||||||
|
if [ -n "$FAILED_MINORS" ]; then
|
||||||
|
echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
backend_options:
|
backend_options:
|
||||||
docker:
|
docker:
|
||||||
resources:
|
resources:
|
||||||
|
|
@ -242,3 +247,7 @@ steps:
|
||||||
commands:
|
commands:
|
||||||
- apk add --no-cache -q bash curl
|
- apk add --no-cache -q bash curl
|
||||||
- bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3"
|
- bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3"
|
||||||
|
# Run it even when a per-minor re-index above failed: the generic index
|
||||||
|
# and the objects were still replaced, and a stale edge keeps them hidden.
|
||||||
|
when:
|
||||||
|
- status: [success, failure]
|
||||||
|
|
|
||||||
|
|
@ -209,10 +209,15 @@ steps:
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
/opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
/opt/R/4.5.3/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
||||||
|
FAILED_MINORS=""
|
||||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||||
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
||||||
/opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
|
/opt/R/4.5.3/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
|
||||||
done
|
done
|
||||||
|
if [ -n "$FAILED_MINORS" ]; then
|
||||||
|
echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
backend_options:
|
backend_options:
|
||||||
docker:
|
docker:
|
||||||
resources:
|
resources:
|
||||||
|
|
@ -240,3 +245,7 @@ steps:
|
||||||
fi
|
fi
|
||||||
apk add --no-cache -q bash curl jq
|
apk add --no-cache -q bash curl jq
|
||||||
bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
|
bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES
|
||||||
|
# Run it even when a per-minor re-index above failed: the generic index
|
||||||
|
# and the objects were still replaced, and a stale edge keeps them hidden.
|
||||||
|
when:
|
||||||
|
- status: [success, failure]
|
||||||
|
|
|
||||||
|
|
@ -162,10 +162,15 @@ steps:
|
||||||
# The codename is detected from the image's /etc/os-release.
|
# The codename is detected from the image's /etc/os-release.
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
||||||
- |
|
- |
|
||||||
|
FAILED_MINORS=""
|
||||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||||
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
||||||
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
|
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
|
||||||
done
|
done
|
||||||
|
if [ -n "$FAILED_MINORS" ]; then
|
||||||
|
echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Purge CDN cache
|
- name: Purge CDN cache
|
||||||
image: reg.devxy.io/docker.io/library/alpine:3.24
|
image: reg.devxy.io/docker.io/library/alpine:3.24
|
||||||
|
|
|
||||||
|
|
@ -165,10 +165,15 @@ steps:
|
||||||
# The codename is detected from the image's /etc/os-release.
|
# The codename is detected from the image's /etc/os-release.
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
||||||
- |
|
- |
|
||||||
|
FAILED_MINORS=""
|
||||||
for RBIN in /opt/R/[0-9]*/bin/R; do
|
for RBIN in /opt/R/[0-9]*/bin/R; do
|
||||||
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2)
|
||||||
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true
|
/opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_port = 15432L, metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'verify-full')" || FAILED_MINORS="$FAILED_MINORS $RMINOR"
|
||||||
done
|
done
|
||||||
|
if [ -n "$FAILED_MINORS" ]; then
|
||||||
|
echo "Could not republish the per-minor index for:$FAILED_MINORS" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Purge CDN cache
|
- name: Purge CDN cache
|
||||||
image: reg.devxy.io/docker.io/library/alpine:3.24
|
image: reg.devxy.io/docker.io/library/alpine:3.24
|
||||||
|
|
|
||||||
|
|
@ -139,6 +139,7 @@ For Alpine, no rules existed until recently, establishing a foundation for semi-
|
||||||
|
|
||||||
The build metadata is stored in a PostgreSQL database.
|
The build metadata is stored in a PostgreSQL database.
|
||||||
The database has a public endpoint at `r-binaries.devxy.io` and port `15432`.
|
The database has a public endpoint at `r-binaries.devxy.io` and port `15432`.
|
||||||
|
It serves a Let's Encrypt certificate for that name, so connect with `sslmode=verify-full`; from R, pass `sslrootcert = bincraft::postgres_sslrootcert("verify-full")` as well, since libpq does not read the system trust store by default.
|
||||||
It contains one table named `single_builds`, which holds the build metadata for each package:
|
It contains one table named `single_builds`, which holds the build metadata for each package:
|
||||||
|
|
||||||
| package_name | tag | platform | error_occurred | build_timestamp | build_duration | error | size | removed |
|
| package_name | tag | platform | error_occurred | build_timestamp | build_duration | error | size | removed |
|
||||||
|
|
|
||||||
|
|
@ -47,7 +47,7 @@ db <- list(
|
||||||
metadata_db_port = 15432L,
|
metadata_db_port = 15432L,
|
||||||
metadata_db_user = "rpkgs",
|
metadata_db_user = "rpkgs",
|
||||||
metadata_db_password = Sys.getenv("PGPASS"),
|
metadata_db_password = Sys.getenv("PGPASS"),
|
||||||
metadata_db_sslmode = "require"
|
metadata_db_sslmode = "verify-full"
|
||||||
)
|
)
|
||||||
|
|
||||||
read_index <- function(index_url) {
|
read_index <- function(index_url) {
|
||||||
|
|
|
||||||
|
|
@ -152,7 +152,8 @@ con <- DBI::dbConnect(
|
||||||
port = 15432,
|
port = 15432,
|
||||||
user = "rpkgs",
|
user = "rpkgs",
|
||||||
password = Sys.getenv("PGPASS"),
|
password = Sys.getenv("PGPASS"),
|
||||||
sslmode = "require"
|
sslmode = "verify-full",
|
||||||
|
sslrootcert = bincraft::postgres_sslrootcert("verify-full")
|
||||||
)
|
)
|
||||||
# Scope the skip to the R minor this pass is running under. `single_builds`
|
# Scope the skip to the R minor this pass is running under. `single_builds`
|
||||||
# records `r_version` per attempt, but querying without it made a non-primary
|
# records `r_version` per attempt, but querying without it made a non-primary
|
||||||
|
|
@ -250,7 +251,7 @@ mapply(
|
||||||
metadata_db_table = "single_builds",
|
metadata_db_table = "single_builds",
|
||||||
metadata_db_user = "rpkgs",
|
metadata_db_user = "rpkgs",
|
||||||
metadata_db_password = Sys.getenv("PGPASS"),
|
metadata_db_password = Sys.getenv("PGPASS"),
|
||||||
metadata_db_sslmode = "require",
|
metadata_db_sslmode = "verify-full",
|
||||||
metadata_db_port = 15432,
|
metadata_db_port = 15432,
|
||||||
archive = TRUE,
|
archive = TRUE,
|
||||||
patches = "local/patches",
|
patches = "local/patches",
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,7 @@ if ("--dry-run" %in% args) {
|
||||||
metadata_db_name = "build_metadata",
|
metadata_db_name = "build_metadata",
|
||||||
metadata_db_user = "rpkgs",
|
metadata_db_user = "rpkgs",
|
||||||
metadata_db_password = Sys.getenv("PGPASS"),
|
metadata_db_password = Sys.getenv("PGPASS"),
|
||||||
metadata_db_sslmode = "require",
|
metadata_db_sslmode = "verify-full",
|
||||||
metadata_db_port = 15432
|
metadata_db_port = 15432
|
||||||
),
|
),
|
||||||
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||||
|
|
@ -61,7 +61,7 @@ if ("--dry-run" %in% args) {
|
||||||
metadata_db_table = "single_builds",
|
metadata_db_table = "single_builds",
|
||||||
metadata_db_user = "rpkgs",
|
metadata_db_user = "rpkgs",
|
||||||
metadata_db_password = Sys.getenv("PGPASS"),
|
metadata_db_password = Sys.getenv("PGPASS"),
|
||||||
metadata_db_sslmode = "require",
|
metadata_db_sslmode = "verify-full",
|
||||||
metadata_db_port = 15432,
|
metadata_db_port = 15432,
|
||||||
archive = TRUE,
|
archive = TRUE,
|
||||||
patches = "local/patches",
|
patches = "local/patches",
|
||||||
|
|
|
||||||
|
|
@ -115,7 +115,7 @@ for (ver in versions) {
|
||||||
metadata_db_table = "single_builds",
|
metadata_db_table = "single_builds",
|
||||||
metadata_db_user = "rpkgs",
|
metadata_db_user = "rpkgs",
|
||||||
metadata_db_password = Sys.getenv("PGPASS"),
|
metadata_db_password = Sys.getenv("PGPASS"),
|
||||||
metadata_db_sslmode = "require",
|
metadata_db_sslmode = "verify-full",
|
||||||
metadata_db_port = 15432
|
metadata_db_port = 15432
|
||||||
)
|
)
|
||||||
if (sensitive) touched_minor <- TRUE else touched_generic <- TRUE
|
if (sensitive) touched_minor <- TRUE else touched_generic <- TRUE
|
||||||
|
|
|
||||||
|
|
@ -69,7 +69,8 @@ con <- DBI::dbConnect(
|
||||||
port = 15432,
|
port = 15432,
|
||||||
user = "rpkgs",
|
user = "rpkgs",
|
||||||
password = Sys.getenv("PGPASS"),
|
password = Sys.getenv("PGPASS"),
|
||||||
sslmode = "require"
|
sslmode = "verify-full",
|
||||||
|
sslrootcert = bincraft::postgres_sslrootcert("verify-full")
|
||||||
)
|
)
|
||||||
on.exit(DBI::dbDisconnect(con), add = TRUE)
|
on.exit(DBI::dbDisconnect(con), add = TRUE)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,8 @@ con <- DBI::dbConnect(
|
||||||
port = 15432,
|
port = 15432,
|
||||||
user = "r_binaries",
|
user = "r_binaries",
|
||||||
password = Sys.getenv("PGPASS"),
|
password = Sys.getenv("PGPASS"),
|
||||||
sslmode = "require"
|
sslmode = "verify-full",
|
||||||
|
sslrootcert = bincraft::postgres_sslrootcert("verify-full")
|
||||||
)
|
)
|
||||||
|
|
||||||
pkgs_db <- query_metadata_table() |>
|
pkgs_db <- query_metadata_table() |>
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,8 @@ con <- DBI::dbConnect(
|
||||||
port = 15432L,
|
port = 15432L,
|
||||||
user = "rpkgs",
|
user = "rpkgs",
|
||||||
password = Sys.getenv("PGPASS"),
|
password = Sys.getenv("PGPASS"),
|
||||||
sslmode = "require"
|
sslmode = "verify-full",
|
||||||
|
sslrootcert = bincraft::postgres_sslrootcert("verify-full")
|
||||||
)
|
)
|
||||||
on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE)
|
on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,8 @@ con <- DBI::dbConnect(
|
||||||
port = 15432,
|
port = 15432,
|
||||||
user = "r_binaries",
|
user = "r_binaries",
|
||||||
password = Sys.getenv("PGPASS"),
|
password = Sys.getenv("PGPASS"),
|
||||||
sslmode = "require"
|
sslmode = "verify-full",
|
||||||
|
sslrootcert = bincraft::postgres_sslrootcert("verify-full")
|
||||||
)
|
)
|
||||||
|
|
||||||
`%nin%` <- Negate(`%in%`)
|
`%nin%` <- Negate(`%in%`)
|
||||||
|
|
@ -45,12 +46,12 @@ data <- dbGetQuery(
|
||||||
"SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;",
|
"SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;",
|
||||||
params = list(platform, arch)
|
params = list(platform, arch)
|
||||||
)
|
)
|
||||||
removed_pkgs = get_removed_cran_packages(lubridate::interval(
|
removed_pkgs <- get_removed_cran_packages(lubridate::interval(
|
||||||
lubridate::today(),
|
lubridate::today(),
|
||||||
lubridate::today() - 2
|
lubridate::today() - 2
|
||||||
))$name
|
))$name
|
||||||
pkgs_db <- unique(data$name)
|
pkgs_db <- unique(data$name)
|
||||||
pkgs_db = setdiff(pkgs_db, removed_pkgs)
|
pkgs_db <- setdiff(pkgs_db, removed_pkgs)
|
||||||
|
|
||||||
pkgs <- setdiff(cran_pkgs, pkgs_db)
|
pkgs <- setdiff(cran_pkgs, pkgs_db)
|
||||||
formatted_pkgs <- gsub(
|
formatted_pkgs <- gsub(
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,8 @@ con <- DBI::dbConnect(
|
||||||
port = 15432,
|
port = 15432,
|
||||||
user = "rpkgs",
|
user = "rpkgs",
|
||||||
password = Sys.getenv("PGPASS"),
|
password = Sys.getenv("PGPASS"),
|
||||||
sslmode = "require"
|
sslmode = "verify-full",
|
||||||
|
sslrootcert = bincraft::postgres_sslrootcert("verify-full")
|
||||||
)
|
)
|
||||||
|
|
||||||
cran_archive <- tools::CRAN_archive_db()
|
cran_archive <- tools::CRAN_archive_db()
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,8 @@ con <- DBI::dbConnect(
|
||||||
port = 15432,
|
port = 15432,
|
||||||
user = "r_binaries",
|
user = "r_binaries",
|
||||||
password = Sys.getenv("PGPASS"),
|
password = Sys.getenv("PGPASS"),
|
||||||
sslmode = "require"
|
sslmode = "verify-full",
|
||||||
|
sslrootcert = bincraft::postgres_sslrootcert("verify-full")
|
||||||
)
|
)
|
||||||
|
|
||||||
cran_pkgs <- tools::CRAN_package_db() |>
|
cran_pkgs <- tools::CRAN_package_db() |>
|
||||||
|
|
@ -27,7 +28,7 @@ pkgs <- data |>
|
||||||
distinct(name) |>
|
distinct(name) |>
|
||||||
pull(name)
|
pull(name)
|
||||||
|
|
||||||
pkgs = setdiff(cran_pkgs, pkgs)
|
pkgs <- setdiff(cran_pkgs, pkgs)
|
||||||
# Format, remove line breaks, and print as a single line
|
# Format, remove line breaks, and print as a single line
|
||||||
formatted_pkgs <- gsub(
|
formatted_pkgs <- gsub(
|
||||||
'"',
|
'"',
|
||||||
|
|
|
||||||
|
|
@ -53,7 +53,8 @@ con <- DBI::dbConnect(
|
||||||
port = 15432,
|
port = 15432,
|
||||||
user = "rpkgs",
|
user = "rpkgs",
|
||||||
password = Sys.getenv("PGPASS"),
|
password = Sys.getenv("PGPASS"),
|
||||||
sslmode = "require"
|
sslmode = "verify-full",
|
||||||
|
sslrootcert = bincraft::postgres_sslrootcert("verify-full")
|
||||||
)
|
)
|
||||||
on.exit(DBI::dbDisconnect(con), add = TRUE)
|
on.exit(DBI::dbDisconnect(con), add = TRUE)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -87,7 +87,8 @@ con <- DBI::dbConnect(
|
||||||
port = 15432,
|
port = 15432,
|
||||||
user = "rpkgs",
|
user = "rpkgs",
|
||||||
password = Sys.getenv("PGPASS"),
|
password = Sys.getenv("PGPASS"),
|
||||||
sslmode = "require"
|
sslmode = "verify-full",
|
||||||
|
sslrootcert = bincraft::postgres_sslrootcert("verify-full")
|
||||||
)
|
)
|
||||||
on.exit(DBI::dbDisconnect(con), add = TRUE)
|
on.exit(DBI::dbDisconnect(con), add = TRUE)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -86,7 +86,8 @@ if (length(explicit) > 0L) {
|
||||||
port = 15432L,
|
port = 15432L,
|
||||||
user = "rpkgs",
|
user = "rpkgs",
|
||||||
password = Sys.getenv("PGPASS"),
|
password = Sys.getenv("PGPASS"),
|
||||||
sslmode = "require"
|
sslmode = "verify-full",
|
||||||
|
sslrootcert = bincraft::postgres_sslrootcert("verify-full")
|
||||||
)
|
)
|
||||||
on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE)
|
on.exit(try(DBI::dbDisconnect(con), silent = TRUE), add = TRUE)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -171,7 +171,7 @@ for (i in seq_along(mine)) {
|
||||||
metadata_db_table = "single_builds",
|
metadata_db_table = "single_builds",
|
||||||
metadata_db_user = "rpkgs",
|
metadata_db_user = "rpkgs",
|
||||||
metadata_db_password = Sys.getenv("PGPASS"),
|
metadata_db_password = Sys.getenv("PGPASS"),
|
||||||
metadata_db_sslmode = "require",
|
metadata_db_sslmode = "verify-full",
|
||||||
metadata_db_port = 15432,
|
metadata_db_port = 15432,
|
||||||
archive = TRUE,
|
archive = TRUE,
|
||||||
upload = TRUE,
|
upload = TRUE,
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue