From 1ad7a6bfe9ae8798f96cbbae23a8a9718f9872a1 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 1 Jul 2026 08:10:02 +0000 Subject: [PATCH] chore: resolve latest bincraft release dynamically (no hardcoded pins) (#107) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Stop hardcoding the bincraft version. Every `.crow` workflow and the build-one image pinned `@vX.Y.Z` (and a `packageVersion() != "X.Y.Z"` guard), so each bincraft release meant editing the version in ~8 places — and it was easy to miss one (the Dockerfile lagged at v4.2.1; v4.4.1 shipped without the empty-env fix because of exactly this churn). ## Change New `local/install-bincraft.R` resolves the **latest release tag dynamically**: - `git ls-remote --tags` on the public repo (no token), - keep `vX.Y.Z` tags, pick the highest version (filtered/sorted in R for portability, not via git `--sort`/refspec which behaved inconsistently under `system2()`), - `pak::pak("git::…@")` — idempotent on the git ref, so re-runs keep the package unless a newer tag exists. All call sites now invoke the helper instead of a pinned version: - `.crow/build-all-versions.yaml` (primary + per-minor pass) - `.crow/build-all-versions-install-deps.yaml` - `.crow/process-updates.yaml` (primary + per-minor pass) - `.crow/weekly-rebuild-missing.yaml` - `.crow/archive-missed-packages.yaml` - `docker/build-one.Dockerfile` (ships the helper into the image; `ensure_bincraft` sources it) ## Effect Tag a new bincraft release → the next CI run / `just rebuild` picks it up automatically. No more pin edits, and no more "forgot to bump the Dockerfile" drift. ## Verified - Resolver returns the current latest tag (`v4.4.2`) via `git ls-remote` + R-side version sort. - All five workflow YAMLs parse; helper R parses; air/editorconfig clean. Note: this tracks the latest **tag**, so cutting a release is still the deliberate gate — CI won't pick up un-tagged main. Reviewed-on: https://git.devxy.io/devxy/build-cran-binaries/pulls/107 --- .crow/archive-missed-packages.yaml | 2 +- .crow/build-all-versions-install-deps.yaml | 2 +- .crow/build-all-versions.yaml | 4 +- .crow/process-updates.yaml | 4 +- .crow/weekly-rebuild-missing.yaml | 2 +- docker/build-one.Dockerfile | 4 +- local/install-bincraft.R | 52 ++++++++++++++++++++++ 7 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 local/install-bincraft.R diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index ca21aa5..71f808b 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -62,7 +62,7 @@ steps: GIT_USER: pat-s R_VERSION: 4.5.3 commands: - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2", dependencies = TRUE)' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' backend_options: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index 050db51..8649ca0 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -78,7 +78,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index f1e5f00..615d273 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -124,7 +124,7 @@ steps: # to a zero-length value and breaks every metadata query and the sysdeps # install). Pin bincraft here, exactly like the R-minor pass below. - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 - | @@ -136,7 +136,7 @@ steps: echo "=== R-minor-sensitive pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true done # archive missed packages; first arg is the codename (e.g. "alpine324"), diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 9ffcaed..3da41fd 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -195,7 +195,7 @@ steps: commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run @@ -211,7 +211,7 @@ steps: echo "=== R-minor-sensitive update pass under R $RV ===" LIB="/mnt/cache/R-pkgs-$RMINOR" mkdir -p "$LIB" - R_LIBS_USER="$LIB" "$(dirname "$RBIN")/R" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' || true + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true done - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 9996182..934c518 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -132,7 +132,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 1eb5ff7..5bc107a 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -16,6 +16,8 @@ ARG CACHEBUST WORKDIR /work COPY build-one.R /work/build-one.R +# Resolve and install the latest bincraft release dynamically (no hardcoded pin). +COPY install-bincraft.R /work/install-bincraft.R # Ship the patch registry so build-one.R's `patches = "local/patches"` resolves # (build context is `local/`, CWD is /work). COPY patches /work/local/patches @@ -43,7 +45,7 @@ RUN --mount=type=secret,id=b2_access,required=true \ echo "No working virtual display; building without xvfb" >&2; \ fi; \ run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \ - ensure_bincraft() { "$1" -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.4.2") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.4.2")'; }; \ + ensure_bincraft() { "$1" -q -e 'source("/work/install-bincraft.R")'; }; \ PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ seen=" $PRIMARY_MINOR "; \ prc=0; failed=""; \ diff --git a/local/install-bincraft.R b/local/install-bincraft.R new file mode 100644 index 0000000..4665b3b --- /dev/null +++ b/local/install-bincraft.R @@ -0,0 +1,52 @@ +#!/usr/bin/env Rscript + +# Install the latest tagged bincraft release, resolved dynamically, so the CI +# workflows and the build-one image never pin a hardcoded version (no more +# editing `@vX.Y.Z` in many places on every release). +# +# Run with the R whose library should receive bincraft: +# Rscript local/install-bincraft.R +# or, to target a specific R from a shell loop: +# "$RBIN" -q -e 'source("local/install-bincraft.R")' +# +# How it works: list the remote tags with `git ls-remote` (no token needed for +# the public repo), keep the `vX.Y.Z` release tags, pick the highest version, +# and install it with pak. pak is idempotent on the git ref, so re-running keeps +# the package when it is already current and only updates when a newer tag ships. +# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is +# identical across git versions and `system2()` argument handling. + +repo_url <- Sys.getenv( + "BINCRAFT_GIT_URL", + unset = "https://codefloe.com/rpkgs/bincraft.git" +) + +# GIT_TERMINAL_PROMPT=0 keeps a non-interactive run from hanging on auth. +refs <- system2( + "git", + c("ls-remote", "--tags", repo_url), + stdout = TRUE, + stderr = FALSE, + env = "GIT_TERMINAL_PROMPT=0" +) +tags <- sub(".*refs/tags/", "", refs) +tags <- tags[!grepl("\\^\\{\\}$", tags)] # drop dereferenced "...^{}" lines +tags <- grep("^v[0-9]", tags, value = TRUE) # only vX.Y.Z release tags +if (length(tags) == 0L) { + stop( + "Could not resolve any bincraft release tag from ", + repo_url, + call. = FALSE + ) +} +latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][ + 1L +] + +message(sprintf("Installing latest bincraft release: %s", latest)) +pak::pak(sprintf("git::%s@%s", repo_url, latest)) +message(sprintf( + "bincraft %s installed (%s)", + as.character(utils::packageVersion("bincraft")), + latest +))