diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile new file mode 100644 index 0000000..ddfd069 --- /dev/null +++ b/docker/build-one.Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1 +# Targeted (re)build of specific package versions, executed on a remote buildx +# builder. The build's effect is the S3 upload performed by build-one.R; no +# image is kept (the justfile uses --output type=cacheonly). CACHEBUST forces +# the RUN to re-execute on every invocation. +# +# Build context is `local/` (see the `rebuild` recipe in the justfile). +ARG OS +ARG OS_VERSION +FROM reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION} + +ARG R_VERSION=4.5.3 +ARG PACKAGE +ARG VERSIONS +ARG CACHEBUST + +WORKDIR /work +COPY build-one.R /work/build-one.R + +RUN --mount=type=secret,id=b2_access,required=true \ + --mount=type=secret,id=b2_secret,required=true \ + --mount=type=secret,id=pgpass,required=true \ + --mount=type=secret,id=github_pat,required=false \ + export B2_S3_ACCESS_KEY="$(cat /run/secrets/b2_access)" && \ + export B2_S3_SECRET_KEY="$(cat /run/secrets/b2_secret)" && \ + export PGPASS="$(cat /run/secrets/pgpass)" && \ + export GITHUB_PAT="$(cat /run/secrets/github_pat 2>/dev/null || true)" && \ + /opt/R/${R_VERSION}/bin/R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE) || packageVersion("bincraft") != "4.2.0") pak::pak("git::https://codefloe.com/rpkgs/bincraft.git@v4.2.0")' && \ + XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); \ + XVFB_ARGS=""; \ + if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi; \ + $XVFB $XVFB_ARGS -- /opt/R/${R_VERSION}/bin/Rscript /work/build-one.R "${PACKAGE}" ${VERSIONS} diff --git a/justfile b/justfile new file mode 100644 index 0000000..a475505 --- /dev/null +++ b/justfile @@ -0,0 +1,62 @@ +# Local helpers for build-cran-binaries. +# +# `rebuild` (re)builds specific versions of a single package on a given OS/arch +# by dispatching to a remote buildx builder (the build runs there, not locally). +# Sensitivity is auto-detected: risky packages land in the per-minor slot +# (contrib//), everything else in the generic slot; the touched index is +# refreshed so the result is immediately servable. +# +# Prerequisites: +# - buildx builders named `artemis` (amd64) and `gaia` (arm64), e.g. +# docker buildx create --name artemis --node artemis ssh:// +# docker buildx create --name gaia --node gaia ssh:// +# - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional) +# +# Overridable (env or `just VAR=… rebuild …`): +# R_VERSION (default 4.5.3) — selects the R minor → the per-minor slot +# AMD64_BUILDER / ARM64_BUILDER — buildx builder names +# +# Examples: +# just rebuild alpine 3.23 amd64 rlang 1.1.4 1.1.3 +# just rebuild redhat 10 arm64 data.table 1.15.4 +# R_VERSION=4.4.3 just rebuild ubuntu noble amd64 Rcpp 1.0.12 + +r_version := env_var_or_default("R_VERSION", "4.5.3") +amd64_builder := env_var_or_default("AMD64_BUILDER", "artemis") +arm64_builder := env_var_or_default("ARM64_BUILDER", "gaia") + +# (re)build PACKAGE at one or more VERSIONS on OS/TAG/ARCH via a remote builder +rebuild os tag arch package *versions: + #!/usr/bin/env bash + set -euo pipefail + if [ -z "{{ versions }}" ]; then + echo "error: provide at least one version, e.g. just rebuild alpine 3.23 amd64 rlang 1.1.4" >&2 + exit 1 + fi + case "{{ arch }}" in + amd64) builder="{{ amd64_builder }}" ;; + arm64) builder="{{ arm64_builder }}" ;; + *) echo "error: arch must be 'amd64' or 'arm64'" >&2; exit 1 ;; + esac + : "${B2_S3_ACCESS_KEY:?set B2_S3_ACCESS_KEY in your environment}" + : "${B2_S3_SECRET_KEY:?set B2_S3_SECRET_KEY in your environment}" + : "${PGPASS:?set PGPASS in your environment}" + : "${GITHUB_PAT:=}" + echo "Dispatching build of {{ package }} ({{ versions }}) on {{ os }}:{{ tag }}/{{ arch }} (R {{ r_version }}) to builder '$builder'" + docker buildx build \ + --builder "$builder" \ + --platform "linux/{{ arch }}" \ + --no-cache \ + --output type=cacheonly \ + --secret id=b2_access,env=B2_S3_ACCESS_KEY \ + --secret id=b2_secret,env=B2_S3_SECRET_KEY \ + --secret id=pgpass,env=PGPASS \ + --secret id=github_pat,env=GITHUB_PAT \ + --build-arg OS="{{ os }}" \ + --build-arg OS_VERSION="{{ tag }}" \ + --build-arg R_VERSION="{{ r_version }}" \ + --build-arg PACKAGE="{{ package }}" \ + --build-arg VERSIONS="{{ versions }}" \ + --build-arg CACHEBUST="$(date +%s)" \ + -f docker/build-one.Dockerfile \ + local diff --git a/local/build-one.R b/local/build-one.R new file mode 100644 index 0000000..d67a206 --- /dev/null +++ b/local/build-one.R @@ -0,0 +1,126 @@ +# Targeted (re)build of specific versions of a single package. +# Invoked inside a build-env container (see docker/build-one.Dockerfile). +# Usage: build-one.R [ ...] +# Sensitivity is auto-detected per version via bincraft's ABI classifier: +# risky packages go to the per-minor slot, everything else to the generic slot. + +sink(stdout(), type = "message") +options(crayon.enabled = TRUE, future.globals.onReference = NULL) + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2L) { + stop("usage: build-one.R [ ...]", call. = FALSE) +} +package <- args[1L] +versions <- args[-1L] + +library(bincraft, quietly = TRUE) + +s3 <- list( + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") +) + +# Clone the CRAN source for a version and ask the ABI classifier whether it +# must be rebuilt per R minor. Fails safe to TRUE so a possibly-fragile binary +# is never served from the cross-minor generic slot by mistake. +classify <- function(pkg, ver) { + dest <- file.path(tempdir(), sprintf("classify_%s_%s", pkg, ver)) + on.exit(unlink(dest, recursive = TRUE, force = TRUE), add = TRUE) + tryCatch( + { + system2( + "git", + c( + "clone", + "-q", + "--branch", + ver, + sprintf("https://github.com/cran/%s", pkg), + dest + ) + ) + isTRUE(as.logical(bincraft::needs_per_minor_recompile(dest))) + }, + error = function(e) { + message(sprintf( + "classify failed for %s %s: %s; treating as r-minor-sensitive", + pkg, + ver, + conditionMessage(e) + )) + TRUE + } + ) +} + +minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) +touched_generic <- FALSE +touched_minor <- FALSE + +for (ver in versions) { + sensitive <- classify(package, ver) + cat(sprintf( + "Building %s %s (r_minor_sensitive=%s, R %s)\n", + package, + ver, + sensitive, + minor + )) + bincraft::build_binary_package( + package, + tag = ver, + is_r_minor_sensitive = sensitive, + force = TRUE, + upload = TRUE, + archive = TRUE, + store_build_metadata = TRUE, + s3_endpoint = s3$s3_endpoint, + s3_region = s3$s3_region, + s3_bucket = s3$s3_bucket, + s3_access_key_id = s3$s3_access_key_id, + s3_secret_access_key = s3$s3_secret_access_key, + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432 + ) + if (sensitive) touched_minor <- TRUE else touched_generic <- TRUE +} + +# Refresh the PACKAGES index for each slot we wrote to, so the (re)built binary +# is immediately resolvable by clients. +codename <- bincraft::set_codename(NULL) +if (touched_generic) { + cat("Refreshing generic index\n") + bincraft::upload_package_index( + codename = codename, + s3_endpoint = s3$s3_endpoint, + s3_region = s3$s3_region, + s3_bucket = s3$s3_bucket, + s3_access_key_id = s3$s3_access_key_id, + s3_secret_access_key = s3$s3_secret_access_key + ) +} +if (touched_minor) { + cat(sprintf("Refreshing per-minor index %s\n", minor)) + bincraft::upload_package_index( + codename = codename, + r_minor = minor, + s3_endpoint = s3$s3_endpoint, + s3_region = s3$s3_region, + s3_bucket = s3$s3_bucket, + s3_access_key_id = s3$s3_access_key_id, + s3_secret_access_key = s3$s3_secret_access_key + ) +}