build-cran-binaries/local/build-all.R

sink(stdout(), type = "message")
options(crayon.enabled = TRUE, future.globals.onReference = NULL)
source(file.path("local", "r-minor-helpers.R"))

args <- commandArgs(trailingOnly = TRUE)
parsed <- parse_build_args(args)
split_into <- parsed$split_into
split_index <- parsed$split_index
ncpus <- parsed$ncpus
sensitive_only <- parsed$sensitive_only
options(Ncpus = ncpus)

# Load bincraft eagerly to avoid lazy-load memory spike during first build call
library(bincraft, quietly = TRUE)
library(future)
plan("sequential")

# The install-deps step precomputes the package snapshot into /mnt/cache, but
# that volume is per-agent: a job landing on a fresh agent (or racing
# install-deps) finds it empty. Recompute the snapshot here when any part is
# missing, so the first job on an agent repopulates the cache for the jobs that
# follow; concurrent jobs that also miss simply redo the work. Write via a
# temp file + atomic rename so a concurrent reader never sees a half-written rds.
package_cache_files <- c(
  "/mnt/cache/packages/pkgs_to_build.rds",
  "/mnt/cache/packages/r_minor_sensitive_pkgs.rds",
  "/mnt/cache/packages/s3_cache.rds"
)
# Existence is not freshness. The snapshot describes S3 and CRAN state at the
# moment it was written, and the volume is per-agent, so an agent that ran an
# earlier pipeline keeps serving that pipeline's answer forever. arm64/alpine324
# reported "0 remaining" for both 4.4 and 4.6 from a stale snapshot listing 43
# sensitive packages, while the install-deps step in the very same pipeline had
# just computed 6871 on another agent.
#
# Keyed on the pipeline when the CI exposes one, so a new pipeline recomputes
# once per agent and its shards then share the result. Off CI, or when no such
# variable is set, fall back to an age check.
snapshot_id_path <- "/mnt/cache/packages/snapshot.id"
snapshot_ttl_hours <- as.numeric(
  Sys.getenv("PACKAGE_SNAPSHOT_TTL_HOURS", unset = "2")
)
current_snapshot_id <- ""
for (v in c("CI_PIPELINE_NUMBER", "CI_BUILD_NUMBER", "CI_PIPELINE_ID")) {
  val <- Sys.getenv(v, unset = "")
  if (nzchar(val)) {
    current_snapshot_id <- paste(v, val, sep = "=")
    break
  }
}

snapshot_is_stale <- function() {
  if (!all(file.exists(package_cache_files))) {
    return(TRUE)
  }
  if (nzchar(current_snapshot_id)) {
    cached <- tryCatch(
      readLines(snapshot_id_path, warn = FALSE)[1L],
      error = function(e) NA_character_,
      warning = function(w) NA_character_
    )
    return(!identical(cached, current_snapshot_id))
  }
  age_hours <- as.numeric(
    difftime(Sys.time(), file.mtime(package_cache_files[1L]), units = "hours")
  )
  isTRUE(age_hours > snapshot_ttl_hours)
}

if (snapshot_is_stale()) {
  message(
    "Package snapshot missing or stale; recomputing via packages-to-build.R"
  )
  dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
  save_rds_atomic <- function(obj, path) {
    tmp <- paste0(path, ".tmp.", Sys.getpid())
    saveRDS(obj, tmp)
    file.rename(tmp, path)
  }
  source(file.path("local", "packages-to-build.R"))
  save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds")
  save_rds_atomic(
    pkgs[r_minor_sensitive == TRUE],
    "/mnt/cache/packages/r_minor_sensitive_pkgs.rds"
  )
  if (nzchar(current_snapshot_id)) {
    writeLines(current_snapshot_id, snapshot_id_path)
  }
  message("Package snapshot recomputed.")
}

pkgs <- if (sensitive_only) {
  readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
} else {
  readRDS("/mnt/cache/packages/pkgs_to_build.rds")
}
# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive)
if (is.null(pkgs$r_minor_sensitive)) {
  pkgs$r_minor_sensitive <- FALSE
}
sprintf(
  "Total# of remaining package versions: %s (sensitive_only=%s)",
  nrow(pkgs),
  sensitive_only
)

# Split into chunks for this worker
chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE))
chunk <- chunks[[split_index]]
sprintf("# of package versions for this job: %s", nrow(chunk))

# Exclude known problematic packages (single source of truth)
exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]]
chunk <- chunk[!chunk$Package %in% exclude, ]

# Skip package versions already attempted in a previous run (built or errored).
# pkgs_to_build.rds is a static snapshot from the install-deps step, so on a
# restart it still lists everything an interrupted run already produced. The
# metadata DB reflects that progress, so we re-derive the remaining set here.
# We exclude *all* attempted versions, not just successful ones: a previously
# errored version is skipped by build_binary_package() anyway, so leaving it in
# the chunk only makes the job cycle through it one-by-one for no benefit.
# Derive platform + arch from the running container, mirroring the codename ->
# platform mapping bincraft uses internally. The OS/OS_VERSION selectors are
# workflow-level CI variables that are not injected into the container
# environment, so Sys.getenv() would return "" and this pre-filter would query
# platform "-" and skip nothing.
codename <- bincraft::set_codename(NULL)
platform <- switch(
  codename,
  jammy = "ubuntu-2204",
  noble = "ubuntu-2404",
  resolute = "ubuntu-2604",
  rhel10 = "redhat-10",
  rhel9 = "redhat-9",
  rhel8 = "redhat-8",
  alpine320 = "alpine-320",
  alpine321 = "alpine-321",
  alpine322 = "alpine-322",
  alpine323 = "alpine-323",
  alpine324 = "alpine-324",
  alpine325 = "alpine-325",
  alpine326 = "alpine-326",
  NA_character_
)
local_machine <- Sys.info()[["machine"]]
arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64"
con <- DBI::dbConnect(
  RPostgres::Postgres(),
  dbname = "build_metadata",
  host = "r-binaries.devxy.io",
  port = 15432,
  user = "rpkgs",
  password = Sys.getenv("PGPASS"),
  sslmode = "require"
)
# Scope the skip to the R minor this pass is running under. `single_builds`
# records `r_version` per attempt, but querying without it made a non-primary
# pass skip everything the primary pass had already attempted under a different
# minor - so `--sensitive-only` under 4.6 skipped packages that had only ever
# been built for 4.5, and the per-minor slots never filled. That is why
# amd64/resolute served 4000 fewer packages to a 4.6 client than to a 4.5 one.
r_minor <- paste(
  R.version$major,
  strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L],
  sep = "."
)
built <- DBI::dbGetQuery(
  con,
  paste(
    "SELECT name, tag FROM single_builds",
    "WHERE platform = $1 AND arch = $2",
    "AND substring(r_version from '^[0-9]+[.][0-9]+') = $3"
  ),
  params = list(platform, arch, r_minor)
)
DBI::dbDisconnect(con)
before <- nrow(chunk)
chunk <- chunk[
  !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag),
]
sprintf(
  "Skipped %d package versions already attempted under R %s; %d remaining for this job",
  before - nrow(chunk),
  r_minor,
  nrow(chunk)
)

# Read pre-computed S3 listing from install-deps step
# This avoids loading s3fs/reticulate/Python in the build container,
# saving significant memory for the dependency-installer subprocesses
s3_cache_paths <- readRDS("/mnt/cache/packages/s3_cache.rds")

# The cache is stored as paths relative to the slot, so a pass can select the
# objects that belong to it. `build_binary_package()` compares basenames, which
# cannot distinguish `4.4/curl_1.0.tar.gz` from `curl_1.0.tar.gz`, so the choice
# has to be made here where the running R minor is known.
#
# Getting this wrong is expensive in both directions: hand it everything and a
# per-minor pass believes the flat slot's binaries are its own and builds
# nothing; hand it nothing and it rebuilds what it already has. amd64/resolute
# recompiled 8683 packages that way.
select_cache_for_pass <- function(paths, sensitive_only, r_minor) {
  # A cache written before #191 holds bare basenames. Filtering those by path
  # would select nothing and trigger a full rebuild, so use them as they are;
  # #190's staleness check replaces it on the next pipeline anyway.
  #
  # The format is read from the marker `packages-to-build.R` sets, not guessed
  # from the content: a slot-relative cache with no per-minor or Archive object
  # holds bare names too, and treating that as legacy would hand a per-minor
  # pass the flat slot's binaries and build nothing.
  if (!isTRUE(attr(paths, "slot_relative"))) {
    message("S3 cache is in the legacy basename format; using it unfiltered.")
    return(paths)
  }
  in_minor <- grepl(sprintf("^%s/", r_minor), paths)
  if (sensitive_only) {
    basename(paths[in_minor])
  } else {
    # The primary pass writes the flat slot. Archive/ counts as present there:
    # those are versions built and later superseded.
    basename(paths[!grepl("^[0-9]+\\.[0-9]+/", paths)])
  }
}

s3_cache <- select_cache_for_pass(s3_cache_paths, sensitive_only, r_minor)
sprintf(
  "S3 cache: %s files (%s of %s objects apply to this pass)",
  length(s3_cache),
  length(s3_cache),
  length(s3_cache_paths)
)

n <- nrow(chunk)
mapply(
  function(pkg, ver, sens, i) {
    cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
    bincraft::build_binary_package(
      pkg,
      tag = ver,
      is_r_minor_sensitive = isTRUE(sens),
      s3_endpoint = "https://s3.eu-central-003.backblazeb2.com",
      s3_region = "eu-central-003",
      s3_bucket = "devxy-rpkgs-binaries",
      s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
      s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
      s3_package_cache = s3_cache,
      metadata_db_host = "r-binaries.devxy.io",
      metadata_db_name = "build_metadata",
      metadata_db_table = "single_builds",
      metadata_db_user = "rpkgs",
      metadata_db_password = Sys.getenv("PGPASS"),
      metadata_db_sslmode = "require",
      metadata_db_port = 15432,
      archive = TRUE,
      patches = "local/patches",
      upload = TRUE,
      store_build_metadata = TRUE
    )
  },
  chunk$Package,
  chunk$Version,
  chunk$r_minor_sensitive,
  seq_len(n)
)