build-cran-binaries/.crow/backfill-flat-safety.yaml

# Record, per flat-slot object, whether it loads under every supported R minor.
#
# `union_index_records()` keeps a carried flat record when that object is
# recorded safe and drops it otherwise. Objects uploaded before that record
# existed have no verdict, so they are dropped: a full re-index removed roughly
# 2000 records per per-minor index (16398 across the 4.4 indexes, 20076 across
# 4.5, 15532 across 4.6), each one resolving to CRAN source for clients the
# routing serves correctly.
#
# This inspects what is already in the bucket and writes the verdict, which is
# what restores that coverage. Decided by symbols rather than heuristics: a
# binary fails at dyn.load() under minor M exactly when it references a symbol M
# does not export, and the volatile set is derived by diffing the R builds in
# the image.
#
# Re-running is cheap; recorded objects are skipped.
#
#   crow pipeline create devxy/build-cran-binaries --var flat_safety=all

variables:
  # A manual pipeline creation instantiates every file in .crow/, so the default
  # must match no matrix row.
  flat_safety:
    description: "Inspect target: a specific <os>-<arch>, 'all' for every slot, or 'none'."
    options:
      - none
      - all
      - alpine-323-amd64
      - alpine-323-arm64
      - alpine-324-amd64
      - alpine-324-arm64
      - redhat-8-amd64
      - redhat-8-arm64
      - redhat-9-amd64
      - redhat-9-arm64
      - redhat-10-amd64
      - redhat-10-arm64
      - ubuntu-2204-amd64
      - ubuntu-2204-arm64
      - ubuntu-2404-amd64
      - ubuntu-2404-arm64
      - ubuntu-2604-amd64
      - ubuntu-2604-arm64
    default: none
  FLAT_SAFETY_LIMIT:
    description: 'Inspect at most this many objects per slot (0 = no limit).'
    default: '0'

when:
  - event: manual
    evaluate: 'flat_safety == "all" || flat_safety == "${OS}-${ARCH}"'

skip_clone: true

labels:
  group: rpkgs-${ARCH}

matrix:
  include:
    - OS: alpine-323
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-323
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.23
    - OS: alpine-324
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: alpine-324
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: alpine:3.24
    - OS: redhat-8
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-8
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:8
    - OS: redhat-9
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-9
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: redhat:9
    - OS: redhat-10
      ARCH: amd64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: redhat-10
      ARCH: arm64
      R_VERSION: 4.5.3
      IMG: redhat:10
    - OS: ubuntu-2204
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2204
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:jammy
    - OS: ubuntu-2404
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2404
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:noble
    - OS: ubuntu-2604
      ARCH: amd64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute
    - OS: ubuntu-2604
      ARCH: arm64
      R_VERSION: 4.4.3
      IMG: ubuntu:resolute

steps:
  - name: 'Inspect flat objects and record safety'
    image: reg.devxy.io/rpkgs/build-env-${IMG}
    pull: true
    environment:
      OTEL_R_TRACES_EXPORTER: none
      OTEL_R_LOGS_EXPORTER: none
      OTEL_R_METRICS_EXPORTER: none
      RED_HAT_DEV_PW:
        from_secret: RED_HAT_DEV_PW
      B2_S3_ACCESS_KEY:
        from_secret: B2_S3_ACCESS_KEY
      B2_S3_SECRET_KEY:
        from_secret: B2_S3_SECRET_KEY
      PGPASS:
        from_secret: PGPASS
      REPO_RO_TOKEN:
        from_secret: REPO_RO_TOKEN
      # bincraft reads the DESCRIPTION of every new or replaced tarball from
      # the CRAN mirror on raw.githubusercontent.com, and authenticates with
      # this token when it is set. A slot whose published index cannot be read
      # is re-read in full, ~25k requests, which the anonymous rate limit would
      # cut short.
      GITHUB_PAT:
        from_secret: GITHUB_PAT
      GIT_USER: pat-s
      R_LIBS_USER: /tmp/R-pkgs
      FLAT_SAFETY_ARCH: ${ARCH}
      FLAT_SAFETY_LIMIT: ${FLAT_SAFETY_LIMIT}
    commands:
      - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
      - mkdir -p /tmp/R-pkgs
      - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
      - |
        set -eu
        LIMIT=""
        if [ "$$FLAT_SAFETY_LIMIT" != "0" ]; then
          LIMIT="$$FLAT_SAFETY_LIMIT"
        fi
        # The codename comes from the image's /etc/os-release, exactly as the
        # re-index step derives it, so it cannot drift from the matrix.
        CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e 'cat(bincraft::set_codename(NULL))')
        echo "slot: $$FLAT_SAFETY_ARCH/$$CODENAME"
        /opt/R/$R_VERSION/bin/Rscript local/backfill-flat-safety.R \
          "$$FLAT_SAFETY_ARCH" "$$CODENAME" $$LIMIT
    backend_options:
      docker:
        resources:
          requests:
            memory: 4Gi
            cpu: 1000m