r-builds/.crow/build.yaml

when:
  event: [cron, manual]

matrix:
  include:
    - PLATFORM: rhel-9
      PLATFORM_ID: el9
      ARCH: arm64
      ARCH_ID: aarch64
    - PLATFORM: rhel-9
      PLATFORM_ID: el9
      ARCH: amd64
      ARCH_ID: x86_64
    - PLATFORM: rhel-10
      PLATFORM_ID: el10
      ARCH: arm64
      ARCH_ID: aarch64
    - PLATFORM: rhel-10
      PLATFORM_ID: el10
      ARCH: amd64
      ARCH_ID: x86_64
    - PLATFORM: centos-8
      PLATFORM_ID: el8
      ARCH: arm64
      ARCH_ID: aarch64
    - PLATFORM: centos-8
      PLATFORM_ID: el8
      ARCH: amd64
      ARCH_ID: x86_64
    - PLATFORM: ubuntu-2204
      PLATFORM_ID: 2204
      ARCH: arm64
      ARCH_ID: arm64
    - PLATFORM: ubuntu-2204
      PLATFORM_ID: 2204
      ARCH: amd64
      ARCH_ID: amd64
    - PLATFORM: ubuntu-2404
      PLATFORM_ID: 2404
      ARCH: arm64
      ARCH_ID: arm64
    - PLATFORM: ubuntu-2404
      PLATFORM_ID: 2404
      ARCH: amd64
      ARCH_ID: amd64
    - PLATFORM: ubuntu-2604
      PLATFORM_ID: 2604
      ARCH: arm64
      ARCH_ID: arm64
    - PLATFORM: ubuntu-2604
      PLATFORM_ID: 2604
      ARCH: amd64
      ARCH_ID: amd64
    # - PLATFORM: alpine-320
    #   PLATFORM_ID: alpine320
    #   ARCH: arm64
    #   ARCH_ID: aarch64
    #   INSTANCE_TYPE: cax31
    - PLATFORM: alpine-323
      PLATFORM_ID: alpine323
      ARCH: arm64
      ARCH_ID: aarch64
    - PLATFORM: alpine-323
      PLATFORM_ID: alpine323
      ARCH: amd64
      ARCH_ID: x86_64
    - PLATFORM: alpine-322
      PLATFORM_ID: alpine322
      ARCH: arm64
      ARCH_ID: aarch64
    - PLATFORM: alpine-322
      PLATFORM_ID: alpine322
      ARCH: amd64
      ARCH_ID: x86_64

steps:
  # Ensures that we're always using the latest available upstream build chain
  # - name: Pull upstream
  #   image: reg.devxy.io/docker.io/library/alpine:3.20
  #   commands:
  #     - apk add -q --no-cache git
  #     - git remote add upstream https://github.com/cynkra/r-builds.git
  #     - git config --global user.email "you@example.com"
  #     - git config --global user.name "Your Name"
  #     - git config pull.ff only
  #     - git fetch upstream
  #     - git merge upstream/main --allow-unrelated-histories
  #.

  - name: Net debug
    image: reg.devxy.io/docker.io/library/alpine:3.23
    privileged: true
    commands:
      - ip link set dev eth0 mtu 1280 2>/dev/null || true
      - echo "=== pod interface ==="
      - cat /sys/class/net/eth0/mtu
      - ip -o link show eth0
      - echo "=== route ==="
      - ip route
      - echo "=== resolv.conf ==="
      - cat /etc/resolv.conf
      - for i in 1 2 3; do apk add -q --no-cache iputils && break; sleep 5; done || echo "apk failed; falling back to busybox ping (no -M do)"
      - |
          echo "=== PMTUD probes (DF=1) to dl-cdn.alpinelinux.org ==="
          for sz in 1472 1392 1352 1252 1200; do
            echo "--- payload=$sz (wire ~$((sz+28))) ---"
            ping -c 2 -W 3 -M do -s $sz dl-cdn.alpinelinux.org || echo "(size=$sz failed)"
          done
      - ping -c 3 -W 5 dl-cdn.alpinelinux.org || true
      - wget --tries=1 --timeout=10 -O /dev/null -nv https://dl-cdn.alpinelinux.org/ 2>&1 || true
    backend_options:
      kubernetes:
        nodeSelector:
          kubernetes.io/arch: "${ARCH}"


  - name: Get R versions to build
    image: reg.devxy.io/docker.io/library/alpine:3.23
    privileged: true
    commands:
      - ip link set dev eth0 mtu 1280 2>/dev/null || true
      - for i in 1 2 3 4 5; do apk add -q --no-cache curl jq && break; sleep 5; done
      # Latest patch release of each of the last 4 minor R versions (e.g. 4.6.0, 4.5.3, 4.4.3, 4.3.3).
      - |
          curl -sf https://cdn.posit.co/r/versions.json | jq -r '
            .r_versions
            | map(select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$")))
            | group_by(split(".")[0:2] | join("."))
            | map(max_by(split(".") | map(tonumber)))
            | sort_by(split(".") | map(tonumber))
            | reverse
            | .[0:4]
            | .[]' > r-versions-to-build.txt
      # To pin specific versions for testing, overwrite the file, e.g.:
      # - printf '4.1.3\n' > r-versions-to-build.txt
      - cat r-versions-to-build.txt
    backend_options:
      kubernetes:
        resources:
          requests:
            memory: 50Mi
            cpu: 200m
          limits:
            memory: 200Mi
            cpu: 500m
        nodeSelector:
          kubernetes.io/arch: "${ARCH}"


  - name: Check which R versions already exist in s3
    image: reg.devxy.io/docker.io/library/alpine:3.23
    privileged: true
    commands:
      - ip link set dev eth0 mtu 1280 2>/dev/null || true
      - for i in 1 2 3 4 5; do apk add -q --no-cache curl && break; sleep 5; done
      - |
          LISTING=$(curl -s "https://s3.eu-central-003.backblazeb2.com/devxy-r-builds?prefix=${PLATFORM_ID}/" | \
            grep -oE '<Key>[^<]+</Key>' | sed 's/<[^>]*>//g')
          : > r-versions-existing.txt
          for VERSION in $(cat r-versions-to-build.txt); do
            if printf '%s\n' "$LISTING" | \
              awk -v arch="${ARCH_ID}" -v ver="$VERSION" 'index($0, arch) && index($0, ver) { found=1 } END { exit !found }'; then
              echo "$VERSION" >> r-versions-existing.txt
            fi
          done
      - echo "Already present in s3:"; cat r-versions-existing.txt
    backend_options:
      kubernetes:
        nodeSelector:
          kubernetes.io/arch: "${ARCH}"


  # ref: https://github.com/woodpecker-ci/woodpecker/discussions/2721
  - name: Build
    image: reg.devxy.io/docker.io/library/alpine:3.23
    privileged: true
    environment:
      DOCKER_HOST: tcp://docker:2375
    commands: |
      ip link set dev eth0 mtu 1280 2>/dev/null || true
      # docker info
      TO_BUILD=""
      for VERSION in $(cat r-versions-to-build.txt); do
        if grep -qxF "$VERSION" r-versions-existing.txt; then
          echo "R $VERSION already exists for ${PLATFORM} (${ARCH_ID}), skipping"
        else
          TO_BUILD="$TO_BUILD $VERSION"
        fi
      done
      if [ -n "$TO_BUILD" ]; then
        for i in 1 2 3 4 5; do apk add -q --no-cache make just docker-compose && break; sleep 5; done
        for VERSION in $TO_BUILD; do
          echo "Building R $VERSION for ${PLATFORM}"
          just build-r-${PLATFORM} "$VERSION"
        done
      fi
    backend_options:
      kubernetes:
        resources:
          requests:
            memory: 200Mi
            cpu: 500m
            ephemeral-storage: 2Gi
          limits:
            memory: 1000Mi
            cpu: 4000m
            ephemeral-storage: 4Gi
        nodeSelector:
          kubernetes.io/arch: "${ARCH}"


  - name: Upload to Backblaze S3
    image: reg.devxy.io/docker.io/woodpeckerci/plugin-s3:1.5.2
    pull: true
    settings:
      bucket: devxy-r-builds
      strip_prefix: /tmp/${PLATFORM}/
      source: /tmp/${PLATFORM}/*
      target: ${PLATFORM_ID}/
      region: eu-central-003
      acl: public-read
      endpoint: https://s3.eu-central-003.backblazeb2.com
      access_key:
        from_secret: B2_S3_ACCESS_KEY_ID
      secret_key:
        from_secret: B2_S3_SECRET_KEY
      overwrite: false
    volumes:
      - ci-tmp-rwx:/tmp # agent on k8s
      # - /tmp:/tmp # agent on VM
    backend_options:
      kubernetes:
        resources:
          requests:
            memory: 100Mi
            cpu: 200m
          limits:
            memory: 1000Mi
            cpu: 1000m
        nodeSelector:
          kubernetes.io/arch: "${ARCH}"


services:
  docker:
    environment:
      DOCKER_TLS_CERTDIR: ""
    image: reg.devxy.io/docker.io/library/docker:29-dind
    commands:
      - ip link set dev eth0 mtu 1280 2>/dev/null || true
      - mkdir -p /tmp/${PLATFORM}/${ARCH}
      - dockerd --tls=false --mtu 1280 --host=tcp://0.0.0.0:2375
    privileged: true
    ports:
      - 2375
    volumes:
      - ci-tmp-rwx:/tmp # agent on k8s
    backend_options:
      kubernetes:
        nodeSelector:
          kubernetes.io/arch: "${ARCH}"