diff --git a/.crow/alpine-322-amd64-install-deps.yaml b/.crow/alpine-322-amd64-install-deps.yaml deleted file mode 100644 index 7e95691..0000000 --- a/.crow/alpine-322-amd64-install-deps.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "alpine-322-amd64"' - - event: manual - evaluate: 'build == "all-amd64"' - -skip_clone: true - -steps: - - name: 'Install common R deps' - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:noble-4.4.3 - pull: true - environment: - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - volumes: - - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - mkdir -p /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraftR - - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - backend_options: - kubernetes: - resources: - requests: - memory: 1000Mi - cpu: 1000m - limits: - memory: 1500Mi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/alpine-322-arm64-install-deps.yaml b/.crow/alpine-322-arm64-install-deps.yaml deleted file mode 100644 index 38109f2..0000000 --- a/.crow/alpine-322-arm64-install-deps.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "alpine-322-arm64"' - - event: manual - evaluate: 'build == "all-arm64"' - -skip_clone: true - -steps: - - name: 'Install common R deps' - image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.22-4.5.0 - pull: true - environment: - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - volumes: - - arm64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - mkdir -p /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraftR - - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - backend_options: - kubernetes: - resources: - requests: - memory: 1000Mi - cpu: 1000m - limits: - memory: 1500Mi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: cax41 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml new file mode 100644 index 0000000..bd127d6 --- /dev/null +++ b/.crow/archive-missed-packages.yaml @@ -0,0 +1,77 @@ +### Required extra vars +# woodpecker-cli pipeline create --var task=archive-missed-packages --branch=main --log-level=info 7 +when: + - event: manual + evaluate: 'task == "archive-missed-packages"' + - event: cron + cron: archive-missed-packages + +matrix: + include: + - CODENAME: jammy + ARCH: amd64 + - CODENAME: jammy + ARCH: arm64 + - CODENAME: noble + ARCH: amd64 + - CODENAME: noble + ARCH: arm64 + - CODENAME: redhat-8 + ARCH: amd64 + - CODENAME: redhat-8 + ARCH: arm64 + - CODENAME: redhat-9 + ARCH: amd64 + - CODENAME: redhat-9 + ARCH: arm64 + - CODENAME: redhat-10 + ARCH: amd64 + - CODENAME: redhat-10 + ARCH: arm64 + - CODENAME: alpine322 + ARCH: amd64 + - CODENAME: alpine322 + ARCH: arm64 + - CODENAME: alpine323 + ARCH: amd64 + - CODENAME: alpine323 + ARCH: arm64 + +steps: + - name: 'Archive missed packages' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + R_VERSION: 4.5.3 + commands: + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e 'bincraft::process_unarchived_pkgs(Sys.getenv("CODENAME"), Sys.getenv("ARCH"), s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), workers = 2L)' + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 2000m + limits: + memory: 1200Mi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml new file mode 100644 index 0000000..7c33c5b --- /dev/null +++ b/.crow/auto-apply-patches.yaml @@ -0,0 +1,76 @@ +# Auto-apply registry patches (issue #115, step 3 automation). +# Classifies `single_builds` failures and, for the top-N auto-proposable +# candidates by failure volume, writes the registry entries onto the reused +# `auto/registry-patch-proposals` branch and opens/updates a single PR. +# Nothing merges: the `trial-build-registry` pipeline is the merge gate, and a +# human reviews the PR. Novel source diffs / unknown signatures are never +# proposed. Global across platforms, so a single job -- no matrix. +# +# FORGEJO_TOKEN is used for both the branch push and opening the PR (no separate +# write-scoped secret needed). Register the `auto-apply-patches` cron in the crow +# UI, or run manually: +# crow pipeline create --branch main \ +# --var auto_apply_patches=true devxy/build-cran-binaries +# +# The gate variable is `auto_apply_patches`, named after the pipeline: a manual +# run instantiates every pipeline in `.crow/`, so one without its own gate runs +# on *any* manual trigger in this repo. This one pushes a branch and opens a PR, +# so it must stay off unless it is what was asked for. +variables: + auto_apply_patches: + description: 'Run the auto-patch proposer. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' + patch_limit: + description: 'Max candidates to propose per run (top by failure volume).' + default: '10' + +when: + - event: manual + evaluate: 'auto_apply_patches == "true"' + - event: cron + cron: auto-apply-patches + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Auto-apply registry patches' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + GIT_USER: devxy-bot + GIT_EMAIL: bot@devxy.io + PATCH_LIMIT: ${patch_limit} + R_VERSION: 4.5.3 + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite + - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 2000m + limits: + memory: 2Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml new file mode 100644 index 0000000..2a6d9c2 --- /dev/null +++ b/.crow/build-all-versions-install-deps.yaml @@ -0,0 +1,97 @@ +# Consolidated install-deps for build-all (both arches in one file). +# Triggered together with build-all-versions; `target_arch` routes the matrix. +# Variables are declared so the manual-run form exposes them (crow #1165); +# they are merged with build-all-versions' identical declarations. +variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. + target_arch: + description: 'Architecture to build, or "none" to run nothing.' + options: + - none + - amd64 + - arm64 + default: none + OS: + description: 'Base OS image name.' + options: + - alpine + - redhat + - ubuntu + default: alpine + OS_VERSION: + description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' + options: + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + default: '3.24' + R_VERSION: + description: 'Primary R version under /opt/R.' + options: + - 4.5.3 + - 4.4.3 + default: 4.5.3 + +when: + - event: manual + evaluate: 'target_arch == "${ARCH}"' + +skip_clone: true + +labels: + platform: linux/${ARCH} + group: rpkgs-${ARCH} + +matrix: + include: + - ARCH: amd64 + - ARCH: arm64 + +steps: + - name: 'Install deps and bincraft' + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + R_LIBS_USER: /mnt/cache/R-pkgs + # Keep uvr's downloads and extracted-package entries on the persistent + # volume instead of the container-local ~/.uvr default. + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + CCACHE_DIR: /mnt/cache/ccache + volumes: + - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache + commands: + # one-time full wipe to fix corrupted .so files from previous failed builds + # - rm -rf /mnt/cache/R-pkgs + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + # Pin the same bincraft version the build steps use, so the precomputed + # snapshot and the per-agent library stay consistent across the pipeline. + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 1000m diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml new file mode 100644 index 0000000..fb1229d --- /dev/null +++ b/.crow/build-all-versions.yaml @@ -0,0 +1,201 @@ +### Manual full rebuild of all CRAN binaries for one OS/arch. +# Set the variables below in the manual-run form (or via --var on the CLI). +# `target_arch` routes the matrix; OS/OS_VERSION/R_VERSION select the platform +# image and cache volume. Placement is via the group label (rpkgs-amd64/rpkgs-arm64). +# Skip list lives in local/excluded-packages.json (read by local/build-all.R). +variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. + target_arch: + description: 'Architecture to build, or "none" to run nothing.' + options: + - none + - amd64 + - arm64 + default: none + OS: + description: 'Base OS image name.' + options: + - alpine + - redhat + - ubuntu + default: alpine + OS_VERSION: + description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' + options: + - '3.22' + - '3.23' + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + - 'resolute' + default: '3.24' + R_VERSION: + description: 'Primary R version under /opt/R.' + options: + - 4.5.3 + - 4.4.3 + default: 4.5.3 + +when: + - event: manual + evaluate: 'target_arch == "${ARCH}"' + +skip_clone: true + +labels: + platform: linux/${ARCH} + group: rpkgs-${ARCH} + +# Empty UVR_CACHE_DIR/UVR_PACKAGES_DIR fall back to uvr's container-local +# ~/.uvr defaults; amd64 deliberately does not persist them (as with the +# pkgcache dir it replaces), arm64 does. +matrix: + include: + - ARCH: amd64 + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' + SPLIT_INTO: 4 + SPLIT_INDEX: 1 + - ARCH: amd64 + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' + SPLIT_INTO: 4 + SPLIT_INDEX: 2 + - ARCH: amd64 + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' + SPLIT_INTO: 4 + SPLIT_INDEX: 3 + - ARCH: amd64 + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' + SPLIT_INTO: 4 + SPLIT_INDEX: 4 + - ARCH: arm64 + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + SPLIT_INTO: 4 + SPLIT_INDEX: 1 + - ARCH: arm64 + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + SPLIT_INTO: 4 + SPLIT_INDEX: 2 + - ARCH: arm64 + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + SPLIT_INTO: 4 + SPLIT_INDEX: 3 + - ARCH: arm64 + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + SPLIT_INTO: 4 + SPLIT_INDEX: 4 + +depends_on: + - build-all-versions-install-deps + +steps: + - name: 'Build binaries' + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + # set the location of uvr's caches, which persist the R package + # dependencies needed to install the packages themselves + UVR_CACHE_DIR: ${UVR_CACHE_DIR} + UVR_PACKAGES_DIR: ${UVR_PACKAGES_DIR} + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + NCPUS: 2 + volumes: + - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # The primary pass must not rely on build-all-versions-install-deps having + # run on *this* agent: depends_on only orders the steps, but the cache + # volume is per-agent, so a job landing on an agent where install-deps did + # not run would otherwise use a stale bincraft (which resolves `platform` + # to a zero-length value and breaks every metadata query and the sysdeps + # install). Pin bincraft here, exactly like the R-minor pass below. + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/build-all.R $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/[0-9]*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive pass under R $RV ===" + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- "$(dirname "$RBIN")/Rscript" local/build-all.R --sensitive-only $SPLIT_INTO $SPLIT_INDEX $NCPUS 2>&1 || true + done + # archive missed packages; first arg is the codename (e.g. "alpine324"), + # derived via bincraft like the upload step, not paste(OS, OS_VERSION). + - /opt/R/$R_VERSION/bin/R -q -e "bincraft::process_unarchived_pkgs(bincraft::set_codename(NULL), Sys.getenv('ARCH'), s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), workers = $NCPUS)" + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m + - name: 'Upload package indexes' + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + R_LIBS_USER: /mnt/cache/R-pkgs + volumes: + - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache + commands: + - | + CODENAME=$(/opt/R/$R_VERSION/bin/Rscript -e "cat(bincraft::set_codename(NULL))") + /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "bincraft::upload_package_index(codename = '$CODENAME', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m diff --git a/.crow/build-alpine-322-amd64-1.yaml b/.crow/build-alpine-322-amd64-1.yaml deleted file mode 100644 index 962c663..0000000 --- a/.crow/build-alpine-322-amd64-1.yaml +++ /dev/null @@ -1,88 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "alpine-322-amd64"' - - event: manual - evaluate: 'build == "alpine-322-amd64-1"' - - event: manual - evaluate: 'build == "all-amd64"' - -skip_clone: true - -depends_on: - - alpine-322-amd64-install-deps - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.22-4.5.0 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - BLOCK_START: 10001 - BLOCK_END: 12000 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - volumes: - - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" - # hanging: 'doBy', 'IDPmisc', 'frailtypack', 'afex' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - BUILD_INDEX=$(cat build_index.txt) - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-amd64-2.yaml b/.crow/build-alpine-322-amd64-2.yaml deleted file mode 100644 index c622c7a..0000000 --- a/.crow/build-alpine-322-amd64-2.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "alpine-322-amd64"' - - event: manual - evaluate: 'build == "alpine-322-amd64-1"' - - event: manual - evaluate: 'build == "all-amd64"' - -skip_clone: true - -depends_on: - - alpine-322-amd64-install-deps - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.22-4.5.0 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - BLOCK_START: 12001 - BLOCK_END: 14000 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - volumes: - - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - BUILD_INDEX=$(cat build_index.txt) - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-amd64-3.yaml b/.crow/build-alpine-322-amd64-3.yaml deleted file mode 100644 index b9a07b6..0000000 --- a/.crow/build-alpine-322-amd64-3.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "alpine-322-amd64"' - - event: manual - evaluate: 'build == "alpine-322-amd64-1"' - - event: manual - evaluate: 'build == "all-amd64"' - -skip_clone: true - -depends_on: - - alpine-322-amd64-install-deps - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - BLOCK_START: 14001 - BLOCK_END: 16000 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - volumes: - - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - BUILD_INDEX=$(cat build_index.txt) - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-amd64-4.yaml b/.crow/build-alpine-322-amd64-4.yaml deleted file mode 100644 index 2fed165..0000000 --- a/.crow/build-alpine-322-amd64-4.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "alpine-322-amd64"' - - event: manual - evaluate: 'build == "alpine-322-amd64-1"' - - event: manual - evaluate: 'build == "all-amd64"' - -skip_clone: true - -depends_on: - - alpine-322-amd64-install-deps - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - BLOCK_START: 16001 - BLOCK_END: 18000 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - volumes: - - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - BUILD_INDEX=$(cat build_index.txt) - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-amd64-5.yaml b/.crow/build-alpine-322-amd64-5.yaml deleted file mode 100644 index d9fc5bd..0000000 --- a/.crow/build-alpine-322-amd64-5.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "alpine-322-amd64"' - - event: manual - evaluate: 'build == "alpine-322-amd64-1"' - - event: manual - evaluate: 'build == "all-amd64"' - -skip_clone: true - -depends_on: - - alpine-322-amd64-install-deps - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - BLOCK_START: 18001 - BLOCK_END: 20000 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - volumes: - - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - BUILD_INDEX=$(cat build_index.txt) - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-arm64-1.yaml b/.crow/build-alpine-322-arm64-1.yaml deleted file mode 100644 index 0e99531..0000000 --- a/.crow/build-alpine-322-arm64-1.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "alpine-322-arm64"' - - event: manual - evaluate: 'build == "alpine-322-arm64-1"' - - event: manual - evaluate: 'build == "all-arm64"' - -skip_clone: true - -depends_on: - - alpine-322-arm64-install-deps - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.22-4.5.0 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - BLOCK_START: 6601 - BLOCK_END: 9000 - ARCH: arm64 - NCPUS: 2 - STRATEGY: sequential - volumes: - - arm64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: cax41 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - BUILD_INDEX=$(cat build_index.txt) - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-arm64-2.yaml b/.crow/build-alpine-322-arm64-2.yaml deleted file mode 100644 index 73c642a..0000000 --- a/.crow/build-alpine-322-arm64-2.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "alpine-322-arm64"' - - event: manual - evaluate: 'build == "alpine-322-arm64-1"' - - event: manual - evaluate: 'build == "all-arm64"' - -skip_clone: true - -depends_on: - - alpine-322-arm64-install-deps - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.22-4.5.0 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - BLOCK_START: 9001 - BLOCK_END: 12000 - ARCH: arm64 - NCPUS: 2 - STRATEGY: sequential - volumes: - - arm64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: cax41 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - BUILD_INDEX=$(cat build_index.txt) - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-arm64-3.yaml b/.crow/build-alpine-322-arm64-3.yaml deleted file mode 100644 index 739d463..0000000 --- a/.crow/build-alpine-322-arm64-3.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "alpine-322-arm64"' - - event: manual - evaluate: 'build == "alpine-322-arm64-1"' - - event: manual - evaluate: 'build == "all-arm64"' - -skip_clone: true - -depends_on: - - alpine-322-arm64-install-deps - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.22-4.5.0 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - BLOCK_START: 12001 - BLOCK_END: 15000 - ARCH: arm64 - NCPUS: 2 - STRATEGY: sequential - volumes: - - arm64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - limits: - memory: 20Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: cax41 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - BUILD_INDEX=$(cat build_index.txt) - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-redhat-8-amd64-single.yaml b/.crow/build-redhat-8-amd64-single.yaml deleted file mode 100644 index 7b83187..0000000 --- a/.crow/build-redhat-8-amd64-single.yaml +++ /dev/null @@ -1,68 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-single-rhel8-amd64"' - # PACKAGE: - # TAG: # set NULL to build all versions - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/amd64-binaries-r-redhat-8 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - ARCH: amd64 - NCPUS: 4 - STRATEGY: sequential - # volumes: - # - amd64-binaries-r-dep-cache-redhat-8:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(CRAN = 'https://cran.devxy.io/$ARCH/rhel8/latest'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); bincraftR::build_binary_package(\"$$package\", tag = \"$$tag\", debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'redhat-8')" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-redhat-8-amd64.yaml b/.crow/build-redhat-8-amd64.yaml deleted file mode 100644 index 2e7c900..0000000 --- a/.crow/build-redhat-8-amd64.yaml +++ /dev/null @@ -1,107 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "redhat-8-amd64"' - - event: manual - evaluate: 'build == "all-amd64"' - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/amd64-binaries-r-redhat-8 - pull: true - # to ensure the PACKAGES upload step is being executed - failure: ignore - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - BLOCK: 1833:1833 - ARCH: amd64 - NCPUS: 4 - # MEMORY: 14Gi - # INSTANCE_TYPE: ccx23 - STRATEGY: sequential - # volumes: - # - amd64-binaries-r-dep-cache-redhat-8:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(CRAN = 'https://cran.devxy.io/amd64/rhel8/latest'))); pkgs = c('aphylo', 'atime'); pkgs = setdiff(pkgs, c('RInno', 'MediaNews', 'later', 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'redhat-8'))" - ### Manual pkg builds - # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 - # RInno: Windows-only - # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) - # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved - # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ - # ROracle: sysdep missing -> create rule - # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 - # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex - # RmecabKo: fatal error: mecab.h: No such file or directory - # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 - # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 - # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex - # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 - # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 - # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed - # Racmacs -> rmarchingcubes - # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: - # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 - # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail - # clrng -> gpuR - # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything - # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 - # filters: -> email sent - # magickGUI, biplotbootGUI - # ymd (alpine, voi) - # tfrmtbuilder -> V8 (alpine) - # Rcmdr (alpine) - # readstata13 (alpine) - # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = "amd64"))' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-redhat-8-arm64-single.yaml b/.crow/build-redhat-8-arm64-single.yaml deleted file mode 100644 index 414f9bc..0000000 --- a/.crow/build-redhat-8-arm64-single.yaml +++ /dev/null @@ -1,68 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-single-rhel8-arm64"' - # PACKAGE: - # TAG: # set NULL to build all versions - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/arm64-binaries-r-redhat-8 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - ARCH: arm64 - NCPUS: 4 - STRATEGY: sequential - # volumes: - # - arm64-binaries-r-dep-cache-redhat-8:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(CRAN = 'https://cran.devxy.io/$ARCH/rhel8/latest'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); bincraftR::build_binary_package(\"$$package\", tag = \"$$tag\", debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'redhat-8')" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-redhat-8-arm64.yaml b/.crow/build-redhat-8-arm64.yaml deleted file mode 100644 index 124cafb..0000000 --- a/.crow/build-redhat-8-arm64.yaml +++ /dev/null @@ -1,130 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "redhat-8-arm64"' - - event: manual - evaluate: 'build == "all-arm64"' - -skip_clone: true - -steps: - # - name: 'Warming up' - # image: busybox:latest - # failure: ignore - # commands: - # - sleep 300 - # - echo "Warmed Up" - # backend_options: - # kubernetes: - # resources: - # requests: - # memory: 14Gi - # cpu: 3000m - # limits: - # memory: 14Gi - # cpu: 3000m - # nodeSelector: - # kubernetes.io/arch: arm64 - # node.kubernetes.io/instance-type: cax31 - # tolerations: - # - key: 'CI' - # operator: 'Equal' - # value: 'true' - # effect: 'NoSchedule' - - - name: 'Build binaries' - image: docker.io/devxygmbh/arm64-binaries-r-redhat-8 - pull: true - # to ensure the PACKAGES upload step is being executed - failure: ignore - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - BLOCK: 20500:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 14Gi - INSTANCE_TYPE: cax31 - STRATEGY: multisession - # volumes: - # - arm64-binaries-r-dep-cache-redhat-8:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c('aphylo', 'atime'); pkgs = setdiff(pkgs, c('RInno', 'MediaNews', 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'redhat-8'))" - ### Manual pkg builds - # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 - # RInno: Windows-only - # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) - # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved - # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ - # ROracle: sysdep missing -> create rule - # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 - # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex - # RmecabKo: fatal error: mecab.h: No such file or directory - # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 - # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 - # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex - # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 - # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 - # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed - # Racmacs -> rmarchingcubes - # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: - # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 - # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail - # clrng -> gpuR - # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything - # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 - # filters: -> email sent - # magickGUI, biplotbootGUI - # ymd (alpine, voi) - # tfrmtbuilder -> V8 (alpine) - # Rcmdr (alpine) - # readstata13 (alpine) - # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = "arm64"))' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 14Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: cax41 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-redhat-9-amd64-single.yaml b/.crow/build-redhat-9-amd64-single.yaml deleted file mode 100644 index 18ca98e..0000000 --- a/.crow/build-redhat-9-amd64-single.yaml +++ /dev/null @@ -1,69 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-single-rhel9-amd64"' - # PACKAGE: - # TAG: # set NULL to build all versions - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/amd64-binaries-r-redhat-9 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: amd64 - NCPUS: 4 - STRATEGY: sequential - # volumes: - # - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(CRAN = 'https://cran.devxy.io/$ARCH/rhel9/latest'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); bincraftR::build_binary_package(\"$$package\", tag = \"$$tag\", debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'redhat-9')" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-redhat-9-amd64.yaml b/.crow/build-redhat-9-amd64.yaml deleted file mode 100644 index ee7b11b..0000000 --- a/.crow/build-redhat-9-amd64.yaml +++ /dev/null @@ -1,105 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "redhat-9-amd64"' - - event: manual - evaluate: 'build == "all-amd64"' - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/amd64-binaries-r-redhat-9 - pull: true - # to ensure the PACKAGES upload step is being executed - failure: ignore - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - BLOCK: 2394:2428 - ARCH: amd64 - NCPUS: 4 - INSTANCE_TYPE: AX42 - STRATEGY: sequential - # volumes: - # - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(CRAN = 'https://cran.devxy.io/$ARCH/rhel9/latest'))); pkgs = c('aphylo', 'atime'); pkgs = setdiff(pkgs, c('RInno', 'MediaNews', 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'redhat-9'))" - ### Manual pkg builds - # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 - # RInno: Windows-only - # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) - # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved - # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ - # ROracle: sysdep missing -> create rule - # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 - # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex - # RmecabKo: fatal error: mecab.h: No such file or directory - # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 - # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 - # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex - # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 - # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 - # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed - # Racmacs -> rmarchingcubes - # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: - # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 - # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail - # clrng -> gpuR - # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything - # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 - # filters: -> email sent - # magickGUI, biplotbootGUI - # ymd (alpine, voi) - # tfrmtbuilder -> V8 (alpine) - # Rcmdr (alpine) - # readstata13 (alpine) - # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = "amd64"))' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-redhat-9-arm64-single.yaml b/.crow/build-redhat-9-arm64-single.yaml deleted file mode 100644 index b9002b5..0000000 --- a/.crow/build-redhat-9-arm64-single.yaml +++ /dev/null @@ -1,69 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-single-rhel9-arm64"' - # PACKAGE: - # TAG: # set NULL to build all versions - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/arm64-binaries-r-redhat-9 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: arm64 - NCPUS: 4 - STRATEGY: sequential - # volumes: - # - arm64-binaries-r-dep-cache-redhat-9:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(CRAN = 'https://cran.devxy.io/$ARCH/rhel9/latest'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); bincraftR::build_binary_package(\"$$package\", tag = \"$$tag\", debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'redhat-9')" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-redhat-9-arm64.yaml b/.crow/build-redhat-9-arm64.yaml deleted file mode 100644 index e5f1251..0000000 --- a/.crow/build-redhat-9-arm64.yaml +++ /dev/null @@ -1,130 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "redhat-9-arm64"' - - event: manual - evaluate: 'build == "all-arm64"' - -skip_clone: true - -steps: - # - name: 'Warming up' - # image: busybox:latest - # failure: ignore - # commands: - # - sleep 300 - # - echo "Warmed Up" - # backend_options: - # kubernetes: - # resources: - # requests: - # memory: 14Gi - # cpu: 3000m - # limits: - # memory: 14Gi - # cpu: 3000m - # nodeSelector: - # kubernetes.io/arch: arm64 - # node.kubernetes.io/instance-type: cax31 - # tolerations: - # - key: 'CI' - # operator: 'Equal' - # value: 'true' - # effect: 'NoSchedule' - - - name: 'Build binaries' - image: docker.io/devxygmbh/arm64-binaries-r-redhat-9 - pull: true - # to ensure the PACKAGES upload step is being executed - failure: ignore - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - BLOCK: 20615:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 14Gi - INSTANCE_TYPE: cax31 - STRATEGY: multisession - # volumes: - # - arm64-binaries-r-dep-cache-redhat-9:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c('aphylo', 'atime'); pkgs = setdiff(pkgs, c('RInno', 'MediaNews', 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'redhat-9'))" - ### Manual pkg builds - # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 - # RInno: Windows-only - # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) - # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved - # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ - # ROracle: sysdep missing -> create rule - # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 - # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex - # RmecabKo: fatal error: mecab.h: No such file or directory - # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 - # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 - # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex - # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 - # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 - # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed - # Racmacs -> rmarchingcubes - # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: - # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 - # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail - # clrng -> gpuR - # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything - # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 - # filters: -> email sent - # magickGUI, biplotbootGUI - # ymd (alpine, voi) - # tfrmtbuilder -> V8 (alpine) - # Rcmdr (alpine) - # readstata13 (alpine) - # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = "arm64"))' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 14Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: cax41 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-ubuntu-2204-amd64-single.yaml b/.crow/build-ubuntu-2204-amd64-single.yaml deleted file mode 100644 index 3b4df7f..0000000 --- a/.crow/build-ubuntu-2204-amd64-single.yaml +++ /dev/null @@ -1,68 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-single-jammy-amd64"' - # PACKAGE: - # TAG: # set NULL to build all versions - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2204 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: amd64 - NCPUS: 4 - STRATEGY: sequential - # volumes: - # - amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(CRAN = 'https://cran.devxy.io/$ARCH/jammy/latest'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); bincraftR::build_binary_package(\"$$package\", tag = \"$$tag\", debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'ubuntu-2204')" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-ubuntu-2204-amd64.yaml b/.crow/build-ubuntu-2204-amd64.yaml deleted file mode 100644 index 05d56e3..0000000 --- a/.crow/build-ubuntu-2204-amd64.yaml +++ /dev/null @@ -1,104 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "ubuntu-2204-amd64"' - - event: manual - evaluate: 'build == "all-amd64"' - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2204 - pull: true - # to ensure the PACKAGES upload step is being executed - failure: ignore - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - BLOCK: 4223:4755 - ARCH: amd64 - NCPUS: 4 - STRATEGY: sequential - # volumes: - # - amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c('aphylo', 'atime'); pkgs = setdiff(pkgs, c('RInno', 'MediaNews', 'CoTiMA', 'later', 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'ubuntu-2204'))" - ### Manual pkg builds - # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 - # RInno: Windows-only - # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) - # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved - # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ - # ROracle: sysdep missing -> create rule - # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 - # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex - # RmecabKo: fatal error: mecab.h: No such file or directory - # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 - # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 - # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex - # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 - # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 - # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed - # Racmacs -> rmarchingcubes - # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: - # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 - # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail - # clrng -> gpuR - # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything - # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 - # filters: -> email sent - # magickGUI, biplotbootGUI - # ymd (alpine, voi) - # tfrmtbuilder -> V8 (alpine) - # Rcmdr (alpine) - # readstata13 (alpine) - # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = "amd64"))' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-ubuntu-2204-arm64-single.yaml b/.crow/build-ubuntu-2204-arm64-single.yaml deleted file mode 100644 index 37717a5..0000000 --- a/.crow/build-ubuntu-2204-arm64-single.yaml +++ /dev/null @@ -1,68 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-single-jammy-arm64"' - # PACKAGE: - # TAG: # set NULL to build all versions - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2204 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: arm64 - NCPUS: 4 - STRATEGY: sequential - # volumes: - # - arm64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(CRAN = 'https://cran.devxy.io/$ARCH/jammy/latest'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); bincraftR::build_binary_package(\"$$package\", tag = \"$$tag\", debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'ubuntu-2204')" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-ubuntu-2204-arm64.yaml b/.crow/build-ubuntu-2204-arm64.yaml deleted file mode 100644 index 47157c7..0000000 --- a/.crow/build-ubuntu-2204-arm64.yaml +++ /dev/null @@ -1,130 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "ubuntu-2204-arm64"' - - event: manual - evaluate: 'build == "all-arm64"' - -skip_clone: true - -steps: - # - name: 'Warming up' - # image: busybox:latest - # failure: ignore - # commands: - # - sleep 300 - # - echo "Warmed Up" - # backend_options: - # kubernetes: - # resources: - # requests: - # memory: 14Gi - # cpu: 3000m - # limits: - # memory: 14Gi - # cpu: 3000m - # nodeSelector: - # kubernetes.io/arch: arm64 - # node.kubernetes.io/instance-type: cax31 - # tolerations: - # - key: 'CI' - # operator: 'Equal' - # value: 'true' - # effect: 'NoSchedule' - - - name: 'Build binaries' - image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2204 - pull: true - # to ensure the PACKAGES upload step is being executed - failure: ignore - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - BLOCK: 20653:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 14Gi - INSTANCE_TYPE: cax31 - STRATEGY: multisession - # volumes: - # - arm64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/gert - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c('aphylo', 'atime'); pkgs = setdiff(pkgs, c('RInno', 'MediaNews', 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'ubuntu-2204'))" - ### Manual pkg builds - # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 - # RInno: Windows-only - # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) - # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved - # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ - # ROracle: sysdep missing -> create rule - # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 - # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex - # RmecabKo: fatal error: mecab.h: No such file or directory - # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 - # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 - # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex - # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 - # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 - # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed - # Racmacs -> rmarchingcubes - # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: - # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 - # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail - # clrng -> gpuR - # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything - # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 - # filters: -> email sent - # magickGUI, biplotbootGUI - # ymd (alpine, voi) - # tfrmtbuilder -> V8 (alpine) - # Rcmdr (alpine) - # readstata13 (alpine) - # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = "arm64"))' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 14Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: cax41 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-ubuntu-2404-amd64-single.yaml b/.crow/build-ubuntu-2404-amd64-single.yaml deleted file mode 100644 index baa09e2..0000000 --- a/.crow/build-ubuntu-2404-amd64-single.yaml +++ /dev/null @@ -1,68 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-single-noble-amd64"' - # PACKAGE: - # TAG: # set NULL to build all versions - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - ARCH: amd64 - NCPUS: 4 - STRATEGY: sequential - # volumes: - # - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(CRAN = 'https://cran.devxy.io/$ARCH/noble/latest'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); bincraftR::build_binary_package(\"$$package\", tag = \"$$tag\", debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'ubuntu-2404')" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-ubuntu-2404-amd64.yaml b/.crow/build-ubuntu-2404-amd64.yaml deleted file mode 100644 index e4b221b..0000000 --- a/.crow/build-ubuntu-2404-amd64.yaml +++ /dev/null @@ -1,106 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "ubuntu-2404-amd64"' - - event: manual - evaluate: 'build == "all-amd64"' - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 - pull: true - # to ensure the PACKAGES upload step is being executed - failure: ignore - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - BLOCK: 3025:4298 - ARCH: amd64 - NCPUS: 4 - MEMORY: 14Gi - # INSTANCE_TYPE: ccx23 - STRATEGY: sequential - # volumes: - # - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c('aphylo', 'atime'); pkgs = setdiff(pkgs, c('RInno', 'MediaNews', 'CoTiMA', 'later', 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'ubuntu-2404'))" - ### Manual pkg builds - # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 - # RInno: Windows-only - # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) - # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved - # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ - # ROracle: sysdep missing -> create rule - # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 - # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex - # RmecabKo: fatal error: mecab.h: No such file or directory - # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 - # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 - # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex - # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 - # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 - # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed - # Racmacs -> rmarchingcubes - # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: - # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 - # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail - # clrng -> gpuR - # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything - # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 - # filters: -> email sent - # magickGUI, biplotbootGUI - # ymd (alpine, voi) - # tfrmtbuilder -> V8 (alpine) - # Rcmdr (alpine) - # readstata13 (alpine) - # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = "amd64"))' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-ubuntu-2404-arm64-single.yaml b/.crow/build-ubuntu-2404-arm64-single.yaml deleted file mode 100644 index 52ccb4b..0000000 --- a/.crow/build-ubuntu-2404-arm64-single.yaml +++ /dev/null @@ -1,68 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-single-noble-arm64"' - # PACKAGE: - # TAG: # set NULL to build all versions - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - ARCH: arm64 - NCPUS: 4 - STRATEGY: sequential - # volumes: - # - arm64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL, repos = structure(c(CRAN = 'https://cran.devxy.io/$ARCH/noble/latest'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); bincraftR::build_binary_package(\"$$package\", tag = \"$$tag\", debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'ubuntu-2404')" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 2000m - limits: - memory: 20Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-ubuntu-2404-arm64.yaml b/.crow/build-ubuntu-2404-arm64.yaml deleted file mode 100644 index 5ac0ebc..0000000 --- a/.crow/build-ubuntu-2404-arm64.yaml +++ /dev/null @@ -1,106 +0,0 @@ -when: - - event: manual - evaluate: 'build == "all"' - - event: manual - evaluate: 'build == "ubuntu-2404-arm64"' - - event: manual - evaluate: 'build == "all-arm64"' - -skip_clone: true - -steps: - - name: 'Build binaries' - image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2404 - pull: true - # to ensure the PACKAGES upload step is being executed - failure: ignore - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - BLOCK: 20126:length(pkgs) - ARCH: arm64 - NCPUS: 2 - MEMORY: 14Gi - INSTANCE_TYPE: cax31 - STRATEGY: multisession - # volumes: - # - arm64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews' - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c('aphylo', 'atime'); pkgs = setdiff(pkgs, c('RInno', 'MediaNews', 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'ubuntu-2404'))" - ### Manual pkg builds - # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 - # RInno: Windows-only - # skedastic: waiting for ROI.plugin.qpoases (https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1) - # rswipl: ccache issues: https://github.com/mgondan/rswipl/issues/3 -> solved - # Rpoppler: https://github.com/rstudio/r-system-requirements/pull/179/ - # ROracle: sysdep missing -> create rule - # ROI.plugin.qpoases: https://gitlab.com/roigrp/solver/ROI.plugin.qpoases/-/issues/1 - # ROI.plugin.cplex: checking for cplex... no -> requires propriatary software IBM Cplex - # RmecabKo: fatal error: mecab.h: No such file or directory - # rmarchingcubes: error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing] -> https://github.com/shwilks/rmarchingcubes/issues/3 - # RcppMeCab: /bin/bash: line 2: mecab-config: command not found -> https://github.com/rstudio/r-system-requirements/pull/180 - # Rcplex: checking for cplex... no -> requires propriatary software IBM Cplex - # Rblpapi: Unknown architecture: aarch64. Exiting. -> https://github.com/Rblp/Rblpapi/issues/399 - # Rbeast: aarch64 issue -> https://github.com/zhaokg/Rbeast/issues/29 - # ravetools: checking for FFTW... configure: error: Package requirements (fftw3) were not met: -> fftw3 rule needed - # Racmacs -> rmarchingcubes - # minic: /usr/include/c++/13/bits/stl_algo.h:5805:5: note: template argument deduction/substitution failed: - # KSgeneral -> aligned_mem.h:6:10: fatal error: mm_malloc.h: No such file or directory -> https://github.com/d-dimitrova/KSgeneral/issues/1 - # gpuR -> /usr/bin/ld: cannot find -lOpenCL: No such file or directory -> no GPU avail - # clrng -> gpuR - # BRugs: configure: error: OpenBUGS 3.2.2 or later not found. as it is only one package, not doing anything - # gamstransfer -> gdx/src/gdlib/gmsstrm.cpp:85:70: error: narrowing conversion of '-1' from 'std::char_traits::int_type' {aka 'int'} to 'char' [-Wnarrowing] -> https://github.com/GAMS-dev/transfer-r/issues/2 - # filters: -> email sent - # magickGUI, biplotbootGUI - # ymd (alpine, voi) - # tfrmtbuilder -> V8 (alpine) - # Rcmdr (alpine) - # readstata13 (alpine) - # - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = "arm64"))' - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 14Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - node.kubernetes.io/instance-type: cax41 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/docs.yaml b/.crow/docs.yaml deleted file mode 100644 index 4daa810..0000000 --- a/.crow/docs.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: manual - evaluate: 'task == "docs"' - - event: cron - cron: 'build == "alpine-320-arm64"' - - event: manual - evaluate: 'build == "all-arm64"' - -steps: - quarto-build: - image: ghcr.io/quarto-dev/quarto:1.8.12 - directory: docs - commands: - - quarto render - backend_options: - kubernetes: - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 1000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - s3-sync: - image: plugins/s3:1.5.3 - directory: docs - settings: - bucket: devxy-r-package-binaries-docs - region: fsn1 - endpoint: https://fsn1.your-objectstorage.com - access_key: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - secret_key: - from_secret: HETZNER_S3_SECRET_KEY_K3S - source: _site/**/* - strip_prefix: _site/ - target: / - backend_options: - kubernetes: - resources: - requests: - memory: 300Mi - cpu: 200m - limits: - memory: 1Gi - cpu: 1000m - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' \ No newline at end of file diff --git a/.crow/images.yaml b/.crow/images.yaml deleted file mode 100644 index 35491ca..0000000 --- a/.crow/images.yaml +++ /dev/null @@ -1,70 +0,0 @@ -when: - - event: cron - cron: images - -matrix: - include: - - OS: redhat - OS_VERSION: 8 - R_VERSION: 4.4.3 - - OS: redhat - OS_VERSION: 9 - R_VERSION: 4.4.3 - - OS: ubuntu - OS_VERSION: jammy - R_VERSION: 4.4.3 - - OS: ubuntu - OS_VERSION: noble - R_VERSION: 4.4.3 - - OS: alpine - OS_VERSION: 3.21 - R_VERSION: 4.4.3 - - OS: alpine - OS_VERSION: 3.22 - R_VERSION: 4.5.0 - -steps: - - name: 'Build image' - image: docker.io/woodpeckerci/plugin-docker-buildx:6.0.1 - settings: - mtu: 1350 - platforms: linux/amd64,linux/arm64 - Dockerfile: docker/Containerfile-${OS} - repo: devxygmbh/rpkgs-build-env-${OS} - tags: ${OS_VERSION}-${R_VERSION} - username: devxygmbh - password: - from_secret: dockerhub_token_devxygmbh - build_args: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - GITHUB_PAT: - from_secret: GITHUB_PAT - build_args_from_env: - - OS - - OS_VERSION - - R_VERSION - backend_options: - kubernetes: - resources: - requests: - memory: 1300Mi - cpu: 1000m - limits: - memory: 1300Mi - cpu: 2000m - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-alpine-320-amd64.yaml b/.crow/process-updates-alpine-320-amd64.yaml deleted file mode 100644 index 0b509e7..0000000 --- a/.crow/process-updates-alpine-320-amd64.yaml +++ /dev/null @@ -1,84 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-320-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-320-amd64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.21-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-320 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-alpine-320:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - # force install pak devel as otherwise sysdeps on Alpine are not resolved correctly - - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'alpine-320', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-alpine-320-arm64.yaml b/.crow/process-updates-alpine-320-arm64.yaml deleted file mode 100644 index 3bb98a7..0000000 --- a/.crow/process-updates-alpine-320-arm64.yaml +++ /dev/null @@ -1,86 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-320-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-320-arm64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.21-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-320 - ARCH: arm64 - NCPUS: 2 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-alpine-320:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - # force install pak devel as otherwise sysdeps on Alpine are not resolved correctly - - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'alpine-320', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - ### PROD - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 14Gi - cpu: 3000m - nodeSelector: - node.kubernetes.io/instance-type: cax31 - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-alpine-321-amd64.yaml b/.crow/process-updates-alpine-321-amd64.yaml deleted file mode 100644 index f485e95..0000000 --- a/.crow/process-updates-alpine-321-amd64.yaml +++ /dev/null @@ -1,84 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-321-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-321-amd64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.21-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-alpine-321:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - # force install pak devel as otherwise sysdeps on Alpine are not resolved correctly - - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-alpine-321-arm64.yaml b/.crow/process-updates-alpine-321-arm64.yaml deleted file mode 100644 index a0ec29b..0000000 --- a/.crow/process-updates-alpine-321-arm64.yaml +++ /dev/null @@ -1,84 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-321-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-321-arm64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.21-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-alpine-321:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - # force install pak devel as otherwise sysdeps on Alpine are not resolved correctly - - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - ### PROD - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 14Gi - cpu: 3000m - nodeSelector: - node.kubernetes.io/instance-type: cax31 - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml deleted file mode 100644 index 76ff290..0000000 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ /dev/null @@ -1,84 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-alpine-322-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "alpine-322-amd64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - # force install pak devel as otherwise sysdeps on Alpine are not resolved correctly - - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml deleted file mode 100644 index 2abaf64..0000000 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ /dev/null @@ -1,83 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-redhat-8-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "redhat-8-amd64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-redhat:8-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - ARCH: amd64 - NCPUS: 2 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-redhat-8:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - rm -rf /mnt/cache/R-pkgs/gert /mnt/cache/R-pkgs/git2r /mnt/cache/R-pkgs/pkgbuild /mnt/cache/R-pkgs/processx /mnt/cache/R-pkgs/curl - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml deleted file mode 100644 index b47613f..0000000 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ /dev/null @@ -1,83 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-redhat-8-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "redhat-8-arm64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-redhat:8-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-redhat-8:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - rm -rf /mnt/cache/R-pkgs/gert /mnt/cache/R-pkgs/git2r /mnt/cache/R-pkgs/pkgbuild /mnt/cache/R-pkgs/processx /mnt/cache/R-pkgs/curl - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - ### PROD - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 14Gi - cpu: 3000m - nodeSelector: - node.kubernetes.io/instance-type: cax31 - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml deleted file mode 100644 index 28be06f..0000000 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ /dev/null @@ -1,83 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-redhat-9-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "redhat-9-amd64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-redhat:9-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: amd64 - NCPUS: 2 - MEMORY: 18Gi - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml deleted file mode 100644 index e35666c..0000000 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ /dev/null @@ -1,81 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-redhat-9-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "redhat-9-arm64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-redhat:9-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-redhat-9:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 14Gi - cpu: 3000m - nodeSelector: - node.kubernetes.io/instance-type: cax31 - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml deleted file mode 100644 index ccacf6a..0000000 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ /dev/null @@ -1,83 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-ubuntu-2204-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "ubuntu-2204-amd64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:jammy-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: amd64 - NCPUS: 2 - MEMORY: 18Gi - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml deleted file mode 100644 index 0cf5743..0000000 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ /dev/null @@ -1,81 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-ubuntu-2204-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "ubuntu-2204-arm64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:jammy-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 14Gi - cpu: 3000m - nodeSelector: - node.kubernetes.io/instance-type: cax31 - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml deleted file mode 100644 index c4d91dc..0000000 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ /dev/null @@ -1,82 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-ubuntu-2404-amd64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "ubuntu-2404-amd64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - ARCH: amd64 - NCPUS: 1 - STRATEGY: sequential - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml deleted file mode 100644 index 3429a2f..0000000 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ /dev/null @@ -1,81 +0,0 @@ -when: - - event: [cron] - cron: process-cran-updates-ubuntu-2404-arm64 - - event: manual - evaluate: 'process_cran_updates == "all"' - - event: manual - evaluate: 'process_cran_updates == "ubuntu-2404-arm64"' - -skip_clone: true - -steps: - - name: 'Processing Updates' - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - ARCH: arm64 - NCPUS: 2 - INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) - # volumes: - # - arm64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'packageVersion("bincraftR")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 14Gi - cpu: 3000m - nodeSelector: - node.kubernetes.io/instance-type: cax31 - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml new file mode 100644 index 0000000..73bfff7 --- /dev/null +++ b/.crow/process-updates.yaml @@ -0,0 +1,256 @@ +# Consolidated process-updates pipeline (all platforms, both arches). +# One matrix row per OS/arch replaces the former per-platform files. +# Routing is preserved 1:1: +# - cron: each existing `process-cran-updates--` cron fires only +# its matching matrix row (via the per-row `cron:` name filter). +# - manual: pick a target from the `process_cran_updates` dropdown; +# "all" fans out every os/arch as parallel matrix workflows. +# Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). +variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. + process_cran_updates: + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: process-cran-updates-${OS}-${ARCH} + - event: manual + evaluate: 'process_cran_updates == "all" || process_cran_updates == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + OS_ID: alpine322 + PROCESS_NEW: 'FALSE' + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + OS_ID: alpine322 + PROCESS_NEW: 'FALSE' + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + OS_ID: alpine323 + PROCESS_NEW: 'FALSE' + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + OS_ID: alpine323 + PROCESS_NEW: 'FALSE' + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + OS_ID: alpine324 + PROCESS_NEW: 'FALSE' + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + OS_ID: alpine324 + PROCESS_NEW: 'FALSE' + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + OS_ID: rhel8 + PROCESS_NEW: 'TRUE' + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + OS_ID: rhel8 + PROCESS_NEW: 'TRUE' + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + OS_ID: rhel9 + PROCESS_NEW: 'TRUE' + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + OS_ID: rhel9 + PROCESS_NEW: 'TRUE' + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + OS_ID: rhel10 + PROCESS_NEW: 'TRUE' + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + OS_ID: rhel10 + PROCESS_NEW: 'TRUE' + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + OS_ID: jammy + PROCESS_NEW: 'TRUE' + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + OS_ID: jammy + PROCESS_NEW: 'TRUE' + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + OS_ID: noble + PROCESS_NEW: 'TRUE' + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + OS_ID: noble + PROCESS_NEW: 'TRUE' + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + OS_ID: resolute + PROCESS_NEW: 'TRUE' + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + OS_ID: resolute + PROCESS_NEW: 'TRUE' + +steps: + - name: 'Processing Updates' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of uvr's caches, which persist the R package + # dependencies needed to install the packages themselves + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ${OS} + OS_ID: ${OS_ID} + ARCH: ${ARCH} + NCPUS: 2 + INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = ${PROCESS_NEW}, process_removed = TRUE, patches = 'local/patches', r_minor_detection = 'classifier',s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - | + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2) + for RBIN in /opt/R/[0-9]*/bin/R; do + RV=$(basename "$(dirname "$(dirname "$RBIN")")") + RMINOR=$(echo "$RV" | cut -d. -f1-2) + [ "$RMINOR" = "$PRIMARY_MINOR" ] && continue + echo "=== R-minor-sensitive update pass under R $RV ===" + LIB="/mnt/cache/R-pkgs-$RMINOR" + mkdir -p "$LIB" + R_LIBS_USER="$LIB" "$(dirname "$RBIN")/Rscript" local/install-bincraft.R || true + R_LIBS_USER="$LIB" UVR_R_BIN="$RBIN" local/uvr-install.sh RPostgres || true + R_LIBS_USER="$LIB" $XVFB $XVFB_ARGS -- "$(dirname "$RBIN")/R" -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = '${OS}', process_updated = TRUE, process_new = FALSE, process_removed = FALSE, patches = 'local/patches', r_minor_detection = 'classifier',r_minor_sensitive_only = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" || true + done + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "${OS_ID}", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(codename = '${OS_ID}', r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 3000m + limits: + memory: 18Gi + cpu: 3000m + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + SUBDOMAIN1: 'cran.devxy.io' + SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' + SUBDOMAIN3: 'cran.rpkgs.com' + OS_ID: ${OS_ID} + ARCH: ${ARCH} + commands: + - apk add --no-cache -q bash curl + - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3" diff --git a/.crow/repair-built-stamp.yaml b/.crow/repair-built-stamp.yaml new file mode 100644 index 0000000..e56e1e9 --- /dev/null +++ b/.crow/repair-built-stamp.yaml @@ -0,0 +1,110 @@ +### Manual repair of a slot whose PACKAGES index advertises a broken `Built` +### stamp (e.g. `Built: R 4.5.0; NA; ...`). +# +# uvr matches the stamp's platform triple plus R minor to decide binary vs +# source, so an unusable triple turns a whole slot source-only. See +# local/repair-built-stamp.R for why this patches PACKAGES.db in place instead +# of forcing a full reparse. +# +# Run with `dry_run: true` first: it reports how many entries are broken per +# slot and changes nothing. Pick the R version the slot should advertise, which +# is the R_VERSION its entry in .crow/process-updates.yaml uses. +# +# The gate variable is `repair_built_stamp`, not `target_arch`: `target_arch` is +# what build-all-versions and build-all-versions-install-deps gate on, so a +# manual run passing it would start a full rebuild alongside this repair. Every +# pipeline here gates on a variable named after itself for exactly that reason. +# +# crow pipeline create --branch main \ +# --var repair_built_stamp=arm64 --var OS=alpine --var OS_VERSION=3.22 \ +# --var R_VERSION=4.5.3 --var dry_run=true devxy/build-cran-binaries +variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. + repair_built_stamp: + description: 'Architecture of the slot to repair, or "none" to run nothing.' + options: + - none + - amd64 + - arm64 + default: none + OS: + description: 'Base OS image name.' + options: + - alpine + - redhat + - ubuntu + default: alpine + OS_VERSION: + description: 'OS image tag. Must match OS (alpine: 3.22/3.23/3.24; redhat: 8/9/10; ubuntu: jammy/noble/resolute).' + options: + - '3.22' + - '3.23' + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + - 'resolute' + default: '3.22' + R_VERSION: + description: 'R version whose stamp the slot should advertise.' + options: + - 4.5.3 + - 4.4.3 + default: 4.5.3 + dry_run: + description: 'Report what would change without writing anything.' + options: + - 'true' + - 'false' + default: 'true' + +when: + - event: manual + evaluate: 'repair_built_stamp == "${ARCH}"' + +skip_clone: true + +labels: + platform: linux/${ARCH} + group: rpkgs-${ARCH} + +matrix: + include: + - ARCH: amd64 + - ARCH: arm64 + +steps: + - name: 'Repair Built stamp' + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - | + if [ "$dry_run" = "false" ]; then + /opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" --apply + else + /opt/R/$R_VERSION/bin/Rscript local/repair-built-stamp.R "$ARCH" + fi + backend_options: + docker: + resources: + requests: + memory: 2Gi + cpu: 1000m + limits: + memory: 8Gi + cpu: 2000m diff --git a/.crow/trial-build-registry.yaml b/.crow/trial-build-registry.yaml new file mode 100644 index 0000000..2195423 --- /dev/null +++ b/.crow/trial-build-registry.yaml @@ -0,0 +1,160 @@ +# Merge gate for the auto-patch PR (issue #115, step 3). +# For each platform, trial-builds every registry entry the auto-patch branch +# ADDS (vs main) in that platform's own `reg.devxy.io/rpkgs/build-env-*` image, +# with the registry applied. A row with no new entries for its platform is a +# fast no-op. The pipeline is green only if every new entry builds, so it gates +# the PR before merge. Nothing is uploaded/archived/recorded. +# +# The repo uses no `pull_request` triggers, so this runs manually against the +# branch (or on a cron); point it at the auto-patch branch via `patch_branch`: +# crow pipeline create --branch main --var trial_build_registry=true \ +# --var patch_branch=auto/registry-patch-proposals devxy/build-cran-binaries +# +# The gate variable is `trial_build_registry`, named after the pipeline: a +# manual run instantiates every pipeline in `.crow/`, so one without its own +# gate runs on *any* manual trigger in this repo. This one starts a build per +# matrix row on both arches, which is far too expensive to fire by accident. +variables: + trial_build_registry: + description: 'Trial-build the branch new registry entries. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' + patch_branch: + description: 'Branch whose new registry entries to trial-build.' + default: auto/registry-patch-proposals + +when: + - event: manual + evaluate: 'trial_build_registry == "true"' + - event: cron + cron: trial-build-registry + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + +steps: + - name: 'Trial-build new registry entries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + PLATFORM: ${OS} + ARCH: ${ARCH} + R_VERSION: ${R_VERSION} + R_LIBS_USER: /mnt/cache/R-pkgs + # Surface the real compiler error when an isolated patched build fails, + # instead of bincraft's opaque "System command 'R' failed" (needs bincraft + # with BINCRAFT_VERBOSE_PATCH_BUILD support; harmless on older versions). + BINCRAFT_VERBOSE_PATCH_BUILD: 'TRUE' + commands: + # Clone main, then check out the auto-patch branch if it exists. When the + # proposer had no candidates it never (re)creates that branch, so a missing + # branch means "nothing to verify" -- no-op cleanly instead of failing the + # clone. + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - git fetch -q origin main + - 'if git ls-remote --exit-code --heads origin ${patch_branch} >/dev/null 2>&1; then git fetch -q origin ${patch_branch} && git checkout -q FETCH_HEAD; else echo "No ${patch_branch} branch; no pending auto-patch proposals to verify."; exit 0; fi' + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/Rscript local/trial-build-registry.R origin/main + backend_options: + kubernetes: + resources: + requests: + memory: 2Gi + cpu: 2000m + limits: + memory: 4Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/update-package-index-alpine-320-amd64.yaml b/.crow/update-package-index-alpine-320-amd64.yaml deleted file mode 100644 index 41d64b5..0000000 --- a/.crow/update-package-index-alpine-320-amd64.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-alpine-320-amd64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "alpine-320-amd64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine320", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'amd64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine320%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine320%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine320%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine320%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine320%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-alpine-320-arm64.yaml b/.crow/update-package-index-alpine-320-arm64.yaml deleted file mode 100644 index b2eb18f..0000000 --- a/.crow/update-package-index-alpine-320-arm64.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-alpine-320-arm64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "alpine-320-arm64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine320", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'arm64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Falpine320%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Falpine320%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Falpine320%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Falpine320%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Falpine320%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-alpine-321-amd64.yaml b/.crow/update-package-index-alpine-321-amd64.yaml deleted file mode 100644 index d84a1d6..0000000 --- a/.crow/update-package-index-alpine-321-amd64.yaml +++ /dev/null @@ -1,89 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-alpine-321-amd64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "alpine-321-amd64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - # volumes: - # - amd64-binaries-r-dep-cache-alpine-321:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - # - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine321", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'amd64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine321%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine321%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine321%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine321%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine321%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-alpine-321-arm64.yaml b/.crow/update-package-index-alpine-321-arm64.yaml deleted file mode 100644 index 538bd72..0000000 --- a/.crow/update-package-index-alpine-321-arm64.yaml +++ /dev/null @@ -1,84 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-alpine-321-arm64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "alpine-321-arm64"' - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - commands: - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine321", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'arm64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: alpine:3.22 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - OS_ID: alpine321 - ARCH: arm64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-alpine-322-amd64.yaml b/.crow/update-package-index-alpine-322-amd64.yaml deleted file mode 100644 index 5611cf5..0000000 --- a/.crow/update-package-index-alpine-322-amd64.yaml +++ /dev/null @@ -1,89 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-alpine-322-amd64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "alpine-322-amd64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - # volumes: - # - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - # - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine322", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'amd64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-alpine-322-arm64.yaml b/.crow/update-package-index-alpine-322-arm64.yaml deleted file mode 100644 index 8ec94cd..0000000 --- a/.crow/update-package-index-alpine-322-arm64.yaml +++ /dev/null @@ -1,89 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-alpine-322-arm64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "alpine-322-arm64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/rpkgs-build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - # volumes: - # - arm64-binaries-r-dep-cache-alpine-322:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - # - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine322", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'arm64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.11.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-redhat-8-amd64.yaml b/.crow/update-package-index-redhat-8-amd64.yaml deleted file mode 100644 index ec7d6be..0000000 --- a/.crow/update-package-index-redhat-8-amd64.yaml +++ /dev/null @@ -1,86 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-redhat-8-amd64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "redhat-8-amd64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel8", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'amd64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Frhel8%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Frhel8%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Frhel8%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Frhel8%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Frhel8%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-redhat-8-arm64.yaml b/.crow/update-package-index-redhat-8-arm64.yaml deleted file mode 100644 index 4fe7647..0000000 --- a/.crow/update-package-index-redhat-8-arm64.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-redhat-8-arm64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "redhat-8-arm64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel8",s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'arm64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Frhel8%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Frhel8%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Frhel8%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Frhel8%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Frhel8%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-redhat-9-amd64.yaml b/.crow/update-package-index-redhat-9-amd64.yaml deleted file mode 100644 index d84c610..0000000 --- a/.crow/update-package-index-redhat-9-amd64.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-redhat-9-amd64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "redhat-9-amd64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel9", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'amd64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: alpine:3.22 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - SUBDOMAIN1: 'cran.devxy.io' - SUBDOMAIN2: 'cran.allianceswisspass.devxy.io' - OS_ID: rhel9 - ARCH: amd64 - commands: - - apk add --no-cache -q bash curl - - bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-redhat-9-arm64.yaml b/.crow/update-package-index-redhat-9-arm64.yaml deleted file mode 100644 index f8ab505..0000000 --- a/.crow/update-package-index-redhat-9-arm64.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-redhat-9-arm64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "redhat-9-arm64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel9", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'arm64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Frhel9%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Frhel9%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Frhel9%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Frhel9%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Frhel9%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-ubuntu-2204-amd64.yaml b/.crow/update-package-index-ubuntu-2204-amd64.yaml deleted file mode 100644 index 4969911..0000000 --- a/.crow/update-package-index-ubuntu-2204-amd64.yaml +++ /dev/null @@ -1,88 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-ubuntu-2204-amd64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "ubuntu-2204-amd64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "jammy", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S")); warnings()' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'amd64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: alpine:3.22 - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - apk add -q --no-cache curl ca-certificates - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Fjammy%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Fjammy%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Fjammy%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Fjammy%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Fjammy%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-ubuntu-2204-arm64.yaml b/.crow/update-package-index-ubuntu-2204-arm64.yaml deleted file mode 100644 index 2cf14fb..0000000 --- a/.crow/update-package-index-ubuntu-2204-arm64.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-ubuntu-2204-arm64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "ubuntu-2204-arm64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2204 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "jammy", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'arm64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Fjammy%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Fjammy%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Fjammy%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Fjammy%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Fjammy%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-ubuntu-2404-amd64.yaml b/.crow/update-package-index-ubuntu-2404-amd64.yaml deleted file mode 100644 index 1a58665..0000000 --- a/.crow/update-package-index-ubuntu-2404-amd64.yaml +++ /dev/null @@ -1,89 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-ubuntu-2404-amd64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "ubuntu-2404-amd64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - # volumes: - # - amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "noble", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'amd64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Fnoble%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Fnoble%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Fnoble%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Fnoble%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Fnoble%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-ubuntu-2404-arm64.yaml b/.crow/update-package-index-ubuntu-2404-arm64.yaml deleted file mode 100644 index 6fb9f96..0000000 --- a/.crow/update-package-index-ubuntu-2404-arm64.yaml +++ /dev/null @@ -1,87 +0,0 @@ -when: - - event: [cron] - cron: update-package-index-ubuntu-2404-arm64 - - event: [cron] - cron: update-package-index - - event: manual - evaluate: 'update_package_index == "ubuntu-2404-arm64"' - -skip_clone: true - -steps: - - name: Upload PACKAGES files - image: docker.io/devxygmbh/arm64-binaries-r-ubuntu-2404 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - HETZNER_S3_ACCESS_KEY_K3S: - from_secret: HETZNER_S3_ACCESS_KEY_K3S - HETZNER_S3_SECRET_KEY_K3S: - from_secret: HETZNER_S3_SECRET_KEY_K3S - PGPASS: - from_secret: PGPASS - GIT_RO_TOKEN: - from_secret: GIT_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - # normal env vars - GIT_USER: pat-s - NTFY_SERVER: - from_secret: NTFY_SERVER - NTFY_TOPIC: - from_secret: NTFY_TOPIC - NTFY_AUTH: TRUE - NTFY_PASSWORD: - from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - commands: - - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - - rm -rf /mnt/cache/packages/* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' - - R -q -e 'pak::pak("pat-s/cranlike@s3")' - - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - - R -q -e 'packageVersion("bincraftR")' - - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "noble", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' - backend_options: - kubernetes: - resources: - requests: - memory: 3000Mi - cpu: 2000m - limits: - memory: 3000Mi - cpu: 8000m - nodeSelector: - kubernetes.io/arch: 'arm64' - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' - - - name: Purge CDN cache - image: curlimages/curl - environment: - BUNNYNET_API_KEY: - from_secret: BUNNYNET_API_KEY - commands: | - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Fnoble%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Fnoble%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Fnoble%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Fnoble%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false - curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Farm64%2Fnoble%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false - sleep 20 # need to wait a bit to successfully deliver API calls - - - name: 'ntfy' - image: binwiederhier/ntfy:v2.12.0 - when: - - status: [failure] - environment: - NTFY_TOKEN: - from_secret: ntfy_token - commands: | - ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml new file mode 100644 index 0000000..428aa04 --- /dev/null +++ b/.crow/weekly-audit-missing.yaml @@ -0,0 +1,160 @@ +# Consolidated weekly-audit-missing pipeline (all platforms, both arches). +# One matrix row per OS/arch replaces the former per-platform files. +# Routing is preserved 1:1: +# - cron: each existing `weekly-audit-missing--` cron fires only +# its matching matrix row (via the per-row `cron:` name filter). +# - manual: pick a target from the `weekly_audit_missing` dropdown; +# "all" fans out every os/arch as parallel matrix workflows. +# Arch placement is via the group label (rpkgs-amd64, rpkgs-arm64). +variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. + weekly_audit_missing: + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-audit-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_audit_missing == "all" || weekly_audit_missing == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: ubuntu:resolute + +steps: + - name: 'Audit missing binaries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + PLATFORM: ${OS} + ARCH: ${ARCH} + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite + - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' + backend_options: + docker: + resources: + requests: + memory: 2Gi + cpu: 2000m + limits: + memory: 4Gi + cpu: 2000m diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml new file mode 100644 index 0000000..712e87d --- /dev/null +++ b/.crow/weekly-patch-proposals.yaml @@ -0,0 +1,72 @@ +# Weekly failure-triage proposals (issue #115, steps 3 + 4). +# Classifies the recorded `single_builds` failures and: +# 1. posts/updates a Forgejo issue with the auto-proposable registry entries +# (human reviews, trial-builds, and opens the PR -- nothing merges), +# 2. posts/updates a second Forgejo issue with the classifier's blind spots +# (unclassified failures = candidates for new signatures, and groups +# blocked on a dependency), and prints the feedback-loop metrics to the log. +# Global across platforms (the classifier groups over all of single_builds), so +# a single job -- no matrix. Clones read-only; the only writes are the two +# Forgejo issues via FORGEJO_TOKEN. +# +# Run manually with: +# crow pipeline create --branch main \ +# --var weekly_patch_proposals=true devxy/build-cran-binaries +# +# The gate variable is `weekly_patch_proposals`, named after the pipeline: a +# manual run instantiates every pipeline in `.crow/`, so one without its own +# gate runs on *any* manual trigger in this repo. This one posts and edits +# Forgejo issues, so an unrelated manual run must not fire it. +variables: + weekly_patch_proposals: + description: 'Run the weekly failure triage. Also gates this pipeline.' + options: + - 'true' + - 'false' + default: 'false' + +when: + - event: manual + evaluate: 'weekly_patch_proposals == "true"' + - event: cron + cron: weekly-patch-proposals + +skip_clone: true + +labels: + group: rpkgs-amd64 + +steps: + - name: 'Propose registry patches + track feedback loop' + image: reg.devxy.io/rpkgs/build-env-alpine:3.24 + pull: true + environment: + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + R_VERSION: 4.5.3 + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite + - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue + - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue + backend_options: + kubernetes: + resources: + requests: + memory: 1Gi + cpu: 2000m + limits: + memory: 2Gi + cpu: 2000m + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml new file mode 100644 index 0000000..9639792 --- /dev/null +++ b/.crow/weekly-rebuild-missing.yaml @@ -0,0 +1,441 @@ +# Consolidated weekly-rebuild-missing pipeline (all platforms, both arches). +# Three matrix rows per OS/arch, one per shard of that slot's rebuild list. +# Routing is preserved 1:1: +# - cron: each existing `weekly-rebuild-missing--` cron fires only +# its matching matrix rows (via the per-row `cron:` name filter), +# which is now all three shards of that slot. +# - manual: pick a target from the `weekly_rebuild_missing` dropdown; +# "all" fans out every os/arch and shard as parallel matrix +# workflows, while a single - runs its three shards. +# Arch placement is handled by the group label (rpkgs-amd64, rpkgs-arm64). +# +# The shard picks up its own slice and re-derives what is still outstanding +# from the bucket, so a restart resumes rather than replaying; see +# local/rebuild-missing.R. +# +# Re-indexing and the CDN purge deliberately do NOT live here. Three shards +# writing one slot's PACKAGES concurrently would race, so they moved to +# .crow/weekly-rebuild-reindex.yaml, which depends on this pipeline. +variables: + # Gates this pipeline. A manual pipeline creation instantiates every file in + # .crow/, and a declared default is applied even when the run never passed + # this variable, so the default must be a value that matches no matrix row. + weekly_rebuild_missing: + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-rebuild-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 1 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 2 + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + SPLIT_INTO: 3 + SPLIT_INDEX: 3 + +steps: + - name: 'Rebuild missing binaries' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + FORGEJO_TOKEN: + from_secret: FORGEJO_TOKEN + GIT_USER: pat-s + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: ${OS} + ARCH: ${ARCH} + NCPUS: 2 + SPLIT_INTO: ${SPLIT_INTO} + SPLIT_INDEX: ${SPLIT_INDEX} + # Wall clock after which the shard stops cleanly instead of having to be + # killed. A kill matches neither `success` nor `failure`, so it would skip + # the dependent re-index and leave rebuilt binaries behind a stale edge. + REBUILD_BUDGET_HOURS: 20 + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' + - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 + - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' + - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- /opt/R/$R_VERSION/bin/Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX $REBUILD_BUDGET_HOURS 2>&1 + backend_options: + docker: + resources: + requests: + memory: 5Gi + cpu: 3000m + limits: + memory: 18Gi + cpu: 3000m diff --git a/.crow/weekly-rebuild-reindex.yaml b/.crow/weekly-rebuild-reindex.yaml new file mode 100644 index 0000000..2e0f2a0 --- /dev/null +++ b/.crow/weekly-rebuild-reindex.yaml @@ -0,0 +1,191 @@ +# Re-index and purge after weekly-rebuild-missing. +# +# weekly-rebuild-missing runs three shards per slot. Each of them replaces +# objects in place, so the slot's index still advertises the old MD5 and, for +# anything that had been served from source, no Built stamp. Re-indexing from +# inside a shard would mean three concurrent `upload_package_index()` calls on +# one prefix: `cranlike::update_PACKAGES()` lists the live bucket, so an early +# lister that uploads last publishes an index missing its siblings' work. +# +# So it happens exactly once per slot, here, after every shard has finished. +# `runs_on: [success, failure]` keeps that true when a shard fails; only an +# explicit cancel skips it, and this pipeline can then be triggered on its own. + +variables: + # Mirrors the gate on weekly-rebuild-missing so a manual run re-indexes + # exactly the slots it rebuilt. A manual pipeline creation instantiates every + # file in .crow/, so the default must match no matrix row. + weekly_rebuild_missing: + description: "Manual run target: a specific -, 'all' to run every os/arch in parallel, or 'none' to run nothing." + options: + - none + - all + - alpine-322-amd64 + - alpine-322-arm64 + - alpine-323-amd64 + - alpine-323-arm64 + - alpine-324-amd64 + - alpine-324-arm64 + - redhat-8-amd64 + - redhat-8-arm64 + - redhat-9-amd64 + - redhat-9-arm64 + - redhat-10-amd64 + - redhat-10-arm64 + - ubuntu-2204-amd64 + - ubuntu-2204-arm64 + - ubuntu-2404-amd64 + - ubuntu-2404-arm64 + - ubuntu-2604-amd64 + - ubuntu-2604-arm64 + default: none + +when: + - event: cron + cron: weekly-rebuild-missing-${OS}-${ARCH} + - event: manual + evaluate: 'weekly_rebuild_missing == "all" || weekly_rebuild_missing == "${OS}-${ARCH}"' + +depends_on: + - weekly-rebuild-missing + +runs_on: [success, failure] + +skip_clone: true + +labels: + group: rpkgs-${ARCH} + +matrix: + include: + - OS: alpine-322 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-322 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.22 + - OS: alpine-323 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-323 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.23 + - OS: alpine-324 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: alpine-324 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: alpine:3.24 + - OS: redhat-8 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-8 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:8 + - OS: redhat-9 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-9 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: redhat:9 + - OS: redhat-10 + ARCH: amd64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: redhat-10 + ARCH: arm64 + R_VERSION: 4.5.3 + IMG: redhat:10 + - OS: ubuntu-2204 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2204 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:jammy + - OS: ubuntu-2404 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2404 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:noble + - OS: ubuntu-2604 + ARCH: amd64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + - OS: ubuntu-2604 + ARCH: arm64 + R_VERSION: 4.4.3 + IMG: ubuntu:resolute + +steps: + - name: 'Re-index the slot' + image: reg.devxy.io/rpkgs/build-env-${IMG} + pull: true + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + B2_S3_ACCESS_KEY: + from_secret: B2_S3_ACCESS_KEY + B2_S3_SECRET_KEY: + from_secret: B2_S3_SECRET_KEY + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GIT_USER: pat-s + R_LIBS_USER: /mnt/cache/R-pkgs + R_VERSION: ${R_VERSION} + PLATFORM: ${OS} + ARCH: ${ARCH} + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - mkdir -p /mnt/cache/R-pkgs + - rm -rf /mnt/cache/R-pkgs/00LOCK-* + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + # The codename is detected from the image's /etc/os-release. + - /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - | + for RBIN in /opt/R/[0-9]*/bin/R; do + RMINOR=$(basename "$(dirname "$(dirname "$RBIN")")" | cut -d. -f1-2) + /opt/R/$R_VERSION/bin/R -q -e "library(bincraft); upload_package_index(r_minor = '$RMINOR', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'))" || true + done + + - name: Purge CDN cache + image: reg.devxy.io/docker.io/library/alpine:3.24 + environment: + OTEL_R_TRACES_EXPORTER: none + OTEL_R_LOGS_EXPORTER: none + OTEL_R_METRICS_EXPORTER: none + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + # cran.rpkgs.com and cran.allianceswisspass.devxy.io are on separate + # Bunny pull zones, so both must be purged after the shared origin changes. + BUNNY_PULLZONES: '3857050 cran.allianceswisspass.devxy.io' + commands: + - apk add --no-cache -q bash curl jq + # Crow carries the checkout from the re-index step into this step. + - bash scripts/purge_cdn_zone.sh "$BUNNYNET_API_KEY" $BUNNY_PULLZONES + # Runs on every row rather than on one designated slot: a cron fires only + # its own slot's row, so gating on a named slot would leave every other + # slot unpurged. A manual "all" run therefore purges the zone 18 times, + # which is a cheap API call and rare. + # + # Run it even when the re-index above failed: the objects were still + # replaced, and a stale edge is exactly what keeps them hidden. + when: + - status: [success, failure] diff --git a/.editorconfig b/.editorconfig index 734c5ee..82fac74 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,6 +11,10 @@ insert_final_newline = true [*.md] trim_trailing_whitespace = false +# Markdown code fences hold verbatim content (logs, snippets) whose +# indentation must not be forced to the source indent rules. +indent_style = unset +indent_size = unset # Ignore paths [/collections/ansible_collections/community/**] diff --git a/.editorconfig-checker.json b/.editorconfig-checker.json new file mode 100644 index 0000000..f7abef3 --- /dev/null +++ b/.editorconfig-checker.json @@ -0,0 +1,3 @@ +{ + "Exclude": ["^LICENSE\\.md$", "^benchmark/", "^docker/reprex/"] +} diff --git a/.gitignore b/.gitignore index 64c54dd..f4124a2 100644 --- a/.gitignore +++ b/.gitignore @@ -95,4 +95,10 @@ terraform.rc .envrc exec.sh exec.R -docs/_site \ No newline at end of file +docs/ +local/test.R +.DS_Store +docs/ + +# Superpowers SDD scratch (briefs, reports, ledger) +.superpowers/ diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml new file mode 100644 index 0000000..032c0ea --- /dev/null +++ b/.markdownlint-cli2.yaml @@ -0,0 +1,7 @@ +# Rules live in .markdownlint.yaml; this file only sets cli2 options. +ignores: + - LICENSE.md + - docs/superpowers/** + # Internal design docs (specs/plans) are not user-facing reference material. + - specs/** + - plans/** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2bf38b9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,60 @@ +# cSpell:ignore autofix autoupdate +# Excluded: verbatim GPL license and auxiliary shell scripts with intentional +# in-string formatting (reformatting would corrupt their output). +exclude: | + (?x)^( + LICENSE\.md| + benchmark/| + docker/reprex/ + ) +# The `^local/patches/.*\.patch$` excludes below keep unified diffs byte-exact: +# a context line for a blank line is a single space, and stripping it (or +# appending a newline) makes `git apply` reject the patch, which surfaces as +# "patch did not apply cleanly" at build time rather than as a lint failure +# here. The exclusions are per-hook, not global, so `validate patch registry` +# still runs when a patch changes. +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + exclude: ^local/patches/.*\.patch$ + - id: trailing-whitespace + exclude: ^local/patches/.*\.patch$ + args: + - --markdown-linebreak-ext=md + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.23.2 + hooks: + - id: markdownlint-cli2 + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.9.6 + hooks: + - id: prettier + - repo: https://github.com/posit-dev/air-pre-commit + rev: 0.11.0 + hooks: + - id: air-format + - repo: https://github.com/editorconfig-checker/editorconfig-checker + rev: v3.11.2 + hooks: + - id: editorconfig-checker + exclude: ^local/patches/.*\.patch$ + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.38.0 + hooks: + - id: yamllint + args: [--strict, -c=.yamllint.yaml] + - repo: local + hooks: + - id: yaml-file-extension + name: Check if YAML files has *.yaml extension. + entry: YAML filenames must have .yaml extension. + language: fail + files: .yml$ + - id: validate-patches + name: validate patch registry + entry: Rscript local/validate-patches.R + language: system + files: ^local/(patches/|validate-patches\.R$) + pass_filenames: false diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..0e4b780 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +LICENSE.md diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 86513fb..9591b83 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,71 +2,79 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/http" { - version = "3.5.0" + version = "3.6.1" hashes = [ - "h1:JSui9Kmmvc4TDx6QhxyQaW5Mbka+6e2q9hoWvcFRI9M=", - "h1:S0yLjN9Fu6sZ+3SWq2zcFjNrZS8EyeGnBBlkErpNNXE=", - "h1:StUoUwy4UcNDl1gn1kuwOl6B37w2zhZTcY8/jabWJy0=", - "h1:YTB70v5NGaoI3T11B7pnwBnSL1ZEN0mp23Yrto05aY0=", - "h1:bSHOZnfmhragoS/ZaMQEffXMipITkKF6jhcmpxAWh9c=", - "h1:czB6w6tVJrPWFImze+cf2fIGqcJ3VxugKRgc6bDxZoE=", - "h1:dm7SwSSfnRFNwgKO7HOy98OEANn9Z6TI7rQAOaletsI=", - "h1:eClUBisXme48lqiUl3U2+H2a2mzDawS9biqfkd9synw=", - "h1:jXH9n2GTk7N9tzTA6inAgftL6wqQi2c0+bTKRw74xfc=", - "h1:yvwvVZ0vdbsTUMru+7Cr0On1FVgDJHAaC6TNvy/OWzM=", - "zh:0a2b33494eec6a91a183629cf217e073be063624c5d3f70870456ddb478308e9", - "zh:180f40124fa01b98b3d2f79128646b151818e09d6a1a9ca08e0b032a0b1e9cb1", - "zh:3e29e1de149dc10bf78620526c7cb8c62cd76087f5630dfaba0e93cda1f3aa7b", - "zh:4420950200cf86042ec940d0e2c9b7c89966bf556bf8038ba36217eae663bca5", - "zh:5d1f7d02109b2e2dca7ec626e5563ee765583792d0fd64081286f16f9433bd0d", - "zh:8500b138d338b1994c4206aa577b5c44e1d7260825babcf43245a7075bfa52a5", - "zh:b42165a6c4cfb22825938272d12b676e4a6946ac4e750f85df870c947685df2d", - "zh:b919bf3ee8e3b01051a0da3433b443a925e272893d3724ee8fc0f666ec7012c9", - "zh:d13b81ea6755cae785b3e11634936cdff2dc1ec009dc9610d8e3c7eb32f42e69", - "zh:f1c9d2eb1a6b618ae77ad86649679241bd8d6aacec06d0a68d86f748687f4eb3", + "h1:7fra+jbUXbG5wMaz5L6RKMBv6gIuenJcBiIww87GoXo=", + "h1:BzSV3Ie9XMXF7sZHKAS54CzV95v5GBZNhQ4nrprUgfQ=", + "h1:CkrbSKS+pNVgvP3bMe2WoYHaFCIWJUkCtlC5vyTAdLI=", + "h1:FboJEwgVIRmqUJkjEoSRpfavVCJotUTe1zzT+pBzcV0=", + "h1:GlXELDLSZrdV3Svx1jjEBAXiJFkkdF/Hgx1qrmRK5hE=", + "h1:VuXFI2IcnZ6t4sDqtvkuIzbPK1CJQa0CkaM0MBuOlSU=", + "h1:WmL2nFQbSzRiDsDiwUbZbBp/cxGQrXrZnB7A4LGSvJU=", + "h1:Zdj26awWJ+m8kMoAMhItsIDcDFg81PWgKKJrvNi3WOI=", + "h1:lHvYYIumeZ+KJgCrmhCLnRGzrvNMjSHBTdV24coyMEc=", + "h1:pAOYMwA6Zki3ujAbG20b49u1IYXdBz56pW1JHqKdX5U=", + "h1:qi9GUp2+g69C8zY6Z68u4fWPwcZlDTa/CtdhvPgWbMA=", + "h1:w5A3xJ2mowj2wgiE3oNfOI0lFJf5X9IgxOJ6SErMczA=", + "h1:xAO03iJyuNGSOqolIcXcofH8cocgUb6Cnzq6yivbWcI=", + "h1:xXigGPwW8MlrB6Br2ce+Bf35BbdzdPKa97T/q/xrrcA=", + "h1:yDYzQ2ncNE9q1288xAgflIPq98bOOYsAb9tq6vkbFzw=", + "zh:129d7d5944b31f40916b1ca86b31cef65a6b02fd36008809d13c561894bfedb9", + "zh:24631608288b0bcd35c1fc63dc5839572254d881c0589ebba036be52b2fc04d6", + "zh:5a0f100d7eb256463fe5a2aa1a7128391147b2c5fc895ff1b1ef54fc5b8f15ab", + "zh:6a8a1126ab9ca61be3b62ec184f6b2e7cbf01cde810acc548cee27d71277b09b", + "zh:6fffef54fd3aada85c074e34d41386aa09c79a308a4679132da31c7272733c6c", + "zh:899c992d2aa290ebe1304da0289c5104a630bca421cc6a88ce55bf0960aab1b4", + "zh:960fd6c2847859a843dd9dbfc95a0037a470aa744094d155a38a057175cf1502", + "zh:9b032b685a644634158ace5529e260dfc4447a280056f02858d205ea26753f69", + "zh:bba5477c97020c28ed12d4f5b36be2c1bf14d946d7e44b3690e5c23cd7ddf5e6", + "zh:c2ff6c33efef52441fa3485137972792031626dcabca2b1d8b6527d45f185279", + "zh:cd492b3dfd150de6bef8ad505293d3d53c6c907706f36d0e497b4fc027d8edb6", + "zh:d1f832bc33c42781454dc020c6937e7d0133155a5a9f64335309d64a34b36bb7", + "zh:d42e9cbebc77643556853b1ebbec14cefe70c57ee86cd3b8c71fbe7f523f07df", + "zh:d4c0466f578d7f990646bb0847e31ba3797f2100b6380ee1ca736887546c7621", + "zh:d9d81ecebfe6edabdd4c527f3f4debde3e052ff87c5ef4c67497ab3d7539e424", ] } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.7.2" - constraints = "~> 0.7" + version = "0.18.2" + constraints = "~> 0.18" hashes = [ - "h1:1i5uplGeSbFVZkIGvmo/H5UFzKuDJIAID6ubI8EvHTc=", - "h1:51+8C6joAlqSrVoM4JgJrbfiSO/uIkEtMjToXmMNJ+4=", - "h1:7dUGBxfASC627BGnE+YZoxkFJY+JHSvJaop6TQRGkM8=", - "h1:9YaYxXUTnCJVNuteA8piYWcrscuCQr/7GYEt0MQVnHk=", - "h1:BGZTM+0QpxZKjaIE1lJc7DMDY002NyxZ/X9hfAEr8UI=", - "h1:GafIKF87KXp9F5tumBkpaYpZnrFL2s6e35jzWhY1Vn8=", - "h1:HfbKHb+pEdTpmsN17rKPuOZrCOB0781mwemVQ0kvxc0=", - "h1:KqzAiiYHMHSNyyRp2ItejrX4T0j1SX1nFaTa9MN8l9o=", - "h1:KtWneY9VCN9D9kANRwdKWmxe49Hngap1fJK+SFqh9UI=", - "h1:LyLXiQ488EeYsUDi8UIjpqen/YiNB6q3A4ibZZwzLbk=", - "h1:OEdBcDqR/oDcyZ9Y1a+/UGXReelogCj1/DxVhzaEXAI=", - "h1:b0SMwkAMFIamrobkzxvo9tsWbBeLUsjE+ZpCy8g/GHg=", - "h1:kg8ZkJFpU1dKIvSiciSZJUL9qkm8QxA9bu2k4BkzAy4=", - "h1:lhiSUzFUZdvcpyDL0uXPA034Ue+YVMYpiBzUqyz8H3c=", - "h1:n6grUzLUt0sN8Q21nh/THw6Bd7p70fe/w2PyXeLo0bE=", - "h1:sGGwDRK7t8UF++47fnT31hdyX5XhGOZBOD8JNMMwYE4=", - "h1:xvWfdq1csFlLLd/bo97VT59J0hLCPhJEgsfU+YZptfM=", - "h1:zQGnnYerpxIBlUDhjiRqelREGejYLTXvS4DUB2sW6pA=", - "zh:2f2a1593bfe4ce42b8fa5fc5f9d86c4a6bdf02ec34b1570df0ca64c9ad649fe5", - "zh:399b7cb3d9367886f6fb48a46ee2032477afb4758c4b515941f0cbc350932327", - "zh:6e43b92851fc1069c34f67f570733277ad975b49b8db438d2b847ed8dbd21d97", - "zh:886a9890dfd77826694addad41265142528c200a0ab90302ab87b4105d5c38be", + "h1:3rZl+Co3WMpwj8SciPaCNXoGA31aSoqp6iweLarr5m4=", + "h1:6d9cKLhz8QOZ4R5yVX1G0TsWL+K1Abtfbm3xngndxto=", + "h1:EBjjkfp5Gx7nXP1DVO+tLhsow6fEUvaIjsCEFRT2fY8=", + "h1:Nu2DoHGOv2YN7ag4kFGpfnPeRDh6bzWqY5anW+ETGpM=", + "h1:OnvZxg28m4/UJeEhHVLU4kM2MZ704sxRzYfLWlLxnhA=", + "h1:PiCse2/UcB7nkPxosveHsJN/jKdBC8AH6tKTxcHSYKw=", + "h1:QAahdtlDBUon7eMwNN0D2V6CxgasOXIi+9/UExik6Sg=", + "h1:Su5z0A7/UaSm/E7FJnFjpDVQaa1Ju5+fZ8Mirf8E+k8=", + "h1:UA3a78FJAPAGqCCvlIg9ekPltpVsrmEhwFLalWCFnew=", + "h1:XAlCTNHRtgUkNjdUItkiak6ajjT7wFJzJN8frXKD5Ms=", + "h1:ZgLBOPebYxH059z1cGHmjYO8CTf+tbWPb3VbO97S2YM=", + "h1:anR91C2F6NDJoQQQIy6KHChodnTaSKnApSWSGM4jSX0=", + "h1:gVmaNmIu4gEiITM+CAb66e+zncAqzNBYkniTZfvxZ5Y=", + "h1:pODlGrkPqHV4yhXiO7LLLu11HtcuxOAB2zUx3B8w1vI=", + "h1:qEYeHEKVRcc78q5xiRGJSY8DGQpLj40KafEXUxFfaQc=", + "h1:qdVz+O0lLHhyf5YX3ujmoVvAGlKqvi+YOPUzVTqpKzY=", + "h1:yTrPkdc9eQkxfPLBYydFf0fpcjarP5w0sdLPzekD9RQ=", + "zh:0fe3987c927d81196c97504470ce4d26c3ad0014f8ee3d0c1be422d08cfcf49c", + "zh:15c36dc69e058876921ac887213e1716217d159b7ee7f0f233e21fb35be85178", + "zh:29d58d7b76dcb142a06d4edd15b8500fe6c1afb7f7c056ada17e2d42bb999fbd", + "zh:33d313836c0e985186b3456c0946e062b27cacfcb08611d0a394f36db9ee1aef", + "zh:47e085e52e9b24ad85fa2988dbb8604256a970a6f53f7fa6aab04d8ae756a738", + "zh:4ba4f87571ca72fbc6c24ab71f2f7b5a086938262e2d8e5c0b39701ed52f8bbc", + "zh:4c6bae97b543c5b328e1ecbcf7c976351b4b381654e9d3e569270dcab3ba816c", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:90d11017b5a65f0b236307981c7917127f2b76f3f87a02183e8f21a14948ae0c", - "zh:90e11fd58cfc5c9ba0447cffb804ab3a6705889a1f783db13943af54a48dd9e1", - "zh:a04bc736192ccb4f5b1d161c96ceb4d52a7cb90adb895c0fe35510cfd1661a9d", - "zh:a8e0e177c64d57325d7861817c3d414aa2b5d6862961f3f7adcad728c0501ca9", - "zh:af471f71e3054c2b03dd456c848de17fbefd25a973ba3644ba5250f9fc28e207", - "zh:c4ebe7679a96881570e3d928ee57e442f7e71ffa74b0aa2665dba1297a174837", - "zh:c68cf3de1372d0f9d1b8d80653f7f03c91f96db74f3cb1bdf7fa1c03d1777fa1", - "zh:cabb18da0d279acb8cd4ac494d7f2ff01fad75fad6c426623636e40cd6933180", - "zh:d23a5b242bd43369dde575bac5d1492e0e8b283620adbd0a91f3db6b2102c67f", - "zh:d327724d0116538be68f200e54e80035480cef429312eb9b4e39f4b765c90b1b", - "zh:df5adbe5ab18349f6f8e282a3d7a3b3fac81eb381e63e911943c83c38100b780", - "zh:e87731fa95a8c9fbfc161b605bc3697b938d2593d9a2690f5e6d5add8a2d6b5b", - "zh:f153303b99e7c1c10b5a65358a556f7aa7b04bb226b0454d7d6fecbcae7e73d2", - "zh:f1c4d21f046cdce6c29a85132cd64b9e6eef19158b1bb6b9f8412e1f59a9f2d6", + "zh:9ba7ab56537963db2449d217528a751469c9dc4e413dec3e3d63fd7daf3db4ef", + "zh:a3c48eda7e11b03b831f2a639797524bb335f155f0dff0e999cf3496994da8b3", + "zh:aab8f4814d55ef8c6c285d2496ae412437017d0fd1be70106f7b3a4a6e764feb", + "zh:b92b9beacf71ae894717c2036ceb68db52c9c43af4a01b8209eceae9f91a2c8e", + "zh:da389285938e22e1249e6a00cebf12a9f67334743f0b3f66399e6881028bda11", + "zh:dadcc33d06e6f64a17d1965478af5e8bbdc971e92ec9b14e384c5d43861d63f7", + "zh:e090c916e6da685125194af4f0a1fd772494a0c63f3f16ab3741782e17f4a8f9", + "zh:e5881e00fa970c08e66e8079b47d69b76def6e7ff3bdc35b68d7811e5ece55d1", + "zh:eeebb25a066a6287d545c91c0fc264acee5b28174d0979faeebdac3bd14f0fff", + "zh:f368195116c9ce0181aa7527c51ae5e7ab23d42fb966acf4eddca344621ae339", ] } diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..d0370a3 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,19 @@ +rules: + comments: + require-starting-space: false + ignore-shebangs: true + min-spaces-from-content: 1 + braces: + min-spaces-inside-empty: 0 + max-spaces-inside-empty: 0 + min-spaces-inside: 0 + max-spaces-inside: 1 + document-start: + present: false + indentation: + spaces: 2 + indent-sequences: true + # .crow/* workflows embed long single-line R commands (>1k chars) + line-length: disable + new-lines: + type: unix diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..faae5f1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,14 @@ +# CLAUDE.md + +Crow CI pipelines (`.crow/`) and local tooling (`local/`) that build CRAN binary packages (incl. Alpine/musl) and upload them to Backblaze B2. + +## Conventions + +- **PRs:** the remote is Forgejo on `codefloe.com`; use `fj -H codefloe.com` (not `gh`). +- **Storage:** Backblaze B2 bucket `devxy-r-builds` (endpoints configured in the `.crow/` pipelines). + +## Gotchas + +- **B2 requires authentication for the list-bucket API.** Anonymous GET only works for individual public-read objects — an empty listing means missing credentials, not an empty bucket. +- Weekly-rebuild pipelines run for hours; watch them as background tasks and fetch Crow logs yourself instead of having the user paste progress. +- musl builds of packages with bundled native deps (e.g. RcppParallel/TBB) recur as failures; check for an existing patch before re-deriving a fix. diff --git a/Justfile b/Justfile deleted file mode 100644 index 8b13789..0000000 --- a/Justfile +++ /dev/null @@ -1 +0,0 @@ - diff --git a/README.md b/README.md index 71647df..989069e 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,14 @@ This project offers a framework for creating R package binaries on Linux across It achieves this through the integration of several components: -- **R package `bincraftR`** +- **R package `bincraft`** - **Containerfiles** that define the build toolchain for each distribution - **S3 storage** for storing the compiled binaries - **PostgreSQL database** for recording build logs ## R Package -The R package [`bincraftR`](https://gitlab.com/devxy/r-package-binaries/bincraftr) powers everything. +The R package [`bincraft`](https://gitlab.com/devxy/r-package-binaries/bincraft) powers everything. It provides the following functionality: - build binaries @@ -25,7 +25,7 @@ It provides the following functionality: - update the package index files (`PACKAGES*`) - store build metadata, including error logs, in a PostgreSQL database -See the function reference on the [pkgdown site](https://devxy.gitlab.io/r-package-binaries/bincraftr/) for a full overview. +See the function reference on the [pkgdown site](https://devxy.gitlab.io/r-package-binaries/bincraft/) for a full overview. The focus of the R package is on usability rather than minimizing dependencies. The individual containerfiles include the package along with its dependencies. @@ -53,6 +53,28 @@ For every package+tag combination: 1. Archive old package versions and keep the latest one in the root 1. Delete local binaries after successful upload +## Patching packages + +Some CRAN packages fail to compile on specific platforms due to compiler- or OS-specific issues unrelated to the package itself. +The canonical example is `RcppParallel`, whose bundled TBB is linked in a way that lets a system TBB on the build host shadow it, so the published binary depends on a library the consumer does not have. +Because such packages are often transitive dependencies of many others, a single failure cascades: all dependents fail even though nothing is wrong with the dependent itself. + +To address this, frequently-failing packages can be "patched" before they are installed — whether as a direct build target or a transitive dependency pulled in by `pak`. + +The patch registry lives in `local/patches/registry.json`. +Each entry specifies a package and the platforms/versions it applies to, along with either lightweight build-time overrides (environment variables, configure arguments, Makevars) or a source diff (for deeper fixes). +See `local/patches/README.md` for the complete schema. + +Patching uses a two-tier approach: + +1. **Lightweight overrides:** environment variables, configure arguments, or Makevars settings applied during build — typically version-independent and fast. +2. **Source diffs:** unified diff patches applied to the unpacked source before building — more powerful but version-pinned. + +The system is implemented in `bincraft`: when a package needs patching, `bincraft` pre-builds it with the patch and serves the patched binary to `pak`, ensuring transitive dependents receive the fixed package. +This way, the fix cascades to all packages that depend on it. + +For the design rationale and architecture, see `specs/2026-06-30-package-patching-design.md`. + ## Build Environment Binaries are built on a mixed-architecture Kubernetes cluster using CI. @@ -68,7 +90,7 @@ Currently, the {cranlike} and {desc} packages only work with files on a local fi This is infeasible if the goal is to store binaries in S3. Storing binaries permanently on a disk-based file system would incur significantly higher costs, especially when operating in the cloud. -Hence, [{cranlike}](https://devxy.gitlab.io/r-package-binaries/bincraftr/) and [{desc}](https://github.com/pat-s/desc/tree/description-from-remote) were forked to handle files in S3 (through {s3fs}). +Hence, [{cranlike}](https://devxy.gitlab.io/r-package-binaries/bincraft/) and [{desc}](https://github.com/pat-s/desc/tree/description-from-remote) were forked to handle files in S3 (through {s3fs}). ### Resources @@ -90,9 +112,9 @@ Processing updates with an existing database file takes around 5 minutes. ### Inferring System Dependencies -R package dependencies and their system dependencies are installed through {pak}. -{pak} allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. -Additionally, it automatically infers package dependencies using JSON rules from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). +R package dependencies and their system dependencies are installed through [uvr](https://github.com/nbafrank/uvr). +`uvr` allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. +Additionally, it automatically infers package dependencies using JSON rules vendored from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). Not all R packages specify required system dependencies in their DESCRIPTION file, and not all listed dependencies have existing rules in `rstudio/r-system-requirements`. For Alpine, no rules existed until recently, establishing a foundation for semi-automated package installation on Alpine Linux. @@ -130,12 +152,13 @@ A Shiny dashboard providing a search functionality of the database and grouped s Is supported by writing `Meta/archive.rds` during each package index update, listing all available archived packages. -### `pak::pak(package@version)` +### `uvr add package@version` -`pak` searches for `Archive/` and can install all versions it finds. +Clients that resolve archived versions search for `Archive/` and can install all versions they find. +This holds for `uvr` as well as for older `pak`-based clients. Ensure to use a clean cache if other repositories have been used previously. -If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. +If in doubt or when testing, call `uvr cache clean`. ## Lessons Learned @@ -158,7 +181,7 @@ If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. ## URL Composition and Platform Identifiers -Platform identifiers have been aligned with those used in to ensure proper recognition by the automatic syslib dependency installer of `pak`, specifically via the environment variable `PKG_SYSREQS_PLATFORM`: +Platform identifiers have been aligned with those used in , which `uvr` vendors and uses for its automatic syslib dependency installer (`uvr sync --install-system-deps`): - redhat-9 - redhat-8 @@ -186,20 +209,20 @@ Below is a collection of raw errors observed during the build process:
-``` +```text * installing to library '/tmp/Rtmp7WPw19/temp_libpath114b846b58'\n* installing *source* package 'ade4' ...\n** using staged installation\nERROR: a 'NAMESPACE' file is required\n* removing '/tmp/Rtmp7WPw19/temp_libpath114b846b58/ade4'\n" ``` Tag does not have a NAMESPACE file and hence cannot be built. -``` +```text "* installing to library '/tmp/RtmpLcCitS/temp_libpath1146aabbe92'\nERROR: dependency 'tripack' is not available for package 'alphahull'\n* removing '/tmp/RtmpLcCitS/temp_libpath1146aabbe92/alphahull'\n" ``` Dependency not available: Either because the dependency was not declared or errored itself during installation. -``` - In function '\033[01m\033[KRcpp::List solveRRBLUP(const mat&, const mat&, const mat&)\033[m\033[K':\n\033[01m\033[KMME.cpp:162:61:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope\n 162 | double ll = -0.5*(double(optRes[\"objective\"])+df+df*log(2*\033[01;31m\033[KPI\033[m\033[K/df));\n | \033[01;31m\033[K^~\033[m\033[K\n\033[01m\033[KMME.cpp:\033[m\033[K In function '\033[01m\033[KRcpp::List solveRRBLUPMV(const mat&, const mat&, const mat&, int, double)\033[m\033[K':\n\033[01m\033[KMME.cpp:277:31:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope; did you mean '\033[01m\033[KHI\033[m\033[K'?\n 277 | ll -= double(n*m)/2.0*log(2*\033[01;31m\033[KPI\033[m\033[K);\n | \033[01;31m\033[K^~\033[m\033[K\n | \033[32m\033[KHI\033[m\033[K\nmake: *** [/opt/R/4.4.1/lib/R/etc/Makeconf:204: MME.o] Error 1\nERROR: compilation failed for package 'AlphaSimR'\n* removing '/tmp/RtmpclI5CE/temp_libpath11135d215d5/AlphaSimR'\n +```text + In function '\033[01m\033[KRcpp::List solveRRBLUP(const mat&, const mat&, const mat&)\033[m\033[K':\n\033[01m\033[KMME.cpp:162:61:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope\n 162 | double ll = -0.5*(double(optRes[\"objective\"])+df+df*log(2*\033[01;31m\033[KPI\033[m\033[K/df));\n | \033[01;31m\033[K^~\033[m\033[K\n\033[01m\033[KMME.cpp:\033[m\033[K In function '\033[01m\033[KRcpp::List solveRRBLUPMV(const mat&, const mat&, const mat&, int, double)\033[m\033[K':\n\033[01m\033[KMME.cpp:277:31:\033[m\033[K \033[01;31m\033[Kerror: \033[m\033[K'\033[01m\033[KPI\033[m\033[K' was not declared in this scope; did you mean '\033[01m\033[KHI\033[m\033[K'?\n 277 | ll -= double(n*m)/2.0*log(2*\033[01;31m\033[KPI\033[m\033[K);\n | \033[01;31m\033[K^~\033[m\033[K\n | \033[32m\033[KHI\033[m\033[K\nmake: *** [/opt/R/4.4.2/lib/R/etc/Makeconf:204: MME.o] Error 1\nERROR: compilation failed for package 'AlphaSimR'\n* removing '/tmp/RtmpclI5CE/temp_libpath11135d215d5/AlphaSimR'\n ``` Compiler error: Possible reasons: too old CXX code which cannot be compiled anymore with CXX14 or CXX17. @@ -215,8 +238,8 @@ internal error 1 in memDecompress Solution: ```sh -rm -rf /mnt/cache/R-pkgs/pak /mnt/cache/pkgcache/ /root/.cache/R/ -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +uvr cache clean +rm -rf /mnt/cache/uvr /root/.cache/R/ ```
@@ -230,26 +253,26 @@ For others it might be due to exotic external dependencies which require manual Help in resolving these issues are highly welcome! -| Name | Platform | Arch | Reason | Solved via | Date Created | Date Solved | -| ------------- | -------------- | ------- | -------------------------------- | ---------- | ------------ | ----------- | -| Apollonius | redhat-8 | | gmp missing | 2024-11-15 | | -| doBy | alpine-320 | amd | hangs | 2024-11-23 | | -| later | ubuntu 22 & 24 | amd | hangs for early versions - xfvb? | 2024-11-23 | | -| CoTiMA | ubuntu 22 & 24 | amd | OOM? | 2024-11-23 | | -| FrF2 | alpine | arm | hangs | 2024-11-23 | | -| FrF2.catlg128 | alpine | arm | hangs | 2024-11-23 | | -| DoE.base | alpine | arm/amd | hangs | 2024-11-23 | | -| eha | alpine | amd | hangs | 2024-11-23 | | -| gRain | alpine | amd | hangs | 2024-11-23 | | -| gRbase | alpine | amd | hangs | 2024-11-24 | | -| IDPmisc | alpine | amd | hangs | 2024-11-24 | | -| RVAideMemoire | alpine | amd | hangs | 2024-11-25 | | -| pbkrtest | alpine | arm | hangs | 2024-11-25 | | -| seewave | alpine | arm | hangs | 2024-11-25 | | -| spdep | alpine | arm | hangs | 2024-12-03 | | -| compareGroups | alpine | arm | hangs | 2024-12-11 | | -| SNPassoc | alpine | arm | hangs | 2024-12-11 | | -| surveillance | alpine | arm | hangs | 2024-12-13 | | +| Name | Platform | Arch | Reason | Date Created | Date Solved | +| ------------- | -------------- | ------- | -------------------------------- | ------------ | ----------- | +| Apollonius | redhat-8 | | gmp missing | 2024-11-15 | | +| doBy | alpine-320 | amd | hangs | 2024-11-23 | | +| later | ubuntu 22 & 24 | amd | hangs for early versions - xfvb? | 2024-11-23 | | +| CoTiMA | ubuntu 22 & 24 | amd | OOM? | 2024-11-23 | | +| FrF2 | alpine | arm | hangs | 2024-11-23 | | +| FrF2.catlg128 | alpine | arm | hangs | 2024-11-23 | | +| DoE.base | alpine | arm/amd | hangs | 2024-11-23 | | +| eha | alpine | amd | hangs | 2024-11-23 | | +| gRain | alpine | amd | hangs | 2024-11-23 | | +| gRbase | alpine | amd | hangs | 2024-11-24 | | +| IDPmisc | alpine | amd | hangs | 2024-11-24 | | +| RVAideMemoire | alpine | amd | hangs | 2024-11-25 | | +| pbkrtest | alpine | arm | hangs | 2024-11-25 | | +| seewave | alpine | arm | hangs | 2024-11-25 | | +| spdep | alpine | arm | hangs | 2024-12-03 | | +| compareGroups | alpine | arm | hangs | 2024-12-11 | | +| SNPassoc | alpine | arm | hangs | 2024-12-11 | | +| surveillance | alpine | arm | hangs | 2024-12-13 | | ## CDN Settings diff --git a/build-all-versions-install-deps.yaml b/build-all-versions-install-deps.yaml new file mode 100644 index 0000000..dd02e8a --- /dev/null +++ b/build-all-versions-install-deps.yaml @@ -0,0 +1,63 @@ +# ### Required extra vars +# # ARCH (amd64) +# # OS (alpine) +# # OS_VERSION (3.22) +# # R_VERSION (4.5) +# # when: +# # - event: manual +# # evaluate: 'task == "build-all-arm64"' +# # - event: manual +# # evaluate: 'task == "build-all-amd64"' + +# skip_clone: true + +# labels: +# platform: linux/amd64 +# backend: kubernetes + +# steps: +# - name: 'Install common R deps' +# image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" +# pull: true +# environment: +# REPO_RO_TOKEN: +# from_secret: REPO_RO_TOKEN +# GITHUB_PAT: +# from_secret: GITHUB_PAT +# HETZNER_S3_ACCESS_KEY_K3S: +# from_secret: HETZNER_S3_ACCESS_KEY_K3S +# HETZNER_S3_SECRET_KEY_K3S: +# from_secret: HETZNER_S3_SECRET_KEY_K3S +# # normal env vars +# GIT_USER: pat-s +# # set the location of uvr's caches, which persist the R package dependencies needed to install the packages themselves +# # UVR_CACHE_DIR: /mnt/cache/uvr/cache +# # UVR_PACKAGES_DIR: /mnt/cache/uvr/packages +# R_LIBS_USER: /mnt/cache/R-pkgs +# CCACHE_DIR: /mnt/cache/ccache +# volumes: +# - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache +# commands: +# - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . +# - mkdir -p /mnt/cache/R-pkgs +# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/uvr +# - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft +# - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' +# # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine322', 'amd64'); saveRDS(pkgs, '/mnt/cache/pkgs_amd64.rds')" +# - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine323', 'arm64'); saveRDS(pkgs, '/mnt/cache/pkgs_arm64.rds')" +# backend_options: +# kubernetes: +# resources: +# requests: +# memory: 1000Mi +# cpu: 1000m +# limits: +# memory: 4500Mi +# cpu: 8000m +# nodeSelector: +# kubernetes.io/arch: ${ARCH} +# tolerations: +# - key: 'CI' +# operator: 'Equal' +# value: 'true' +# effect: 'NoSchedule' diff --git a/cdn.tf b/cdn.tf index cdd5f90..96877b2 100644 --- a/cdn.tf +++ b/cdn.tf @@ -1,99 +1,140 @@ # https://registry.terraform.io/providers/BunnyWay/bunnynet/latest/docs/resources/pullzone # terraform import bunnynet_pullzone.devxy-r-binaries cran -resource "bunnynet_pullzone" "devxy-r-binaries" { - name = "cran" +# resource "bunnynet_pullzone" "devxy-r-binaries" { +# name = "cran" - origin { - type = "OriginUrl" - url = "https://devxy-r-package-binaries-hel1.hel1.your-objectstorage.com" - } +# origin { +# type = "OriginUrl" +# url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" +# } - routing { - tier = "Standard" - } +# routing { +# tier = "Standard" +# } - s3_auth_enabled = true - s3_auth_key = var.HETZNER_S3_ACCESS_KEY_K3S - s3_auth_secret = var.HETZNER_S3_SECRET_KEY_K3S - s3_auth_region = "hel1" +# s3_auth_enabled = true +# s3_auth_key = var.B2_S3_ACCESS_KEY +# s3_auth_secret = var.B2_S3_SECRET_KEY +# s3_auth_region = "eu-central-003" - cache_enabled = true - cache_errors = true - request_coalescing_enabled = true - block_post_requests = true +# cache_enabled = true +# cache_errors = true +# request_coalescing_enabled = true +# block_post_requests = true - limit_requests = 60 - limit_connections = 10 +# limit_requests = 500 +# limit_connections = 50 - safehop_enabled = true +# safehop_enabled = true - add_canonical_header = true +# add_canonical_header = true - cache_stale = ["offline", "updating"] - use_background_update = true +# cache_stale = ["offline", "updating"] +# use_background_update = true - block_ips = [ - "185.172.53.0" - ] +# block_ips = var.cdn_block_ips - # 50 TB - limit_bandwidth = 50000000000000 +# # 50 TB +# limit_bandwidth = 50000000000000 - permacache_storagezone = bunnynet_storage_zone.devxy-r-binaries.id +# permacache_storagezone = bunnynet_storage_zone.devxy-r-binaries.id - # rec from docs: https://www.backblaze.com/docs/cloud-storage-integrate-bunnynet-with-backblaze-b2 - block_root_path = true -} +# # rec from docs: https://www.backblaze.com/docs/cloud-storage-integrate-bunnynet-with-backblaze-b2 +# block_root_path = true +# } -resource "bunnynet_pullzone_hostname" "devxy-r-binaries" { - pullzone = bunnynet_pullzone.devxy-r-binaries.id - name = "cran.devxy.io" - force_ssl = true - tls_enabled = true -} +# resource "bunnynet_pullzone_hostname" "devxy-r-binaries" { +# pullzone = bunnynet_pullzone.devxy-r-binaries.id +# name = "cran.devxy.io" +# force_ssl = true +# tls_enabled = true +# } ### cran.rpkgs.com +# The edge middleware that resolves the bare cran.rpkgs.com form to an +# / slot and routes PACKAGES* to the per-R-minor slot. The source of +# truth is edge/rpkgs-router.ts; `tofu apply` publishes a new release. +# +# The script pre-dates this configuration, so it is adopted rather than created: +# tofu import bunnynet_compute_script.rpkgs_router 29277 +resource "bunnynet_compute_script" "rpkgs_router" { + type = "middleware" + name = "rpkgs-router" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +# Slots ("/", comma separated) whose per-minor index bincraft has +# already republished as a union of the per-minor and flat slots. Routing to a +# slot that is not listed here would hide every package the per-minor index does +# not carry, so this stays empty until a slot has been backfilled. +resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + default_value = "" + required = false +} + resource "bunnynet_pullzone" "cran_rpkgs_com" { name = "cran-rpkgs" + cache_errors = false + + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true + origin { - type = "OriginUrl" - url = "https://devxy-r-package-binaries-hel1.hel1.your-objectstorage.com" + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router.id } routing { - tier = "Standard" + filters = [ + "scripting", + ] } s3_auth_enabled = true - s3_auth_key = var.HETZNER_S3_ACCESS_KEY_K3S - s3_auth_secret = var.HETZNER_S3_SECRET_KEY_K3S - s3_auth_region = "hel1" + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" cache_enabled = true - cache_errors = true request_coalescing_enabled = true block_post_requests = true - limit_requests = 60 - limit_connections = 10 + # Set on the zone since before this configuration existed; declared here so + # `tofu apply` stops silently removing it. + # + # The router makes it redundant on paper: the only UA-dependent responses it + # produces are redirects, and those carry `Cache-Control: no-store`, while + # their targets are concrete per-slot, per-minor URLs whose content depends + # only on the path. Dropping it would also be a real win, because otherwise + # every distinct R version string keys its own copy of every tarball. + # + # It stays for now anyway: it is the second line of defence against the one + # failure that would be quiet and confusing (an R 4.6 client served the 4.5 + # index), and removing it is worth doing on its own once per-minor routing is + # confirmed live, not as a side effect of enabling that routing. + cache_vary_headers = ["User-Agent"] + + limit_requests = 5000 + limit_connections = 1000 safehop_enabled = true add_canonical_header = true - cache_stale = ["offline", "updating"] - use_background_update = true + cache_stale = ["offline", "updating"] - block_ips = [ - "185.172.53.0" - ] + block_ips = var.cdn_block_ips # 50 TB limit_bandwidth = 50000000000000 - permacache_storagezone = bunnynet_storage_zone.devxy-r-binaries.id + # permacache_storagezone = bunnynet_storage_zone.devxy-r-binaries.id # rec from docs: https://www.backblaze.com/docs/cloud-storage-integrate-bunnynet-with-backblaze-b2 block_root_path = true @@ -106,56 +147,69 @@ resource "bunnynet_pullzone_hostname" "cran_rpkgs_com" { tls_enabled = true } -resource "bunnynet_storage_zone" "devxy-r-binaries" { - name = "devxy-r-binaries-storage" - region = "DE" - zone_tier = "Standard" - # Los Angeles and Singapore - replication_regions = ["LA", "SG"] +# Alliance SwissPass historically used a separate, manually configured pull +# zone. Adopt it so both public repositories use the same B2 origin, middleware +# release and cache behavior. +import { + to = bunnynet_pullzone.cran_allianceswisspass + id = "3265648" } -resource "bunnynet_pullzone" "r-package-binaries-docs" { +resource "bunnynet_pullzone" "cran_allianceswisspass" { + name = "cran-allianceswisspass" - name = "r-package-binaries-docs" + cache_errors = false + cache_expiration_time = 31919000 + websockets_enabled = false + errorpage_whitelabel = true origin { - type = "OriginUrl" - url = "https://devxy-r-package-binaries-docs.fsn1.your-objectstorage.com" + type = "OriginUrl" + url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + middleware_script = bunnynet_compute_script.rpkgs_router.id } routing { - tier = "Standard" + filters = [ + "scripting", + ] } s3_auth_enabled = true - s3_auth_key = var.HETZNER_S3_ACCESS_KEY_K3S - s3_auth_secret = var.HETZNER_S3_SECRET_KEY_K3S - s3_auth_region = "fsn1" + s3_auth_key = var.B2_S3_ACCESS_KEY + s3_auth_secret = var.B2_S3_SECRET_KEY + s3_auth_region = "eu-central-003" - cache_enabled = true - cache_errors = true - # 3 months (override origin cache policy which is 30d) - cache_expiration_time = 7776000 - # not so great in case data from users is being sent (at some point) - request_coalescing_enabled = false + cache_enabled = true + request_coalescing_enabled = true block_post_requests = true + cache_vary_headers = ["User-Agent"] - originshield_enabled = true - originshield_concurrency_limit = true - originshield_zone = "FR" + limit_requests = 5000 + limit_connections = 1000 - safehop_enabled = true + safehop_enabled = true + add_canonical_header = true + cache_stale = ["offline", "updating"] + block_ips = var.cdn_block_ips - # otherwise this will expose the S3 origin URL as a secondary canonical URL - add_canonical_header = false + # 50 TB + limit_bandwidth = 50000000000000 - block_root_path = false + block_root_path = true } -resource "bunnynet_pullzone_hostname" "r-package-binaries-docs" { - - pullzone = bunnynet_pullzone.r-package-binaries-docs.id - name = "docs.r-package-binaries.devxy.io" +resource "bunnynet_pullzone_hostname" "cran_allianceswisspass" { + pullzone = bunnynet_pullzone.cran_allianceswisspass.id + name = "cran.allianceswisspass.devxy.io" force_ssl = true tls_enabled = true } + +# resource "bunnynet_storage_zone" "devxy-r-binaries" { +# name = "devxy-r-binaries-storage" +# region = "DE" +# zone_tier = "Standard" +# # Los Angeles and Singapore +# replication_regions = ["LA", "SG"] +# } diff --git a/docker/Containerfile-shiny-app b/docker/Containerfile-shiny-app index 02432c6..4b0bd6c 100644 --- a/docker/Containerfile-shiny-app +++ b/docker/Containerfile-shiny-app @@ -1,7 +1,15 @@ FROM devxygmbh/r-alpine:4.4-3.20 AS build # ARG GITHUB_PAT -RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +RUN apk add --no-cache curl ca-certificates \ + && curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh \ + | UVR_INSTALL_DIR=/usr/local/bin sh + +# One uvr project drives both dependency installs below. UVR_LIBRARY points the +# syncs at the image's R library instead of the project-local .uvr/library/, so +# `Rscript app.R` finds the packages without a uvr-aware .Rprofile. +ENV UVR_LIBRARY=/usr/lib/R/library +RUN mkdir -p /uvr && cd /uvr && uvr init --here COPY --link ./DESCRIPTION . COPY --link ./R ./R @@ -11,13 +19,19 @@ COPY --link ./shiny/app.R /app/app.R RUN ls -la -# install R package deps -RUN R -q -e 'pak::pak()' +# install R package deps. uvr has no equivalent of bare `pak::pak()`, which +# reads the DESCRIPTION in the working directory, so extract the dependency +# names and hand them to `uvr add`. +RUN R -q --no-echo -e "d <- read.dcf('DESCRIPTION'); f <- intersect(colnames(d), c('Depends', 'Imports', 'LinkingTo')); p <- trimws(sub('[(].*', '', unlist(strsplit(paste(d[, f], collapse = ','), ',')))); writeLines(setdiff(p[nzchar(p)], c('R', rownames(installed.packages()))), '/tmp/deps.txt')" \ + && cd /uvr && xargs -r uvr add --no-install < /tmp/deps.txt \ + && uvr sync --install-system-deps RUN R CMD INSTALL --no-docs --without-keep.source . # install shiny app deps -RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraftR'); pak::pak(pkgs)" +RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); writeLines(pkgs, '/tmp/app-deps.txt')" \ + && cd /uvr && xargs -r uvr add --no-install < /tmp/app-deps.txt \ + && uvr sync --install-system-deps ENV PGPASS="" @@ -26,4 +40,4 @@ RUN echo "options(shiny.port=3838, shiny.host='0.0.0.0')" >> /root/.Rprofile WORKDIR /app EXPOSE 3838 -CMD Rscript app.R \ No newline at end of file +CMD Rscript app.R diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile new file mode 100644 index 0000000..8e61ebd --- /dev/null +++ b/docker/build-one.Dockerfile @@ -0,0 +1,76 @@ +# syntax=docker/dockerfile:1 +# Targeted (re)build of specific package versions, executed on a remote buildx +# builder. The build's effect is the S3 upload performed by build-one.R; no +# image is kept (the justfile uses --output type=cacheonly). CACHEBUST forces +# the RUN to re-execute on every invocation. +# +# Build context is `local/` (see the `rebuild` recipe in the justfile). +ARG OS +ARG OS_VERSION +FROM reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION} + +ARG R_VERSION=4.5.3 +ARG PACKAGE +ARG VERSIONS +ARG CACHEBUST + +WORKDIR /work +COPY build-one.R /work/build-one.R +# Resolve and install the latest bincraft release dynamically (no hardcoded pin). +# uvr-install.sh lands under /work/local/ because install-bincraft.R looks for it +# there when the working directory is not a repo checkout. +COPY install-bincraft.R /work/install-bincraft.R +COPY uvr-install.sh /work/local/uvr-install.sh +# Ship the patch registry so build-one.R's `patches = "local/patches"` resolves +# (build context is `local/`, CWD is /work). +COPY patches /work/local/patches + +RUN --mount=type=secret,id=b2_access,required=true \ + --mount=type=secret,id=b2_secret,required=true \ + --mount=type=secret,id=pgpass,required=true \ + --mount=type=secret,id=github_pat,required=false \ + export B2_S3_ACCESS_KEY="$(cat /run/secrets/b2_access)" && \ + export B2_S3_SECRET_KEY="$(cat /run/secrets/b2_secret)" && \ + export PGPASS="$(cat /run/secrets/pgpass)" && \ + export GITHUB_PAT="$(cat /run/secrets/github_pat 2>/dev/null || true)" && \ + export GIT_TERMINAL_PROMPT=0 && \ + export OTEL_SDK_DISABLED=true && \ + export OTEL_R_TRACES_EXPORTER=none && \ + export OTEL_R_LOGS_EXPORTER=none && \ + export OTEL_R_METRICS_EXPORTER=none && \ + XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run || true); \ + XVFB_ARGS=""; \ + if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi; \ + USE_XVFB=0; \ + if [ -n "$XVFB" ] && $XVFB -a $XVFB_ARGS -- true >/dev/null 2>&1; then \ + USE_XVFB=1; echo "Using virtual display via $XVFB"; \ + else \ + echo "No working virtual display; building without xvfb" >&2; \ + fi; \ + run_build() { if [ "$USE_XVFB" = 1 ]; then $XVFB -a $XVFB_ARGS -- "$@"; else "$@"; fi; }; \ + ensure_bincraft() { "$1" -q -e 'source("/work/install-bincraft.R")'; }; \ + PRIMARY_MINOR=$(echo "$R_VERSION" | cut -d. -f1-2); \ + seen=" $PRIMARY_MINOR "; \ + prc=0; failed=""; \ + rm -f .r_minor_sensitive; \ + echo "=== primary pass under R $R_VERSION ==="; \ + ensure_bincraft /opt/R/${R_VERSION}/bin/R; \ + run_build /opt/R/${R_VERSION}/bin/Rscript /work/build-one.R "${PACKAGE}" ${VERSIONS} || prc=$?; \ + if [ -f .r_minor_sensitive ]; then \ + for RBIN in /opt/R/*/bin/Rscript; do \ + [ -x "$RBIN" ] || continue; \ + RV=$(basename "$(dirname "$(dirname "$RBIN")")"); \ + case "$RV" in [0-9]*) ;; *) continue ;; esac; \ + RMINOR=$(echo "$RV" | cut -d. -f1-2); \ + case "$seen" in *" $RMINOR "*) continue ;; esac; \ + seen="$seen$RMINOR "; \ + echo "=== sensitive-only pass under R $RV ==="; \ + ensure_bincraft "$(dirname "$RBIN")/R"; \ + run_build "$RBIN" /work/build-one.R --sensitive-only "${PACKAGE}" ${VERSIONS} || failed="$failed $RV"; \ + done; \ + else \ + echo "Package not r-minor-sensitive; skipping per-minor passes."; \ + fi; \ + if [ -n "$failed" ]; then echo "WARNING: extra-minor build(s) failed (non-fatal; e.g. a version too old to compile on a newer R):$failed" >&2; fi; \ + if [ "$prc" != 0 ]; then echo "primary pass under R $R_VERSION failed (exit $prc)" >&2; fi; \ + exit "$prc" diff --git a/docker/buildkitd.toml b/docker/buildkitd.toml new file mode 100644 index 0000000..ebbeb38 --- /dev/null +++ b/docker/buildkitd.toml @@ -0,0 +1,41 @@ +# BuildKit GC config for the remote buildx builders (artemis/amd64, gaia/arm64). +# +# Apply when creating the docker-container builders: +# docker buildx create --name artemis --driver docker-container \ +# --config docker/buildkitd.toml ssh:// +# docker buildx create --name gaia --driver docker-container \ +# --config docker/buildkitd.toml ssh:// +# +# Why: BuildKit's default GC caps the ephemeral cache tier — RUN +# --mount=type=cache mounts, local build context, git checkouts — at a +# hardcoded 512 MB (shown as "488.3 MiB" in `buildx inspect`). Across our +# 7-distro build matrix that fills instantly and forces re-downloads of +# system + R packages every rebuild. The first rule below raises that tier. +# +# Limits are absolute (not %) on purpose: artemis and gaia have very +# different free space (Hetzner ~42 GiB free vs Mac mini ~279 GiB), so a +# percentage would mean wildly different real budgets. minFreeSpace = 20 GB +# keeps the tight Hetzner host safe while staying modest on the Mac mini. + +[worker.oci] + gc = true + + # Tier 1 — ephemeral caches (cache mounts, local context, git checkouts). + # Raised from the 512 MB default to 8 GB, retained for 7 days so weekly + # rebuilds reuse downloaded packages instead of re-fetching them. + [[worker.oci.gcpolicy]] + filters = [ + "type==source.local", + "type==exec.cachemount", + "type==source.git.checkout", + ] + keepDuration = "168h" + maxUsedSpace = "8GB" + + # Tier 2 — everything else (image layers, RUN exec results). Bounds the + # whole buildkit cache and always leaves 20 GB free on the host disk. + [[worker.oci.gcpolicy]] + all = true + reservedSpace = "2GB" + maxUsedSpace = "40GB" + minFreeSpace = "20GB" diff --git a/docker/reprex/alpine.sh b/docker/reprex/alpine.sh index 94806e5..851a9cc 100644 --- a/docker/reprex/alpine.sh +++ b/docker/reprex/alpine.sh @@ -1,10 +1,14 @@ docker run --rm -it --platform linux/arm64 alpine sh -apk add --no-cache R R-dev g++ +apk add --no-cache R R-dev g++ curl ca-certificates -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | UVR_INSTALL_DIR=/usr/local/bin sh -R -q -e 'pak::pak(c("gert", "purrr"))' +# `uvr add` always writes to .uvr/library/; only `uvr sync` honours UVR_LIBRARY, +# so add without installing and let the sync place the packages. +export UVR_LIBRARY=/usr/lib/R/library +mkdir -p /uvr && cd /uvr && uvr init --here +uvr add --no-install gert purrr && uvr sync --install-system-deps R @@ -22,14 +26,20 @@ unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE) tag <- all_tags$name package_name <- rep(package_name, length(tag)) +# uvr has no `pak::local_install_deps()`; read the DESCRIPTION of the checkout +# and `uvr add` the dependency names instead. purrr::walk2(package_name[1], tag, \(x, y) { print(y) system("git config --global advice.detachedHead false") + src <- sprintf("/tmp/%s_%s", x[1], y) system2("git", args = c( "clone", "-q", sprintf("--branch=%s", tail(y, 1)), - sprintf("https://github.com/cran/%s", x[1]), sprintf("/tmp/%s_%s", x[1], y) + sprintf("https://github.com/cran/%s", x[1]), src )) - pak::local_install_deps(sprintf("/tmp/%s_%s", x[1], y)) + d <- read.dcf(file.path(src, "DESCRIPTION")) + f <- intersect(colnames(d), c("Depends", "Imports", "LinkingTo")) + deps <- trimws(sub("[(].*", "", unlist(strsplit(paste(d[, f], collapse = ","), ",")))) + deps <- setdiff(deps[nzchar(deps)], "R") + system2("uvr", c("add", "--no-install", deps)) + system2("uvr", c("sync", "--install-system-deps")) }) - - diff --git a/edge/rpkgs-router.test.ts b/edge/rpkgs-router.test.ts new file mode 100644 index 0000000..9493f8a --- /dev/null +++ b/edge/rpkgs-router.test.ts @@ -0,0 +1,167 @@ +/** + * Routing matrix for `edge/rpkgs-router.ts`. + * + * The script is exercised through the SDK's local server rather than by + * importing its internals, so what is tested is the artifact that gets + * deployed. Requests that the script passes through are proxied to the real + * origin, which keeps the "no redirect" cases honest: they assert that the + * client reached the flat slot, not merely that no `Location` was set. + * + * Run with `just edge-test`. + */ +import { assertEquals } from 'jsr:@std/assert@1'; + +const SCRIPT = new URL('./rpkgs-router.ts', import.meta.url).pathname; +const BASE = 'http://127.0.0.1:8080'; +const UNION_SLOTS = 'amd64/alpine324'; + +const UA_R45_MUSL = 'R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R46_MUSL = 'R (4.6.0 x86_64-pc-linux-musl x86_64 linux-musl)'; +const UA_R45_ALPINE = 'R/4.5.3 R (4.5.3 x86_64-pc-linux-musl x86_64 linux-musl) Alpine Linux 3.24'; +const UA_R45_DARWIN = 'R (4.5.1 aarch64-apple-darwin20 aarch64 darwin20)'; +const UA_CURL = 'curl/8.0.1'; + +const SLOT = '/amd64/alpine324/latest/src/contrib'; +const OTHER_SLOT = '/amd64/noble/latest/src/contrib'; + +interface Probe { + status: number; + location: string | null; + cacheControl: string | null; +} + +async function probe(path: string, userAgent: string): Promise { + const res = await fetch(BASE + path, { + headers: { 'User-Agent': userAgent }, + redirect: 'manual', + }); + await res.body?.cancel(); + return { + status: res.status, + location: res.headers.get('location'), + cacheControl: res.headers.get('cache-control'), + }; +} + +/** Kill tolerantly: the child has already exited if the script failed to load. */ +async function stopServer(child: Deno.ChildProcess): Promise { + try { + child.kill(); + } catch { + // already gone + } + await child.status; +} + +async function startServer(): Promise { + const child = new Deno.Command(Deno.execPath(), { + args: ['run', '-A', SCRIPT], + env: { UNION_SLOTS }, + stdout: 'null', + stderr: 'inherit', + }).spawn(); + + for (let attempt = 0; attempt < 150; attempt++) { + try { + const res = await fetch(`${BASE}/`, { + headers: { 'User-Agent': UA_CURL }, + redirect: 'manual', + }); + await res.body?.cancel(); + return child; + } catch { + await new Promise((resolve) => setTimeout(resolve, 200)); + } + } + + await stopServer(child); + throw new Error('edge script did not start listening on ' + BASE); +} + +Deno.test('rpkgs-router', async (t) => { + const server = await startServer(); + + try { + await t.step("routes an index request to the client's R minor", async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.status, 302); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + + await t.step('routes R 4.6 to its own slot', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R46_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.6/PACKAGES.gz`); + }); + + await t.step('routes PACKAGES and PACKAGES.rds too', async () => { + for (const file of ['PACKAGES', 'PACKAGES.rds']) { + const res = await probe(`${SLOT}/${file}`, UA_R45_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/${file}`, `expected ${file} to be routed`); + } + }); + + await t.step('marks the redirect uncacheable', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.cacheControl, 'no-store'); + }); + + await t.step('leaves a slot outside UNION_SLOTS alone', async () => { + const res = await probe(`${OTHER_SLOT}/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('never routes a tarball', async () => { + const res = await probe(`${SLOT}/jsonlite_2.0.0.tar.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('serves an archived binary when it exists', async () => { + const path = `${SLOT}/Archive/xml2/xml2_1.5.2.tar.gz`; + const res = await probe(path, UA_R45_MUSL); + assertEquals(res.status, 200); + assertEquals(res.location, null); + }); + + await t.step('does not redirect a path already under a minor', async () => { + const res = await probe(`${SLOT}/4.5/PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('leaves a client without an R version alone', async () => { + const res = await probe(`${SLOT}/PACKAGES.gz`, UA_CURL); + assertEquals(res.location, null); + assertEquals(res.status, 200); + }); + + await t.step('resolves the bare root to slot and minor', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_ALPINE); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + + await t.step('sends an unidentifiable distro to CRAN', async () => { + const res = await probe('/src/contrib/PACKAGES.gz', UA_R45_MUSL); + assertEquals(res.location, 'https://cran.r-project.org/src/contrib/PACKAGES.gz'); + }); + + await t.step('keeps the macOS rewrite', async () => { + const res = await probe('/src/contrib/foo_1.0.tar.gz', UA_R45_DARWIN); + assertEquals(res.location, 'https://cran.rpkgs.com/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz'); + }); + + await t.step('keeps the macOS binary passthrough to CRAN', async () => { + const path = '/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz'; + const res = await probe(path, UA_R45_DARWIN); + assertEquals(res.location, `https://cran.r-project.org${path}`); + }); + + await t.step('collapses duplicate slashes before matching', async () => { + const res = await probe(`/amd64/alpine324//latest/src/contrib//PACKAGES.gz`, UA_R45_MUSL); + assertEquals(res.location, `https://cran.rpkgs.com${SLOT}/4.5/PACKAGES.gz`); + }); + } finally { + await stopServer(server); + } +}); diff --git a/edge/rpkgs-router.ts b/edge/rpkgs-router.ts new file mode 100644 index 0000000..9278d4a --- /dev/null +++ b/edge/rpkgs-router.ts @@ -0,0 +1,258 @@ +/** + * Edge middleware for cran.rpkgs.com. + * + * Two jobs: + * + * 1. Resolve the bare `https://cran.rpkgs.com` form to a concrete + * `/` slot from the User-Agent, or send the client to CRAN when + * the distro cannot be identified. + * 2. Route `PACKAGES*` requests to the per-R-minor slot + * (`…/latest/src/contrib//`), so a stock `install.packages()` sees the + * packages that only exist there. + * + * Only index files are routed. Tarballs are deliberately left alone: R keeps + * the contrib URL it asked for, not the one it was redirected to, so every + * tarball URL is resolved against the flat directory and the union index steers + * the per-minor ones with a `Path: ` field. Rewriting a tarball request + * here would send flat-slot packages into a directory that does not hold them. + * + * Routing is gated on UNION_SLOTS. The raw per-minor index holds only the + * ABI-sensitive subset of a slot; it is safe to route to it only once bincraft + * has republished it as a union of the per-minor and flat slots. + * + * Deployed by OpenTofu from this file (`bunnynet_compute_script.rpkgs_router`). + * Test with `just edge-test`. + */ +import * as BunnySDK from 'https://esm.sh/@bunny.net/edgescript-sdk@0.12'; + +const PUBLIC_CDN_ORIGIN = 'https://cran.rpkgs.com'; +const CRAN_ORIGIN = 'https://cran.r-project.org'; +const PUBLIC_CDN_HOSTS = new Set(['cran.rpkgs.com', 'cran.allianceswisspass.devxy.io']); + +/** Slots ("/", comma separated) whose per-minor index is a union. */ +const UNION_SLOTS = new Set( + (Deno.env.get('UNION_SLOTS') ?? '') + .split(',') + .map((slot) => slot.trim()) + .filter((slot) => slot.length > 0), +); + +/** `///latest/src/contrib[/]` */ +const SLOT_PATH_REGEX = /^\/(amd64|arm64)\/([a-z0-9._-]+)\/latest\/src\/contrib\/?(.*)$/; + +/** A path that already sits in a per-minor slot, e.g. `4.5/PACKAGES.gz`. */ +const MINOR_DIR_REGEX = /^\d+\.\d+\//; + +/** The only files this script routes. */ +const INDEX_FILE_REGEX = /^PACKAGES(\.gz|\.rds)?$/; + +const SRC_CONTRIB_REGEX = /^\/src\/contrib\/(.+)$/; + +/** A binary archive URL whose upstream source counterpart CRAN can serve. */ +const ARCHIVE_TARBALL_REGEX = + /^\/(?:amd64|arm64)\/[a-z0-9._-]+\/latest\/src\/contrib\/Archive\/([^/]+)\/([^/]+\.tar\.gz)$/; + +const MACOS_BIN_REGEX = + /^\/bin\/macosx\/(big-sur-arm64|big-sur-x86_64|monterey-arm64|monterey-x86_64)\/contrib\/([0-9.]+)\/(.+)$/; + +const RHEL_REGEX = /(almalinux|rocky)[^\d]*(\d+)/i; + +const UBUNTU_REGEX = /Ubuntu ([\d.]+)/i; +const UBUNTU_CODENAMES: Record = { + '24.04': 'noble', + '22.04': 'jammy', +}; + +const ALPINE_REGEX = /(?:Alpine Linux(?:\s+VERSION_ID=)?|alpine-)\s*(\d+)\.(\d+)/i; + +/** + * R's own User-Agent is `R (4.5.3 x86_64-pc-linux-musl …)`; the Posit-style one + * some sites configure is `R/4.5.3 R (…)`. Both carry the minor, which is why + * per-minor routing works without the distro being identifiable. + */ +const R_MINOR_REGEXES = [/\bR\/(\d+)\.(\d+)/, /\bR \((\d+)\.(\d+)/]; + +function normalizePathname(pathname: string): string { + return pathname.replace(/\/{2,}/g, '/'); +} + +function redirectTo(location: string, status = 302): Response { + return new Response(null, { + status, + headers: { + Location: location, + // The target depends on the User-Agent, so the redirect itself must + // never be cached; only its target is a cacheable, UA-independent URL. + 'Cache-Control': 'no-store', + 'X-Via': 'MyMiddleware', + 'X-Rewritten-By': 'rpkgs-edge-middleware', + }, + }); +} + +function publicCdnOrigin(url: URL): string { + return PUBLIC_CDN_HOSTS.has(url.hostname) ? url.origin : PUBLIC_CDN_ORIGIN; +} + +function extractRMinor(userAgent: string): string | null { + for (const regex of R_MINOR_REGEXES) { + const match = userAgent.match(regex); + if (match) { + return `${match[1]}.${match[2]}`; + } + } + return null; +} + +function mapArch(arch: string): string { + if (arch === 'aarch64') return 'arm64'; + if (arch === 'x86_64') return 'amd64'; + return arch; +} + +function extractArch(userAgent: string): string { + const match = userAgent.match(/(x86_64|aarch64|arm64|i386|i686)/); + return match ? mapArch(match[1]) : ''; +} + +/** + * Identify the `/` slot from the User-Agent, or null. + * + * A stock R User-Agent carries only `linux-gnu` / `linux-musl`, which are not + * slot names: returning them produced redirects into slots that do not exist + * (`/amd64/linux-musl/latest/…`, a guaranteed 404). An unidentifiable distro + * is reported as such so the caller can fall back to CRAN. + */ +function parseSlot(userAgent: string): string | null { + const arch = extractArch(userAgent); + if (!arch) { + return null; + } + + const rhel = userAgent.match(RHEL_REGEX); + if (rhel) { + return `${arch}/rhel${rhel[2]}`; + } + + const ubuntu = userAgent.match(UBUNTU_REGEX); + if (ubuntu) { + const codename = UBUNTU_CODENAMES[ubuntu[1]]; + if (codename) { + return `${arch}/${codename}`; + } + } + + const alpine = userAgent.match(ALPINE_REGEX); + if (alpine) { + return `${arch}/alpine${alpine[1]}${alpine[2]}`; + } + + return null; +} + +function parseMacUserAgent(userAgent: string): { os: string; arch: string; rver: string } | null { + const rverMatch = userAgent.match(/R \((\d+)\.(\d+)/); + const archMatch = userAgent.match(/(aarch64|arm64|x86_64)/); + const osMatch = userAgent.match(/darwin(\d+)/); + + if (!rverMatch || !archMatch || !osMatch) { + return null; + } + + const arch = archMatch[1] === 'aarch64' ? 'arm64' : archMatch[1]; + const darwinVer = parseInt(osMatch[1], 10); + const os = darwinVer >= 21 && darwinVer < 22 ? `monterey-${arch}` : `big-sur-${arch}`; + + return { os, arch, rver: `${rverMatch[1]}.${rverMatch[2]}` }; +} + +/** + * The contrib path a request should be served from, relative to the slot. + * + * Returns the per-minor path for an index file when the slot is known to carry + * a union index and the client's R minor is known; otherwise the flat path, + * which is what every client sees today. + */ +function contribPath(slot: string, rest: string, userAgent: string): string { + const flat = rest ? `/${slot}/latest/src/contrib/${rest}` : `/${slot}/latest/src/contrib`; + + if (!INDEX_FILE_REGEX.test(rest) || !UNION_SLOTS.has(slot)) { + return flat; + } + + const rMinor = extractRMinor(userAgent); + return rMinor ? `/${slot}/latest/src/contrib/${rMinor}/${rest}` : flat; +} + +BunnySDK.net.http + .servePullZone({ url: 'https://cran.rpkgs.com/' }) + .onOriginRequest((ctx) => { + const url = new URL(ctx.request.url); + const path = normalizePathname(url.pathname); + const userAgent = ctx.request.headers.get('User-Agent') || ''; + const publicOrigin = publicCdnOrigin(url); + + // macOS clients are served from CRAN's own binary tree. + const srcContrib = path.match(SRC_CONTRIB_REGEX); + if (srcContrib && /darwin/.test(userAgent)) { + const mac = parseMacUserAgent(userAgent); + if (mac) { + return Promise.resolve(redirectTo(`${publicOrigin}/bin/macosx/${mac.os}/contrib/${mac.rver}/${srcContrib[1]}`)); + } + } + + if (MACOS_BIN_REGEX.test(path)) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + + // Already-qualified slot URLs: what the runtime images have baked in. + const slotPath = path.match(SLOT_PATH_REGEX); + if (slotPath) { + const slot = `${slotPath[1]}/${slotPath[2]}`; + const rest = slotPath[3]; + + // Never rewrite a request that is already in a per-minor slot, or the + // redirect would chase its own tail. + if (MINOR_DIR_REGEX.test(rest)) { + return Promise.resolve(ctx.request); + } + + const target = contribPath(slot, rest, userAgent); + if (target === path) { + return Promise.resolve(ctx.request); + } + return Promise.resolve(redirectTo(`${publicOrigin}${target}`)); + } + + // The bare `https://cran.rpkgs.com` form, resolved from the User-Agent. + if (path === '/' || path === '/src/contrib' || path.startsWith('/src/contrib/')) { + const slot = parseSlot(userAgent); + if (!slot) { + return Promise.resolve(redirectTo(`${CRAN_ORIGIN}${path}`)); + } + + const rest = srcContrib ? srcContrib[1] : ''; + return Promise.resolve(redirectTo(`${publicOrigin}${contribPath(slot, rest, userAgent)}`)); + } + + return Promise.resolve(ctx.request); + }) + .onOriginResponse(async (ctx) => { + const path = normalizePathname(new URL(ctx.request.url).pathname); + const archive = path.match(ARCHIVE_TARBALL_REGEX); + + // Binary archives can be incomplete when an older build never succeeded. + // Preserve renv/remotes version restores by falling back to CRAN's source + // package only for an absent archived tarball. A requested version can be + // either archived upstream or still current, so probe the archive first. + // Other 404s remain visible. + if (ctx.response.status === 404 && archive) { + const archiveUrl = `${CRAN_ORIGIN}/src/contrib/Archive/${archive[1]}/${archive[2]}`; + const archiveResponse = await fetch(archiveUrl, { method: 'HEAD' }); + const sourceUrl = archiveResponse.ok ? archiveUrl : `${CRAN_ORIGIN}/src/contrib/${archive[2]}`; + return redirectTo(sourceUrl); + } + + ctx.response.headers.append('X-Via', 'MyMiddleware'); + return Promise.resolve(ctx.response); + }); diff --git a/justfile b/justfile new file mode 100644 index 0000000..6c75b13 --- /dev/null +++ b/justfile @@ -0,0 +1,89 @@ +# Local helpers for build-cran-binaries. +# +# `rebuild` (re)builds specific versions of a single package on a given OS/arch +# by dispatching to a remote buildx builder (the build runs there, not locally). +# Sensitivity is auto-detected: risky packages land in the per-minor slot +# (contrib//), everything else in the generic slot; the touched index is +# refreshed so the result is immediately servable. +# +# Prerequisites: +# - buildx builders named `artemis` (amd64) and `gaia` (arm64), created with the +# docker-container driver (runs BuildKit on the remote host's docker daemon over +# SSH). The default `remote` driver does NOT work with an ssh:// docker host. +# Pass --config docker/buildkitd.toml so BuildKit's GC keeps a usable cache +# (the default caps the cache-mount tier at 512 MB, forcing re-downloads). +# docker buildx create --name artemis --driver docker-container --config docker/buildkitd.toml ssh:// +# docker buildx create --name gaia --driver docker-container --config docker/buildkitd.toml ssh:// +# - exported secrets: B2_S3_ACCESS_KEY, B2_S3_SECRET_KEY, PGPASS (GITHUB_PAT optional) +# +# Overridable (env or `just VAR=… rebuild …`): +# R_VERSION (default 4.5.3) — selects the R minor → the per-minor slot +# AMD64_BUILDER / ARM64_BUILDER — buildx builder names +# +# Examples: +# just rebuild alpine 3.23 amd64 rlang 1.1.4 1.1.3 +# just rebuild redhat 10 arm64 data.table 1.15.4 +# R_VERSION=4.4.3 just rebuild ubuntu noble amd64 Rcpp 1.0.12 + +r_version := env_var_or_default("R_VERSION", "4.5.3") +amd64_builder := env_var_or_default("AMD64_BUILDER", "artemis") +arm64_builder := env_var_or_default("ARM64_BUILDER", "gaia") + +# (re)build PACKAGE at one or more VERSIONS on OS/TAG/ARCH via a remote builder +rebuild os tag arch package *versions: + #!/usr/bin/env bash + set -euo pipefail + if [ -z "{{ versions }}" ]; then + echo "error: provide at least one version, e.g. just rebuild alpine 3.23 amd64 rlang 1.1.4" >&2 + exit 1 + fi + case "{{ arch }}" in + amd64) builder="{{ amd64_builder }}" ;; + arm64) builder="{{ arm64_builder }}" ;; + *) echo "error: arch must be 'amd64' or 'arm64'" >&2; exit 1 ;; + esac + # Accept TF_VAR_-prefixed names (direnv) or plain names. + export B2_S3_ACCESS_KEY="${B2_S3_ACCESS_KEY:-${TF_VAR_B2_S3_ACCESS_KEY:-}}" + export B2_S3_SECRET_KEY="${B2_S3_SECRET_KEY:-${TF_VAR_B2_S3_SECRET_KEY:-}}" + : "${B2_S3_ACCESS_KEY:?set B2_S3_ACCESS_KEY or TF_VAR_B2_S3_ACCESS_KEY}" + : "${B2_S3_SECRET_KEY:?set B2_S3_SECRET_KEY or TF_VAR_B2_S3_SECRET_KEY}" + : "${PGPASS:?set PGPASS}" + secret_args=( + --secret id=b2_access,env=B2_S3_ACCESS_KEY + --secret id=b2_secret,env=B2_S3_SECRET_KEY + --secret id=pgpass,env=PGPASS + ) + # GITHUB_PAT is optional (raises the GitHub API rate limit); pass only if set. + if [ -n "${GITHUB_PAT:-}" ]; then + secret_args+=(--secret id=github_pat,env=GITHUB_PAT) + fi + echo "Dispatching build of {{ package }} ({{ versions }}) on {{ os }}:{{ tag }}/{{ arch }} (R {{ r_version }}) to builder '$builder'" + docker buildx build \ + --builder "$builder" \ + --platform "linux/{{ arch }}" \ + --no-cache \ + --progress=plain \ + --output type=cacheonly \ + "${secret_args[@]}" \ + --build-arg OS="{{ os }}" \ + --build-arg OS_VERSION="{{ tag }}" \ + --build-arg R_VERSION="{{ r_version }}" \ + --build-arg PACKAGE="{{ package }}" \ + --build-arg VERSIONS="{{ versions }}" \ + --build-arg CACHEBUST="$(date +%s)" \ + -f docker/build-one.Dockerfile \ + local + +# run the edge middleware routing matrix (uses a local deno, else the deno image) +edge-test: + #!/usr/bin/env bash + set -euo pipefail + if command -v deno >/dev/null 2>&1; then + deno test -A edge/rpkgs-router.test.ts + else + docker run --rm \ + -v "$PWD:/w" -w /w \ + -v deno-cache:/deno-dir \ + denoland/deno:latest \ + deno test -A edge/rpkgs-router.test.ts + fi diff --git a/local/archive-missed-pkgs.R b/local/archive-missed-pkgs.R index 29c1af4..f817c3e 100644 --- a/local/archive-missed-pkgs.R +++ b/local/archive-missed-pkgs.R @@ -1,5 +1,5 @@ ### Scope: Archive packages that have more than one entry in repository root -library(bincraftR) +library(bincraft) s3fs::s3_file_system( aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"), @@ -9,18 +9,36 @@ s3fs::s3_file_system( ) arch = "arm64" -os = "alpine320" +os = "alpine321" -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", arch, os)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", + arch, + os +)) length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1]))) -non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) x[1])[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))]) +non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) { + x[1] +})[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))]) # RInno is not built because it only exists for Windows non_archived = setdiff(non_archived, "RInno") # future::plan("sequential", workers = 8) future::plan("sequential", workers = 1) # future.apply::future_lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch)) -lapply(non_archived, function(x) archive_package(x, codename = os, arch = arch)) +lapply(non_archived, function(x) { + archive_package( + x, + codename = os, + arch = arch, + s3_region = "hel1", + s3_endpoint = "https://hel1.your-objectstorage.com", + s3_bucket = "devxy-r-package-binaries-hel1", + s3_access_key_id = Sys.getenv( + "HETZNER_S3_ACCESS_KEY_K3S", + s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S") + ) + ) +}) # archive_package() - -grep("duckdb_", files, value = T) +# grep("duckdb_", files, value = T) diff --git a/local/build-all.R b/local/build-all.R new file mode 100644 index 0000000..37fc593 --- /dev/null +++ b/local/build-all.R @@ -0,0 +1,166 @@ +sink(stdout(), type = "message") +options(crayon.enabled = TRUE, future.globals.onReference = NULL) +source(file.path("local", "r-minor-helpers.R")) + +args <- commandArgs(trailingOnly = TRUE) +parsed <- parse_build_args(args) +split_into <- parsed$split_into +split_index <- parsed$split_index +ncpus <- parsed$ncpus +sensitive_only <- parsed$sensitive_only +options(Ncpus = ncpus) + +# Load bincraft eagerly to avoid lazy-load memory spike during first build call +library(bincraft, quietly = TRUE) +library(future) +plan("sequential") + +# The install-deps step precomputes the package snapshot into /mnt/cache, but +# that volume is per-agent: a job landing on a fresh agent (or racing +# install-deps) finds it empty. Recompute the snapshot here when any part is +# missing, so the first job on an agent repopulates the cache for the jobs that +# follow; concurrent jobs that also miss simply redo the work. Write via a +# temp file + atomic rename so a concurrent reader never sees a half-written rds. +package_cache_files <- c( + "/mnt/cache/packages/pkgs_to_build.rds", + "/mnt/cache/packages/r_minor_sensitive_pkgs.rds", + "/mnt/cache/packages/s3_cache.rds" +) +if (!all(file.exists(package_cache_files))) { + message( + "Package snapshot missing from cache; recomputing via packages-to-build.R" + ) + dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) + save_rds_atomic <- function(obj, path) { + tmp <- paste0(path, ".tmp.", Sys.getpid()) + saveRDS(obj, tmp) + file.rename(tmp, path) + } + source(file.path("local", "packages-to-build.R")) + save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds") + save_rds_atomic( + pkgs[r_minor_sensitive == TRUE], + "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" + ) + message("Package snapshot recomputed.") +} + +pkgs <- if (sensitive_only) { + readRDS("/mnt/cache/packages/r_minor_sensitive_pkgs.rds") +} else { + readRDS("/mnt/cache/packages/pkgs_to_build.rds") +} +# Back-compat: tolerate an older RDS without the column (treat all as non-sensitive) +if (is.null(pkgs$r_minor_sensitive)) { + pkgs$r_minor_sensitive <- FALSE +} +sprintf( + "Total# of remaining package versions: %s (sensitive_only=%s)", + nrow(pkgs), + sensitive_only +) + +# Split into chunks for this worker +chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE)) +chunk <- chunks[[split_index]] +sprintf("# of package versions for this job: %s", nrow(chunk)) + +# Exclude known problematic packages (single source of truth) +exclude <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] +chunk <- chunk[!chunk$Package %in% exclude, ] + +# Skip package versions already attempted in a previous run (built or errored). +# pkgs_to_build.rds is a static snapshot from the install-deps step, so on a +# restart it still lists everything an interrupted run already produced. The +# metadata DB reflects that progress, so we re-derive the remaining set here. +# We exclude *all* attempted versions, not just successful ones: a previously +# errored version is skipped by build_binary_package() anyway, so leaving it in +# the chunk only makes the job cycle through it one-by-one for no benefit. +# Derive platform + arch from the running container, mirroring the codename -> +# platform mapping bincraft uses internally. The OS/OS_VERSION selectors are +# workflow-level CI variables that are not injected into the container +# environment, so Sys.getenv() would return "" and this pre-filter would query +# platform "-" and skip nothing. +codename <- bincraft::set_codename(NULL) +platform <- switch( + codename, + jammy = "ubuntu-2204", + noble = "ubuntu-2404", + resolute = "ubuntu-2604", + rhel10 = "redhat-10", + rhel9 = "redhat-9", + rhel8 = "redhat-8", + alpine320 = "alpine-320", + alpine321 = "alpine-321", + alpine322 = "alpine-322", + alpine323 = "alpine-323", + alpine324 = "alpine-324", + alpine325 = "alpine-325", + alpine326 = "alpine-326", + NA_character_ +) +local_machine <- Sys.info()[["machine"]] +arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +built <- DBI::dbGetQuery( + con, + "SELECT name, tag FROM single_builds WHERE platform = $1 AND arch = $2", + params = list(platform, arch) +) +DBI::dbDisconnect(con) +before <- nrow(chunk) +chunk <- chunk[ + !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), +] +sprintf( + "Skipped %d already-attempted package versions; %d remaining for this job", + before - nrow(chunk), + nrow(chunk) +) + +# Read pre-computed S3 listing from install-deps step +# This avoids loading s3fs/reticulate/Python in the build container, +# saving significant memory for the dependency-installer subprocesses +s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") +sprintf("S3 cache: %s files", length(s3_cache)) + +n <- nrow(chunk) +mapply( + function(pkg, ver, sens, i) { + cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) + bincraft::build_binary_package( + pkg, + tag = ver, + is_r_minor_sensitive = isTRUE(sens), + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + s3_package_cache = s3_cache, + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + patches = "local/patches", + upload = TRUE, + store_build_metadata = TRUE + ) + }, + chunk$Package, + chunk$Version, + chunk$r_minor_sensitive, + seq_len(n) +) diff --git a/local/build-one.R b/local/build-one.R new file mode 100644 index 0000000..07c0b6a --- /dev/null +++ b/local/build-one.R @@ -0,0 +1,164 @@ +# Targeted (re)build of specific versions of a single package. +# Invoked inside a build-env container (see docker/build-one.Dockerfile). +# Usage: build-one.R [--sensitive-only] [ ...] +# Sensitivity is auto-detected per version via bincraft's ABI classifier: +# risky packages go to the per-minor slot, everything else to the generic slot. +# With --sensitive-only, non-risky versions are skipped (used for the extra +# per-minor passes under non-primary R versions). + +options(crayon.enabled = TRUE, future.globals.onReference = NULL) + +args <- commandArgs(trailingOnly = TRUE) +sensitive_only <- "--sensitive-only" %in% args +args <- args[args != "--sensitive-only"] +if (length(args) < 2L) { + stop( + "usage: build-one.R [--sensitive-only] [ ...]", + call. = FALSE + ) +} +package <- args[1L] +versions <- args[-1L] + +library(bincraft, quietly = TRUE) + +s3 <- list( + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") +) + +# Clone the CRAN source for a version and ask the ABI classifier whether it +# must be rebuilt per R minor. Fails safe to TRUE so a possibly-fragile binary +# is never served from the cross-minor generic slot by mistake. +classify <- function(pkg, ver) { + dest <- file.path(tempdir(), sprintf("classify_%s_%s", pkg, ver)) + on.exit(unlink(dest, recursive = TRUE, force = TRUE), add = TRUE) + tryCatch( + { + message(sprintf( + "[classify] cloning %s@%s from github.com/cran ...", + pkg, + ver + )) + system2( + "git", + c( + "clone", + "--depth", + "1", + "--branch", + ver, + sprintf("https://github.com/cran/%s", pkg), + dest + ) + ) + message(sprintf("[classify] running ABI classifier on %s ...", pkg)) + isTRUE(as.logical(bincraft::needs_per_minor_recompile(dest))) + }, + error = function(e) { + message(sprintf( + "classify failed for %s %s: %s; treating as r-minor-sensitive", + pkg, + ver, + conditionMessage(e) + )) + TRUE + } + ) +} + +minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." +) +touched_generic <- FALSE +touched_minor <- FALSE + +for (ver in versions) { + sensitive <- classify(package, ver) + if (sensitive_only && !sensitive) { + message(sprintf( + "Skipping %s %s under R %s (not r-minor-sensitive)", + package, + ver, + minor + )) + next + } + cat(sprintf( + "Building %s %s (r_minor_sensitive=%s, R %s)\n", + package, + ver, + sensitive, + minor + )) + bincraft::build_binary_package( + package, + tag = ver, + is_r_minor_sensitive = sensitive, + force = TRUE, + upload = TRUE, + archive = TRUE, + patches = "local/patches", + store_build_metadata = TRUE, + s3_endpoint = s3$s3_endpoint, + s3_region = s3$s3_region, + s3_bucket = s3$s3_bucket, + s3_access_key_id = s3$s3_access_key_id, + s3_secret_access_key = s3$s3_secret_access_key, + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432 + ) + if (sensitive) touched_minor <- TRUE else touched_generic <- TRUE +} + +# Signal the container wrapper whether per-minor passes are warranted (a single +# package's sensitivity is the same across R minors, so a non-sensitive package +# need not touch any other minor's library). +if (touched_minor) { + file.create(".r_minor_sensitive") +} + +# Refresh the PACKAGES index for each slot we wrote to, so the (re)built binary +# is immediately resolvable by clients. +codename <- bincraft::set_codename(NULL) +# cranlike keeps a working PACKAGES.db in the CWD; clear any copy left by a +# previous pass so each per-slot index is built fresh. Otherwise the 2nd index +# update in the same container fails with "table packages already exists". +clean_index_workdir <- function() { + unlink(c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db")) +} +if (touched_generic) { + cat("Refreshing generic index\n") + clean_index_workdir() + bincraft::upload_package_index( + codename = codename, + s3_endpoint = s3$s3_endpoint, + s3_region = s3$s3_region, + s3_bucket = s3$s3_bucket, + s3_access_key_id = s3$s3_access_key_id, + s3_secret_access_key = s3$s3_secret_access_key + ) +} +if (touched_minor) { + cat(sprintf("Refreshing per-minor index %s\n", minor)) + clean_index_workdir() + bincraft::upload_package_index( + codename = codename, + r_minor = minor, + s3_endpoint = s3$s3_endpoint, + s3_region = s3$s3_region, + s3_bucket = s3$s3_bucket, + s3_access_key_id = s3$s3_access_key_id, + s3_secret_access_key = s3$s3_secret_access_key + ) +} diff --git a/local/check-NA.R b/local/check-NA.R index 7405db4..95c158e 100644 --- a/local/check-NA.R +++ b/local/check-NA.R @@ -1,4 +1,3 @@ - ### Scope: Check whether there are any incomplete entries in PACKAGES.rds. These will result in NA when calling available.packages() foo = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib") sum((is.na(foo[, "Version"]))) @@ -17,7 +16,9 @@ foo = available.packages("https://cran.devxy.io/amd64/rhel9/latest/src/contrib") sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) -foo = available.packages("https://cran.devxy.io/amd64/alpine320/latest/src/contrib") +foo = available.packages( + "https://cran.devxy.io/amd64/alpine320/latest/src/contrib" +) sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) @@ -40,6 +41,8 @@ foo = available.packages("https://cran.devxy.io/arm64/rhel9/latest/src/contrib") sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) -foo = available.packages("https://cran.devxy.io/arm64/alpine320/latest/src/contrib") +foo = available.packages( + "https://cran.devxy.io/arm64/alpine320/latest/src/contrib" +) sum((is.na(foo[, "Version"]))) which((is.na(foo[, "Version"]))) diff --git a/local/debug-packages-writing.R b/local/debug-packages-writing.R index 5a0fd24..533ae2b 100644 --- a/local/debug-packages-writing.R +++ b/local/debug-packages-writing.R @@ -2,42 +2,58 @@ library(future) future::plan(multisession) future::plan(sequential) time = Sys.time() -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive"), recurse = T) +files <- s3fs::s3_dir_ls( + sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib/Archive" + ), + recurse = T +) Sys.time() - time - archive <- llply(dirs, function(dir) { - files <- list.files(dir, recursive = FALSE, full.names = TRUE, pattern = "*.tar.gz") + files <- list.files( + dir, + recursive = FALSE, + full.names = TRUE, + pattern = "*.tar.gz" + ) if (length(files) == 0) { print(paste0("Error: Empty directory: ", dir)) return(NULL) } info <- file.info(files) - tryCatch({ - rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) - }, error = function(e) { - print(paste0("Error: Exception catched for Archived directory: ", dir)) - print(e) - return(NULL) - }) + tryCatch( + { + rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) + }, + error = function(e) { + print(paste0("Error: Exception catched for Archived directory: ", dir)) + print(e) + return(NULL) + } + ) info }) -tryCatch({ - rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) -}, error = function(e) { - print(paste0("Error: Exception catched for Archived directory: ", dir)) - print(e) - return(NULL) -}) +tryCatch( + { + rownames(info) <- paste0(basename(dirname(files)), "/", basename(files)) + }, + error = function(e) { + print(paste0("Error: Exception catched for Archived directory: ", dir)) + print(e) + return(NULL) + } +) - - -curl::curl_download("https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db") +curl::curl_download( + "https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", + "PACKAGES.db" +) con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db") df = DBI::dbReadTable(con, "packages") @@ -50,6 +66,14 @@ which(grepl("digest", df$Package)) system("cat /tmp/PACKAGES | grep '^Package: digest' | wc -l") -curl::curl_fetch_memory("https://cloud.r-project.org/src/contrib/Meta/archive.rds") -remotes = readRDS(url("https://cloud.r-project.org/src/contrib/Meta/archive.rds", "rb")) -packages = readRDS(url("https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds", "rb")) +curl::curl_fetch_memory( + "https://cloud.r-project.org/src/contrib/Meta/archive.rds" +) +remotes = readRDS(url( + "https://cloud.r-project.org/src/contrib/Meta/archive.rds", + "rb" +)) +packages = readRDS(url( + "https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds", + "rb" +)) diff --git a/local/dedupe-audit-issue.R b/local/dedupe-audit-issue.R new file mode 100644 index 0000000..c78015a --- /dev/null +++ b/local/dedupe-audit-issue.R @@ -0,0 +1,150 @@ +# One-off maintenance: collapse the duplicate arch subsections that accumulated +# in the "Missing package binaries for latest version ()" issues. +# +# A bug in weekly-missing-binaries-audit.R matched the existing "### " +# subsection by its bare header while writing headers with a +# " (N missing, M to rebuild)" suffix, so every audit run appended a fresh block +# instead of replacing it. This script rewrites each "## " section to +# keep only the *last* (freshest) block per arch. The audit fix prevents further +# accumulation; this cleans up what is already there. +# +# Env: FORGEJO_TOKEN (required). DRY_RUN=1 to preview counts without patching. + +library(httr2, quietly = TRUE) + +forgejo_base <- "https://git.devxy.io/api/v1" +repo <- "devxy/build-cran-binaries" +token <- Sys.getenv("FORGEJO_TOKEN") +dry_run <- nchar(Sys.getenv("DRY_RUN")) > 0 + +if (nchar(token) == 0) { + stop("FORGEJO_TOKEN env var is not set") +} + +issue_titles <- c( + "Missing package binaries for latest version (Alpine)", + "Missing package binaries for latest version (Ubuntu)", + "Missing package binaries for latest version (Red Hat)" +) + +# Collapse one "## " block: keep only the last block per arch, +# emitted in order of first appearance. `pl[1]` is the "## " header. +dedupe_platform <- function(pl) { + sub_hdr <- which(grepl("^### ", pl)) + if (length(sub_hdr) == 0) { + return(pl) + } + preamble <- pl[seq_len(sub_hdr[1] - 1)] + sub_end <- c(sub_hdr[-1] - 1, length(pl)) + blocks <- lapply(seq_along(sub_hdr), function(k) { + pl[seq(sub_hdr[k], sub_end[k])] + }) + arches <- vapply( + blocks, + function(b) sub("^### (\\S+).*", "\\1", b[1]), + character(1) + ) + # Index of the last block for each arch, kept in first-appearance order. + last_idx <- vapply( + unique(arches), + function(a) max(which(arches == a)), + integer(1) + ) + keep <- sort(last_idx) + out <- preamble + for (i in keep) { + out <- c(out, blocks[[i]]) + } + out +} + +process_issue <- function(title) { + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(title, reserved = TRUE) + ) + resp <- request(search_url) |> + req_headers(Authorization = paste("token", token)) |> + req_perform() + issues <- resp_body_json(resp, simplifyVector = FALSE) + match_idx <- which(vapply(issues, function(x) x$title, character(1)) == title) + if (length(match_idx) == 0) { + cat(sprintf("[skip] No issue found: %s\n", title)) + return(invisible()) + } + + issue_number <- issues[[match_idx[1]]]$number + body <- issues[[match_idx[1]]]$body + if (is.null(body) || nchar(body) == 0) { + cat(sprintf("[skip] Empty body: #%d %s\n", issue_number, title)) + return(invisible()) + } + + lines <- strsplit(body, "\n", fixed = TRUE)[[1]] + before <- sum(grepl("^### ", lines)) + + # Split off the "## Excluded packages" footer so it is preserved verbatim. + excl_idx <- which(lines == "## Excluded packages") + footer <- character(0) + if (length(excl_idx) > 0) { + pre_dash <- which(lines == "---" & seq_along(lines) < excl_idx[1]) + cut <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_idx[1] + footer <- lines[seq(cut, length(lines))] + lines <- lines[seq_len(cut - 1)] + } + + # Platform headers ("## "); everything before the first is preamble. + plat_idx <- which(grepl("^## ", lines)) + if (length(plat_idx) == 0) { + cat(sprintf("[skip] No platform sections: #%d %s\n", issue_number, title)) + return(invisible()) + } + top <- lines[seq_len(plat_idx[1] - 1)] + plat_end <- c(plat_idx[-1] - 1, length(lines)) + + new_lines <- top + for (j in seq_along(plat_idx)) { + pl <- lines[seq(plat_idx[j], plat_end[j])] + new_lines <- c(new_lines, dedupe_platform(pl)) + } + if (length(footer) > 0) { + new_lines <- c(new_lines, footer) + } + + after <- sum(grepl("^### ", new_lines)) + cat(sprintf( + "#%d %s: %d -> %d arch subsections%s\n", + issue_number, + title, + before, + after, + if (dry_run) " (dry run, not patched)" else "" + )) + + if (dry_run) { + return(invisible()) + } + + patch_url <- sprintf( + "%s/repos/%s/issues/%d", + forgejo_base, + repo, + issue_number + ) + request(patch_url) |> + req_headers( + Authorization = paste("token", token), + `Content-Type` = "application/json" + ) |> + req_body_json(list(body = paste(new_lines, collapse = "\n"))) |> + req_method("PATCH") |> + req_perform() + cat(sprintf(" patched #%d\n", issue_number)) +} + +for (t in issue_titles) { + process_issue(t) +} +cat("Done.\n") diff --git a/local/detect-duplicates.R b/local/detect-duplicates.R index 13f8603..f7e2c18 100644 --- a/local/detect-duplicates.R +++ b/local/detect-duplicates.R @@ -1,27 +1,66 @@ +library(future.apply) + +access_key <- Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S") +secret_key <- Sys.getenv("HETZNER_S3_SECRET_KEY_K3S") + +# Set up S3 connection s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), - aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"), + aws_access_key_id = access_key, + aws_secret_access_key = secret_key, endpoint = "https://hel1.your-objectstorage.com", - region_name = "hel1", + region_name = "hel1" ) -### Scope: Shows if there are duplicated packages at the S3 repository root (e.g. due to missed/aborted archiving) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine321/latest/src/contrib")) +# Define architectures and codenames +arches <- c("amd64", "arm64") +codenames <- c("jammy", "noble", "rhel8", "rhel9", "alpine320", "alpine321") -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib")) +# Create all combinations as a data frame +combos <- expand.grid(arch = arches, codename = codenames) -files_b=basename(files) -files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) -files_b_split[duplicated(files_b_split)] - -bincraftR::archive_package("Keng", codename = "alpine320", arch = "arm64") +# Set up parallel plan +future::plan(multisession) # or multicore +# Parallel processing +future.apply::future_lapply( + seq_len(nrow(combos)), + function(i) { + s3fs::s3_file_system( + aws_access_key_id = access_key, + aws_secret_access_key = secret_key, + endpoint = "https://hel1.your-objectstorage.com", + region_name = "hel1", + refresh = TRUE + ) + arch <- combos$arch[i] + codename <- combos$codename[i] + files <- s3fs::s3_dir_ls( + sprintf( + "devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", + arch, + codename + ) + ) + if (length(files) == 0) { + return(NULL) + } + files_b <- basename(files) + pkg_names <- sapply(strsplit(files_b, "_"), `[`, 1) + dupes <- unique(pkg_names[duplicated(pkg_names)]) + for (pkg in dupes) { + bincraft::archive_package( + pkg, + codename = codename, + arch = arch, + s3_region = "hel1", + s3_endpoint = "https://hel1.your-objectstorage.com", + s3_bucket = "devxy-r-package-binaries-hel1", + s3_access_key_id = access_key, + s3_secret_access_key = secret_key + ) + } + NULL + }, + future.globals = TRUE, + future.seed = TRUE +) diff --git a/local/excluded-packages.json b/local/excluded-packages.json new file mode 100644 index 0000000..6489998 --- /dev/null +++ b/local/excluded-packages.json @@ -0,0 +1,77 @@ +[ + { "package": "RInno", "reason": "windows-only" }, + { "package": "KeyboardSimulator", "reason": "windows-only" }, + { "package": "R2PPT", "reason": "windows-only" }, + { "package": "RWinEdt", "reason": "windows-only" }, + { "package": "blatr", "reason": "windows-only" }, + { "package": "excel.link", "reason": "windows-only" }, + { "package": "spectrino", "reason": "windows-only" }, + { "package": "taskscheduleR", "reason": "windows-only" }, + { "package": "MDSGUI", "reason": "windows-only" }, + { "package": "BiplotGUI", "reason": "windows-only" }, + { "package": "R2wd", "reason": "windows-only" }, + { "package": "rFUSION", "reason": "windows-only" }, + { "package": "MediaNews", "reason": "windows-only" }, + { "package": "doBy", "reason": "hang" }, + { "package": "IDPmisc", "reason": "hang" }, + { "package": "frailtypack", "reason": "hang" }, + { "package": "afex", "reason": "hang" }, + { "package": "FrF2", "reason": "hang" }, + { "package": "DoE.base", "reason": "hang" }, + { "package": "agricolae", "reason": "hang" }, + { "package": "doFuture", "reason": "hang" }, + { "package": "fscaret", "reason": "hang" }, + { "package": "PHYLOGR", "reason": "hang" }, + { "package": "seewave", "reason": "hang" }, + { "package": "pls", "reason": "hang" }, + { "package": "relaimpo", "reason": "hang" }, + { "package": "geepack", "reason": "hang" }, + { "package": "gggenes", "reason": "hang" }, + { "package": "NPCirc", "reason": "hang" }, + { "package": "repmis", "reason": "hang" }, + { "package": "PNDSIBGE", "reason": "hang" }, + { "package": "lidR", "reason": "hang" }, + { "package": "poismf", "reason": "hang" }, + { "package": "neonstore", "reason": "hang" }, + { "package": "MachineShop", "reason": "hang" }, + { "package": "mvst", "reason": "hang" }, + { "package": "MacBehaviour", "reason": "hang" }, + { "package": "mcmcderive", "reason": "hang" }, + { "package": "RGIFT", "reason": "hang" }, + { "package": "KnowBR", "reason": "hang" }, + { "package": "netmeta", "reason": "hang" }, + { "package": "spdep", "reason": "hang" }, + { "package": "Rfast", "reason": "hang" }, + { "package": "compareGroups", "reason": "hang" }, + { "package": "ff", "reason": "hang" }, + { "package": "GsymPoint", "reason": "hang" }, + { "package": "RcppDynProg", "reason": "hang" }, + { "package": "comtradr", "reason": "hang" }, + { "package": "FD", "reason": "hang" }, + { "package": "PearsonDS", "reason": "hang" }, + { "package": "DCluster", "reason": "hang" }, + { "package": "gRc", "reason": "hang" }, + { "package": "mixlm", "reason": "hang" }, + { "package": "geospt", "reason": "hang" }, + { "package": "fdth", "reason": "hang" }, + { "package": "ffmanova", "reason": "hang" }, + { "package": "fiery", "reason": "hang" }, + { "package": "ffscrapr", "reason": "hang" }, + { "package": "cold", "reason": "hang" }, + { "package": "RcmdrPlugin.DoE", "reason": "hang" }, + { "package": "RcmdrPlugin.NMBU", "reason": "hang" }, + { "package": "RcmdrPlugin.RiskDemo", "reason": "hang" }, + { "package": "RcmdrPlugin.ROC", "reason": "hang" }, + { "package": "RcmdrPlugin.TeachStat", "reason": "hang" }, + { "package": "RcmdrPlugin.TeachingDemos", "reason": "hang" }, + { "package": "RcmdrPlugin.UCA", "reason": "hang" }, + { "package": "RcmdrPlugin.WorldFlora", "reason": "hang" }, + { "package": "RcmdrPlugin.aRnova", "reason": "hang" }, + { "package": "RcmdrPlugin.depthTools", "reason": "hang" }, + { "package": "RcmdrPlugin.orloca", "reason": "hang" }, + { "package": "RcmdrPlugin.sos", "reason": "hang" }, + { "package": "RcmdrPlugin.survival", "reason": "hang" }, + { "package": "RcmdrPlugin.temis", "reason": "hang" }, + { "package": "GWlasso", "reason": "hang" }, + { "package": "GWmodelVis", "reason": "hang" } +] diff --git a/local/failing-builds-classify.R b/local/failing-builds-classify.R new file mode 100644 index 0000000..73922b3 --- /dev/null +++ b/local/failing-builds-classify.R @@ -0,0 +1,399 @@ +# Pure, side-effect-free helpers for triaging `single_builds` failures: +# normalise a raw `error_text` into a stable fingerprint, and classify it +# against a seed set of known failure signatures (issue #115, steps 1 + 2). +# +# Kept free of DB/IO so it can be sourced by both `failing-builds-report.R` +# and the unit tests in `local/tests/`. + +# --------------------------------------------------------------------------- +# Signature table +# --------------------------------------------------------------------------- +# Each rule maps a recurring compile/link/load error to a suggested fix tier. +# `pattern` is a case-insensitive regex matched against the raw `error_text`. +# `auto` marks whether the fix is a *known lever* safe to auto-propose as a PR +# (env / makevars / an already-curated package-specific patch). Rules that +# would require a brand-new source diff for a previously-unseen package stay +# `auto = FALSE` -> classified, but always routed to human triage, per the +# issue's guardrail against shipping autonomous novel source diffs. +# `applies_to` (optional) pins a package-specific fix to the package it targets: +# a signature whose registry patch belongs to one package (e.g. RcppParallel's +# disable-tbb patch) is only proposed for that package. Other packages matching +# the signature are downstream failures blocked on that dependency, not +# individually patchable, so they are reported but never proposed an entry. +# +# Seeded from the existing registry entries and known recurring failures; add +# a row here as new signatures are confirmed. Order matters: the first match +# wins, so keep more specific patterns above broader ones. +build_signatures <- function() { + list( + list( + id = "tbb-stddef-removed", + label = "removed TBB header tbb/tbb_stddef.h", + pattern = "tbb/tbb_stddef\\.h.*No such file", + tier = "makevars", + confidence = "high", + auto = TRUE, + fix = "add CPPFLAGS += -DTBB_INTERFACE_NEW so the source stops including the removed tbb/tbb_stddef.h header", + example = "StanHeaders / rstan (#114)", + registry = list( + env = NULL, + configure_args = NULL, + makevars = list(CPPFLAGS = "-DTBB_INTERFACE_NEW"), + patch = NULL, + reason = "package includes the removed tbb/tbb_stddef.h; -DTBB_INTERFACE_NEW selects the new oneTBB interface path" + ) + ), + list( + id = "rcppparallel-bundled-tbb", + label = "bundled Intel TBB build fails/hangs (musl / new g++)", + pattern = "USE_TBB[^\\n]*(not supported|unsupported)|RcppParallel[^\\n]*TBB|tbb[^\\n]*(Alpine|musl)", + tier = "patch", + confidence = "high", + auto = TRUE, + # The fix is a RcppParallel-specific source patch, so it is only ever + # proposed for RcppParallel itself. The hundreds of packages that fail + # merely because their RcppParallel *dependency* did not build carry + # RcppParallel's error text and match this signature too -- they are + # blocked on RcppParallel, not individually patchable (applying this diff + # to their source is meaningless). `applies_to` pins the proposal so those + # downstream failures are never proposed a bogus per-package entry. + applies_to = "RcppParallel", + fix = "apply the curated RcppParallel/disable-tbb.patch so the bundled TBB build is skipped and the TinyThread backend is used", + example = "RcppParallel", + registry = list( + env = NULL, + configure_args = NULL, + makevars = NULL, + patch = "RcppParallel/disable-tbb.patch", + reason = "bundled Intel TBB build hangs/fails on musl (Alpine) and newer toolchains; patch forces the TinyThread backend" + ) + ), + list( + id = "system-libuv-link-leak", + label = "binary links system libuv (NEEDED libuv.so.1)", + pattern = "libuv\\.so", + tier = "patch", + confidence = "medium", + # A novel per-package source diff is required to force the vendored lib; + # never auto-propose, only surface for a human (guardrail). + auto = FALSE, + fix = "force the vendored/static library instead of the system one via a human-authored source patch (see fs/force-vendored-libuv.patch as precedent)", + example = "fs", + registry = list( + env = NULL, + configure_args = NULL, + makevars = NULL, + patch = "/force-vendored-.patch", + reason = "binary links the system library and fails to dyn.load on consumer machines; force the vendored/static build" + ) + ) + ) +} + +# --------------------------------------------------------------------------- +# Normalisation +# --------------------------------------------------------------------------- +# Collapse build-specific noise (temp paths, version numbers, hex addresses, +# the package name) so the same root cause across packages/platforms/versions +# maps to one fingerprint bucket. +normalise_error <- function(error_text, package = NULL) { + if (length(error_text) == 0L || is.na(error_text) || !nzchar(error_text)) { + return("") + } + x <- as.character(error_text) + # Package-specific token first (before version/number stripping mangles it). + if (!is.null(package) && length(package) == 1L && nzchar(package)) { + # \Q..\E quotes the name literally so metachars (e.g. data.table's dot) + # are matched verbatim rather than as regex. + x <- gsub(paste0("\\b\\Q", package, "\\E\\b"), "", x, perl = TRUE) + } + # R temp dirs/files: /tmp/RtmpAbC123, RtmpXXXX, /tmp/Rtmp.../file123. + x <- gsub("/tmp/[^ \t\n]*", "", x, perl = TRUE) + x <- gsub("\\bRtmp[A-Za-z0-9]+", "Rtmp", x, perl = TRUE) + # Hex addresses and version-like number runs. + x <- gsub("0x[0-9a-fA-F]+", "0x", x, perl = TRUE) + x <- gsub("[0-9]+(\\.[0-9]+)+", "", x, perl = TRUE) + x <- gsub("\\b[0-9]{2,}\\b", "", x, perl = TRUE) + # Whitespace and case. + x <- tolower(x) + x <- gsub("[ \t\r\n]+", " ", x, perl = TRUE) + trimws(x) +} + +# Extract the single most informative line from a multi-line error, then +# normalise it. This is the grouping key; a short salient line groups far +# better than the whole (often huge) transcript. +fingerprint_error <- function(error_text, package = NULL, max_chars = 200L) { + if (length(error_text) == 0L || is.na(error_text) || !nzchar(error_text)) { + return("") + } + lines <- strsplit(as.character(error_text), "\n", fixed = TRUE)[[1L]] + lines <- trimws(lines) + lines <- lines[nzchar(lines)] + if (length(lines) == 0L) { + return("") + } + salient_re <- paste( + "error:", + "fatal error:", + "no such file", + "undefined reference", + "cannot find -l", + "cannot open shared object", + "configuration failed", + "non-zero exit", + "installation of package", + "compilation failed", + sep = "|" + ) + hit <- lines[grepl(salient_re, lines, ignore.case = TRUE)] + chosen <- if (length(hit) > 0L) hit[[1L]] else lines[[length(lines)]] + fp <- normalise_error(chosen, package) + if (nchar(fp) > max_chars) { + fp <- paste0(substr(fp, 1L, max_chars), "...") + } + fp +} + +# --------------------------------------------------------------------------- +# Dependency-cascade detection +# --------------------------------------------------------------------------- +# If a build's error_text shows the failure was actually in a DIFFERENT package +# (a dependency that would not compile), return that dependency's name; +# otherwise NA. Used to avoid proposing a per-package fix for a package that +# only fails because a shared dependency does not build (e.g. the ~73 Stan +# packages that fail while building `rstan`). Generalises the RcppParallel +# `applies_to` guard to any dependency named in the log. +failing_dependency <- function(error_text, package) { + if (length(error_text) == 0L || is.na(error_text) || !nzchar(error_text)) { + return(NA_character_) + } + x <- as.character(error_text) + # optional opening quote before the package name: apostrophe, double-quote, + # backtick, or curly quotes -- written as \u escapes so the pattern stays + # valid UTF-8 regardless of source encoding. + q <- "[\u0027\u0022\u0060\u2018\u2019]?" + name <- "([A-Za-z][A-Za-z0-9._]+)" + # Markers R/pak emit naming the package that actually failed to compile. + pats <- c( + paste0("compilation failed for package ", q, name), + paste0("Failed to build source package ", q, name), + paste0("Error in building package ", q, name), + paste0("dependenc(?:y|ies) ", q, name, q, "?[^\\n]*not available") + ) + deps <- character(0L) + for (p in pats) { + hits <- regmatches(x, gregexpr(p, x, perl = TRUE))[[1L]] + if (length(hits) > 0L) { + deps <- c(deps, sub(p, "\\1", hits, perl = TRUE)) + } + } + deps <- sub("[._]+$", "", deps) # drop a trailing sentence period (e.g. "rstan.") + deps <- setdiff(unique(deps), package) # a package failing on its OWN code is not a cascade + if (length(deps) == 0L) NA_character_ else deps[[1L]] +} + +# --------------------------------------------------------------------------- +# Classification +# --------------------------------------------------------------------------- +# Return the first matching signature (as a list) enriched with `matched`, or +# the unclassified fallback. Never guesses: an unmatched error is routed to a +# human, not assigned a fix. +classify_error <- function(error_text, signatures = build_signatures()) { + txt <- if (length(error_text) == 0L || is.na(error_text)) { + "" + } else { + as.character(error_text) + } + for (sig in signatures) { + if ( + nzchar(txt) && grepl(sig$pattern, txt, ignore.case = TRUE, perl = TRUE) + ) { + sig$matched <- TRUE + return(sig) + } + } + list( + id = "unclassified", + label = "unknown signature", + tier = NA_character_, + confidence = NA_character_, + auto = FALSE, + fix = "no known signature; flag for human triage", + example = NA_character_, + registry = NULL, + matched = FALSE + ) +} + +# Build a suggested registry.json entry (as an R list) for a classified group, +# filling package/versions/platforms from the observed failures. Only +# meaningful when the signature carries a `registry` template. +propose_registry_entry_list <- function( + signature, + package, + platforms, + versions = "*" +) { + if (is.null(signature$registry)) { + return(NULL) + } + tmpl <- signature$registry + entry <- list( + package = package, + versions = versions, + # I() keeps this a JSON array even when a single platform is affected. + platforms = I(sort(unique(as.character(platforms)))) + ) + for (k in c("env", "configure_args", "makevars", "patch")) { + if (!is.null(tmpl[[k]])) { + entry[[k]] <- tmpl[[k]] + } + } + entry$reason <- tmpl$reason + entry +} + +# Same, rendered as a pretty JSON string. +propose_registry_entry <- function( + signature, + package, + platforms, + versions = "*" +) { + entry <- propose_registry_entry_list(signature, package, platforms, versions) + if (is.null(entry)) { + return(NULL) + } + jsonlite::toJSON(entry, auto_unbox = TRUE, pretty = TRUE, null = "null") +} + +# Resolve the representative signature for a group by its id, re-attaching the +# `matched` flag the report/proposer rely on. Unknown ids -> unclassified. +resolve_signature <- function(sig_id, signatures = build_signatures()) { + hit <- Filter(function(s) s$id == sig_id, signatures) + if (length(hit) > 0L) { + s <- hit[[1L]] + s$matched <- TRUE + s + } else { + classify_error("", signatures) # unclassified fallback (matched = FALSE) + } +} + +# --------------------------------------------------------------------------- +# Triage report (shared by the report printer and the patch proposer) +# --------------------------------------------------------------------------- +# `failures` is a data.frame with at least: name, platform, arch, error_text. +# `registered_pkgs` is the set of packages that already carry a registry entry. +# Returns a list of group records (one per root-cause bucket), ordered by +# number of affected builds descending. Pure: no IO, no printing. +build_triage_report <- function( + failures, + registered_pkgs = character(0L), + signatures = build_signatures() +) { + if (nrow(failures) == 0L) { + return(list()) + } + idx <- seq_len(nrow(failures)) + failures$fingerprint <- vapply( + idx, + function(i) fingerprint_error(failures$error_text[[i]], failures$name[[i]]), + character(1L) + ) + failures$sig_id <- vapply( + idx, + function(i) classify_error(failures$error_text[[i]], signatures)$id, + character(1L) + ) + # Group by root cause: classified failures collapse by signature id (same fix + # candidate is one bucket regardless of log noise); unclassified failures fall + # back to the fingerprint so distinct unknowns stay separate for discovery. + failures$group_key <- ifelse( + failures$sig_id == "unclassified", + failures$fingerprint, + failures$sig_id + ) + groups <- split(failures, failures$group_key) + groups <- groups[order(-vapply(groups, nrow, integer(1L)))] + + lapply(unname(groups), function(g) { + sig_id <- names(sort(table(g$sig_id), decreasing = TRUE))[[1L]] + sig <- resolve_signature(sig_id, signatures) + + fp_tab <- sort(table(g$fingerprint), decreasing = TRUE) + pkgs <- sort(unique(g$name)) + plats <- sort(unique(g$platform)) + arches <- sort(unique(g$arch)) + unregistered <- setdiff(pkgs, registered_pkgs) + auto_proposable <- isTRUE(sig$auto) && sig$matched + + # Decide, per package, whether it is genuinely fixable or merely blocked on + # a dependency (so a per-package entry would be useless). Two blocking modes: + # 1. `applies_to`: a package-specific patch (e.g. RcppParallel's) is only + # valid for its own package; other matches are downstream of it. + # 2. data-driven cascade: the package's error_text shows a *different* + # package failed to compile (e.g. the ~73 Stan packages blocked on rstan). + # `blocked_map` maps a blocked package -> the dependency it waits on. + g$blocked_dep <- vapply( + seq_len(nrow(g)), + function(i) failing_dependency(g$error_text[[i]], g$name[[i]]), + character(1L) + ) + proposable <- unregistered + blocked_map <- list() + if (!is.null(sig$applies_to)) { + for (p in setdiff(pkgs, sig$applies_to)) { + blocked_map[[p]] <- sig$applies_to + } + proposable <- intersect(proposable, sig$applies_to) + } + for (p in proposable) { + deps <- g$blocked_dep[g$name == p] + if (!any(is.na(deps))) { + # every failing build of p is a cascade -> blocked, not fixable here + blocked_map[[p]] <- unique(deps) + } + } + proposable <- setdiff(proposable, names(blocked_map)) + blocked_packages <- names(blocked_map) + blocked_on <- unique(unlist(blocked_map, use.names = FALSE)) + if (length(blocked_on) == 0L) { + blocked_on <- NULL + } + + proposed <- list() + if (auto_proposable) { + for (p in proposable) { + proposed[[p]] <- propose_registry_entry_list( + sig, + p, + g$platform[g$name == p] + ) + } + } + + list( + signature = sig$id, + label = sig$label, + matched = sig$matched, + auto_proposable = auto_proposable, + tier = sig$tier, + confidence = sig$confidence, + suggested_fix = sig$fix, + applies_to = sig$applies_to, + blocked_on = blocked_on, + blocked_packages = blocked_packages, + blocked_map = if (length(blocked_map) > 0L) blocked_map else NULL, + fingerprint = names(fp_tab)[[1L]], + fingerprint_variants = length(fp_tab), + build_count = nrow(g), + packages = pkgs, + packages_without_entry = unregistered, + platforms = plats, + arches = arches, + proposed_entries = if (length(proposed) > 0L) proposed else NULL + ) + }) +} diff --git a/local/failing-builds-report.R b/local/failing-builds-report.R new file mode 100644 index 0000000..ce7b683 --- /dev/null +++ b/local/failing-builds-report.R @@ -0,0 +1,234 @@ +#!/usr/bin/env Rscript + +# Read-only failure triage over the `single_builds` metadata table (issue #115, +# steps 1 + 2): query every recorded build failure, group by a normalised error +# fingerprint, classify each bucket against the known signature set, and print a +# triaged report with a pre-filled `registry.json` suggestion where a *known, +# safe* fix lever applies. Novel source diffs and unknown signatures are routed +# to human triage; this script never writes to the DB or the registry. +# +# Usage: +# PGPASS=... Rscript local/failing-builds-report.R [--platform P] [--arch A] +# [--json out.json] [--min N] +# +# --platform / --arch restrict to one platform/arch (default: all) +# --min N only show fingerprint groups with >= N failing builds +# --json PATH also write the machine-readable report to PATH +# +# Env fallbacks: PLATFORM, ARCH (same effect as the flags). + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(DBI, quietly = TRUE) + library(RPostgres, quietly = TRUE) + library(jsonlite, quietly = TRUE) +}) + +# Locate helpers relative to this script so it runs from any CWD. +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "failing-builds-classify.R")) + +# --------------------------------------------------------------------------- +# Arguments +# --------------------------------------------------------------------------- +args <- commandArgs(trailingOnly = TRUE) +opt_val <- function(flag, default = NA_character_) { + i <- match(flag, args) + if (!is.na(i) && i < length(args)) args[[i + 1L]] else default +} +platform <- opt_val("--platform", Sys.getenv("PLATFORM", "")) +arch <- opt_val("--arch", Sys.getenv("ARCH", "")) +json_out <- opt_val("--json") +min_count <- suppressWarnings(as.integer(opt_val("--min", "1"))) +if (is.na(min_count)) { + min_count <- 1L +} + +if (nchar(Sys.getenv("PGPASS")) == 0L) { + stop( + "PGPASS env var is not set; a DB password is required (no read-only role exists)." + ) +} + +# --------------------------------------------------------------------------- +# Query failing builds +# --------------------------------------------------------------------------- +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(DBI::dbDisconnect(con), add = TRUE) + +where <- "error_occurred = TRUE AND removed = FALSE" +params <- list() +if (nzchar(platform)) { + where <- paste0(where, " AND platform = $", length(params) + 1L) + params <- c(params, platform) +} +if (nzchar(arch)) { + where <- paste0(where, " AND arch = $", length(params) + 1L) + params <- c(params, arch) +} +failures <- DBI::dbGetQuery( + con, + paste0( + "SELECT name, tag, platform, arch, r_version, timestamp, error_text ", + "FROM single_builds WHERE ", + where + ), + params = if (length(params) > 0L) params else NULL +) + +cat(sprintf( + "Queried single_builds: %d failing builds%s\n", + nrow(failures), + if (nzchar(platform) || nzchar(arch)) { + sprintf( + " (filter: platform=%s arch=%s)", + if (nzchar(platform)) platform else "*", + if (nzchar(arch)) arch else "*" + ) + } else { + "" + } +)) +if (nrow(failures) == 0L) { + cat("No failing builds to triage.\n") + q(status = 0) +} + +# --------------------------------------------------------------------------- +# Packages that already carry a registry entry (so we don't re-propose) +# --------------------------------------------------------------------------- +`%||%` <- function(a, b) if (is.null(a)) b else a +registry_file <- file.path(script_dir, "patches", "registry.json") +registered_pkgs <- character(0L) +if (file.exists(registry_file)) { + reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) + registered_pkgs <- unique(vapply( + reg, + function(e) as.character(e$package %||% ""), + character(1L) + )) +} + +# --------------------------------------------------------------------------- +# Classify + group (shared logic), then render +# --------------------------------------------------------------------------- +report <- build_triage_report(failures, registered_pkgs) +report <- Filter(function(r) r$build_count >= min_count, report) + +cat(sprintf( + "\n%d distinct failure group(s); showing groups with >= %d build(s).\n", + length(report), + min_count +)) +cat(strrep("=", 78L), "\n", sep = "") + +for (r in report) { + status <- if (!r$matched) { + "HUMAN TRIAGE (unknown signature)" + } else if (!r$auto_proposable) { + sprintf("HUMAN TRIAGE (classified: %s; novel source diff)", r$signature) + } else { + sprintf("AUTO-PROPOSABLE (%s, %s confidence)", r$signature, r$confidence) + } + + cat(sprintf( + "\n[%d builds | %d pkgs | %s] %s\n", + r$build_count, + length(r$packages), + toString(r$platforms), + status + )) + cat(sprintf( + " fingerprint: %s%s\n", + r$fingerprint, + if (r$fingerprint_variants > 1L) { + sprintf(" (+%d fingerprint variant(s))", r$fingerprint_variants - 1L) + } else { + "" + } + )) + if (r$matched) { + cat(sprintf(" signature : %s\n", r$label)) + cat(sprintf(" suggested : [%s] %s\n", r$tier, r$suggested_fix)) + } + cat(sprintf(" arch : %s\n", toString(r$arches))) + cat(sprintf( + " packages : %s\n", + paste( + vapply( + r$packages, + function(p) { + if (p %in% registered_pkgs) paste0(p, " (has entry)") else p + }, + character(1L) + ), + collapse = ", " + ) + )) + + if (!is.null(r$proposed_entries)) { + cat(" proposed registry entries (validate + trial-build before merge):\n") + for (entry in r$proposed_entries) { + j <- jsonlite::toJSON( + entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(paste0(" ", gsub("\n", "\n ", j)), "\n", sep = "") + } + } else if (r$auto_proposable && length(r$blocked_packages) == 0L) { + cat(" (all affected packages already have a registry entry)\n") + } + # Blocked packages are shown even when the group also has proposals. + if (length(r$blocked_packages) > 0L) { + cat(sprintf( + " (blocked on %s -- %d package(s) fail because that dependency does not build; fix %s, do not patch each dependent)\n", + toString(r$blocked_on), + length(r$blocked_packages), + toString(r$blocked_on) + )) + } +} + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +matched <- Filter(function(r) r$matched, report) +auto <- Filter(function(r) r$auto_proposable, report) +cat("\n", strrep("=", 78L), "\n", sep = "") +cat(sprintf( + "Summary: %d groups | %d classified | %d auto-proposable | %d for human triage\n", + length(report), + length(matched), + length(auto), + length(report) - length(matched) +)) + +if (!is.na(json_out)) { + jsonlite::write_json( + report, + json_out, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf("Wrote machine-readable report to %s\n", json_out)) +} diff --git a/local/fetch-rebuild-packages-from-issue.R b/local/fetch-rebuild-packages-from-issue.R new file mode 100644 index 0000000..9d8043c --- /dev/null +++ b/local/fetch-rebuild-packages-from-issue.R @@ -0,0 +1,140 @@ +library(httr2, quietly = TRUE) + +forgejo_base <- "https://git.devxy.io/api/v1" +repo <- "devxy/build-cran-binaries" +platform <- Sys.getenv("PLATFORM") +arch <- Sys.getenv("ARCH") +output_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") + +if (nchar(platform) == 0) { + stop("PLATFORM env var is not set") +} +if (nchar(arch) == 0) { + stop("ARCH env var is not set") +} + +# Prefer the audit's freshly-written RDS. The audit overwrites it each run +# (saveRDS), so unlike the Forgejo issue body it is never subject to the +# duplicate-subsection accumulation bug. Fall back to parsing the issue when the +# RDS is absent (e.g. a fresh runner with no shared cache). +rds_file <- file.path( + Sys.getenv("REBUILD_PKG_RDS_DIR", "/mnt/cache/packages"), + sprintf("weekly_rebuild_%s_%s.rds", platform, arch) +) +if (file.exists(rds_file)) { + pkgs <- tryCatch(as.character(readRDS(rds_file)), error = function(e) NULL) + if (!is.null(pkgs) && length(pkgs) > 0) { + cat(sprintf( + "Using audit RDS %s: %d rebuildable packages for %s/%s\n", + rds_file, + length(pkgs), + platform, + arch + )) + writeLines(pkgs, output_file) + cat(sprintf("Wrote package list to %s\n", output_file)) + q("no") + } + cat(sprintf( + "RDS %s present but empty/unreadable -- falling back to issue\n", + rds_file + )) +} + +token <- Sys.getenv("FORGEJO_TOKEN") +if (nchar(token) == 0) { + stop("FORGEJO_TOKEN env var is not set") +} + +os_family <- if (grepl("^ubuntu", platform)) { + "Ubuntu" +} else if (grepl("^alpine", platform)) { + "Alpine" +} else if (grepl("^redhat", platform)) { + "Red Hat" +} else { + platform +} + +issue_title <- sprintf( + "Missing package binaries for latest version (%s)", + os_family +) +cat(sprintf("Searching for issue: %s\n", issue_title)) + +search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(issue_title, reserved = TRUE) +) +search_resp <- request(search_url) |> + req_headers(Authorization = paste("token", token)) |> + req_perform() + +issues <- resp_body_json(search_resp, simplifyVector = FALSE) +match_idx <- which( + vapply(issues, function(x) x$title, character(1)) == issue_title +) + +if (length(match_idx) == 0) { + cat("No matching issue found - nothing to rebuild\n") + writeLines(character(0), output_file) + q("no") +} + +body <- issues[[match_idx[1]]]$body +if (is.null(body) || nchar(body) == 0) { + cat("Issue body is empty - nothing to rebuild\n") + writeLines(character(0), output_file) + q("no") +} + +lines <- strsplit(body, "\n", fixed = TRUE)[[1]] + +# Find the platform section (## alpine-322) +plat_header <- sprintf("## %s", platform) +plat_idx <- which(lines == plat_header) +if (length(plat_idx) == 0) { + cat(sprintf( + "No section found for platform %s - nothing to rebuild\n", + platform + )) + writeLines(character(0), output_file) + q("no") +} + +# Find the arch subsection (### arm64 (...)) +arch_pattern <- sprintf("^### %s ", arch) +arch_idx <- which(grepl(arch_pattern, lines) & seq_along(lines) > plat_idx[1]) +if (length(arch_idx) == 0) { + cat(sprintf( + "No section found for arch %s under %s - nothing to rebuild\n", + arch, + platform + )) + writeLines(character(0), output_file) + q("no") +} + +ai <- arch_idx[1] + +# Find the end of the rebuildable section: stop at ####, ###, ##, ---, or EOF +end_idx <- which(grepl("^####|^###|^##|^---", lines) & seq_along(lines) > ai) +end_idx <- if (length(end_idx) > 0) end_idx[1] - 1 else length(lines) + +section_lines <- lines[seq(ai + 1, end_idx)] + +# Extract package names from "- PackageName (Version)" lines +pkg_lines <- section_lines[grepl("^- ", section_lines)] +pkgs <- sub("^- ([^ ]+) \\(.*\\)$", "\\1", pkg_lines) +pkgs <- pkgs[nchar(pkgs) > 0 & pkgs != "_None_"] + +cat(sprintf( + "Found %d rebuildable packages for %s/%s\n", + length(pkgs), + platform, + arch +)) +writeLines(pkgs, output_file) +cat(sprintf("Wrote package list to %s\n", output_file)) diff --git a/local/find-R-api-packages.sh b/local/find-R-api-packages.sh new file mode 100644 index 0000000..1f36961 --- /dev/null +++ b/local/find-R-api-packages.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Directory to clone repos into +WORKDIR="cran_repos" +mkdir -p "$WORKDIR" +cd "$WORKDIR" + +# GitHub API paginates results, so we loop through pages +PAGE=1 +PER_PAGE=100 +MATCHES=() + +while :; do + # Fetch a page of repos + REPOS=$(curl -s "https://api.github.com/orgs/cran/repos?per_page=$PER_PAGE&page=$PAGE" | jq -r '.[].clone_url') + [ -z "$REPOS" ] && break + + for REPO_URL in $REPOS; do + REPO_NAME=$(basename "$REPO_URL" .git) + # Skip if already cloned + [ -d "$REPO_NAME" ] && continue + git clone --depth 1 "$REPO_URL" "$REPO_NAME" >/dev/null 2>&1 + if [ -d "$REPO_NAME/src" ]; then + # Search for Rinternals.h in src/ + if grep -r -q 'R_VERSION < R_Version(' "$REPO_NAME/src"; then + echo "$REPO_NAME" + fi + fi + # Clean up to save space + rm -rf "$REPO_NAME" + done + + PAGE=$((PAGE + 1)) +done diff --git a/local/install-bincraft.R b/local/install-bincraft.R new file mode 100644 index 0000000..e39cb4f --- /dev/null +++ b/local/install-bincraft.R @@ -0,0 +1,89 @@ +#!/usr/bin/env Rscript + +# Install the latest tagged bincraft release, resolved dynamically, so the CI +# workflows and the build-one image never pin a hardcoded version (no more +# editing `@vX.Y.Z` in many places on every release). +# +# Run with the R whose library should receive bincraft: +# Rscript local/install-bincraft.R +# or, to target a specific R from a shell loop: +# "$RBIN" -q -e 'source("local/install-bincraft.R")' +# +# How it works: list the remote tags with `git ls-remote` (no token needed for +# the public repo), keep the `vX.Y.Z` release tags, pick the highest version, +# and install it with uvr via `local/uvr-install.sh`. uvr is idempotent on the +# git ref, so re-running keeps the package when it is already current and only +# updates when a newer tag ships. Filtering/sorting is done in R (not via git's +# `--sort`/refspec) so behaviour is identical across git versions and +# `system2()` argument handling. + +repo_url <- Sys.getenv( + "BINCRAFT_GIT_URL", + unset = "https://codefloe.com/rpkgs/bincraft.git" +) + +# GIT_TERMINAL_PROMPT=0 keeps a non-interactive run from hanging on auth. +refs <- system2( + "git", + c("ls-remote", "--tags", repo_url), + stdout = TRUE, + stderr = FALSE, + env = "GIT_TERMINAL_PROMPT=0" +) +tags <- sub(".*refs/tags/", "", refs) +tags <- tags[!grepl("\\^\\{\\}$", tags)] # drop dereferenced "...^{}" lines +tags <- grep("^v[0-9]", tags, value = TRUE) # only vX.Y.Z release tags +if (length(tags) == 0L) { + stop( + "Could not resolve any bincraft release tag from ", + repo_url, + call. = FALSE + ) +} +latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][ + 1L +] + +message(sprintf("Installing latest bincraft release: %s", latest)) + +# uvr addresses Forgejo repos as `forgejo::host/owner/repo@ref` rather than as a +# git URL, so drop the scheme and the trailing `.git` from `repo_url`. +spec <- sprintf( + "forgejo::%s@%s", + sub("\\.git$", "", sub("^[a-z]+://", "", repo_url)), + latest +) + +# `local/uvr-install.sh` when run from the repo root, `/work/local/` in the +# build-one image, which copies the two scripts into a flatter layout. +helper <- Sys.getenv("UVR_INSTALL_SH", unset = "") +if (!nzchar(helper)) { + candidates <- c("local/uvr-install.sh", "/work/local/uvr-install.sh") + found <- candidates[file.exists(candidates)] + if (length(found) == 0L) { + stop("Could not locate uvr-install.sh; set UVR_INSTALL_SH", call. = FALSE) + } + helper <- found[1L] +} + +# Point uvr at the R running this script and at the library it would install +# into, so the per-R-minor passes in the build pipelines (which call a different +# Rscript with R_LIBS_USER pointed elsewhere) target their own R and library. +Sys.setenv( + UVR_R_BIN = file.path(R.home("bin"), "R"), + UVR_TARGET_LIB = .libPaths()[1L] +) + +status <- system2(helper, shQuote(spec)) +if (!identical(status, 0L)) { + stop( + sprintf("uvr failed to install %s (exit %s)", spec, status), + call. = FALSE + ) +} + +message(sprintf( + "bincraft %s installed (%s)", + as.character(utils::packageVersion("bincraft")), + latest +)) diff --git a/local/label-removed-cran-packages.R b/local/label-removed-cran-packages.R index 6fc9f68..12dff9d 100644 --- a/local/label-removed-cran-packages.R +++ b/local/label-removed-cran-packages.R @@ -1,19 +1,23 @@ ### Scope: List decomisisoned CRAN packages and updates them in the metadata DB library(dplyr) -library(bincraftR) +library(bincraft) archived <- quickcode::archivedPkg() |> filter(arch.status == "decom") |> pull(name) # Query all distinct pkgs in the DB -con <- DBI::dbConnect(RPostgres::Postgres(), - dbname = "build_metadata", host = "r-binaries.devxy.io", - port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "r_binaries", + password = Sys.getenv("PGPASS"), sslmode = "require" ) pkgs_db <- query_metadata_table() |> - filter(removed == FALSE) |> + filter(removed == FALSE) |> distinct(name) |> pull(name) @@ -22,7 +26,9 @@ to_process <- pkgs_db[pkgs_db %in% archived] # for all matches, set 'removed = TRUE' sapply(to_process, function(.x) { - DBI::dbExecute(con, "UPDATE single_builds SET removed = 'TRUE' where name = $1", + DBI::dbExecute( + con, + "UPDATE single_builds SET removed = 'TRUE' where name = $1", params = list(.x) ) }) diff --git a/local/last-processed-by-platform.R b/local/last-processed-by-platform.R index 1a0b2b1..398b6c1 100644 --- a/local/last-processed-by-platform.R +++ b/local/last-processed-by-platform.R @@ -1,10 +1,10 @@ ### Scope: Show last processed pkgs and timestamp by platform/OS library(dplyr) -library(bincraftR) +library(bincraft) ### Show last processed package by each platform/arch query_metadata_table() |> - group_by(platform, arch) |> - arrange(desc(timestamp)) |> - select(name, timestamp) |> - filter(row_number()==1) + group_by(platform, arch) |> + arrange(desc(timestamp)) |> + select(name, timestamp) |> + filter(row_number() == 1) diff --git a/local/manual-package-index-update.R b/local/manual-package-index-update.R deleted file mode 100644 index c786c1f..0000000 --- a/local/manual-package-index-update.R +++ /dev/null @@ -1,39 +0,0 @@ -# R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' -# remotes::install_gitlab("devxy/r-package-binaries/bincraftR") - -library(bincraftR) -library(dplyr) -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), - aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"), - endpoint = "https://hel1.your-objectstorage.com", - region_name = "hel1", - refresh = TRUE -) - -s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES.db") -s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES.gz") -s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES.rds") -s3fs::s3_file_delete("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib/PACKAGES") - -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/duckdbfs_0.0.9.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/Archive/duckplyr/duckplyr_0.4.1.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/duckplyr_1.0.0.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/duckduckr_1.0.0.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/getDTeval_0.0.2.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/dualtrees_0.1.5.tar.gz")$etag -s3fs::s3_file_info("devxy-r-package-binaries-hel1/arm64/alpine321/latest/src/contrib/tidyselect_1.2.1.tar.gz") -s3fs::s3_file_info("devxy-r-package-binaries-hel1/arm64/alpine321/latest/src/contrib/tidysq_1.2.3.tar.gz") -# s3fs::s3_file_info("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib/Archive/curl/curl_6.0.1.tar.gz") - -s3fs::s3_file_download("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db") - -upload_package_index(codename = "noble", arch = "arm64") - -fs::file_delete("PACKAGES.db") - - -con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES-debug.db") - -DBI::dbListTables(con) -df = DBI::dbReadTable(con, "packages") diff --git a/local/migrate-s3-hetzner-to-backblaze.sh b/local/migrate-s3-hetzner-to-backblaze.sh new file mode 100644 index 0000000..8cbd829 --- /dev/null +++ b/local/migrate-s3-hetzner-to-backblaze.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Migrate S3 buckets from Hetzner Object Storage to Backblaze B2 via rclone. +# +# Prerequisites: +# 1. Install rclone: https://rclone.org/install/ +# 2. Configure two rclone remotes: +# rclone config create hetzner s3 \ +# provider=Other \ +# env_auth=false \ +# access_key_id=YOUR_HETZNER_KEY \ +# secret_access_key=YOUR_HETZNER_SECRET \ +# endpoint=fsn1.your-objectstorage.com # adjust region +# +# rclone config create backblaze s3 \ +# provider=Other \ +# env_auth=false \ +# access_key_id=YOUR_B2_KEY \ +# secret_access_key=YOUR_B2_APP_KEY \ +# endpoint=s3.us-west-004.backblazeb2.com # adjust region +# +# Usage: +# ./migrate-s3-hetzner-to-backblaze.sh [src:dst] ... +# ./migrate-s3-hetzner-to-backblaze.sh hetzner-bucket:backblaze-bucket + +HETZNER_REMOTE="${HETZNER_REMOTE:-hetzner}" +BACKBLAZE_REMOTE="${BACKBLAZE_REMOTE:-backblaze}" +RCLONE_FLAGS="${RCLONE_FLAGS:---transfers=64 --checkers=64 --fast-list}" + +if [[ $# -eq 0 ]]; then + echo "Usage: $0 [src-bucket:dst-bucket...]" + echo "" + echo " Each argument is a source:destination bucket pair separated by a colon." + echo "" + echo "Environment variables:" + echo " HETZNER_REMOTE rclone remote name for Hetzner (default: hetzner)" + echo " BACKBLAZE_REMOTE rclone remote name for Backblaze (default: backblaze)" + echo " RCLONE_FLAGS extra rclone flags (default: --transfers=16 --checkers=16 --fast-list)" + echo " DRY_RUN=1 show what would be copied without copying" + exit 1 +fi + +for cmd in rclone; do + if ! command -v "$cmd" &>/dev/null; then + echo "Error: $cmd is not installed." >&2 + exit 1 + fi +done + +# Verify remotes exist +for remote in "$HETZNER_REMOTE" "$BACKBLAZE_REMOTE"; do + if ! rclone listremotes | grep -q "^${remote}:$"; then + echo "Error: rclone remote '${remote}' not found. Run 'rclone config' to set it up." >&2 + exit 1 + fi +done + +DRY_RUN_FLAG="" +if [[ "${DRY_RUN:-0}" == "1" ]]; then + DRY_RUN_FLAG="--dry-run" + echo "=== DRY RUN MODE ===" +fi + +for pair in "$@"; do + if [[ "$pair" != *:* ]]; then + echo "Error: '$pair' is not a valid src:dst pair. Use format 'hetzner-bucket:backblaze-bucket'." >&2 + exit 1 + fi + + src_bucket="${pair%%:*}" + dst_bucket="${pair#*:}" + src="${HETZNER_REMOTE}:${src_bucket}" + dst="${BACKBLAZE_REMOTE}:${dst_bucket}" + + echo "" + echo "--- Migrating: ${src} -> ${dst} ---" + + # shellcheck disable=SC2086 + rclone sync \ + ${RCLONE_FLAGS} \ + ${DRY_RUN_FLAG} \ + --progress \ + "$src" "$dst" + + echo "--- Done: ${src_bucket} -> ${dst_bucket} ---" +done + +echo "" +echo "Migration complete." diff --git a/local/missing-cran-packages-db.R b/local/missing-cran-packages-db.R index 8a81c7b..882bca4 100644 --- a/local/missing-cran-packages-db.R +++ b/local/missing-cran-packages-db.R @@ -1,26 +1,38 @@ ### Scope: Lists packages which are missing any DB entry compared to what is available on CRAN -library(bincraftR) +library(bincraft) library(dplyr) library(DBI) -con <- DBI::dbConnect(RPostgres::Postgres(), - dbname = "build_metadata", host = "r-binaries.devxy.io", - port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "r_binaries", + password = Sys.getenv("PGPASS"), sslmode = "require" ) `%nin%` <- Negate(`%in%`) -new_packages <- get_new_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name -removed_pkgs <- get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name +new_packages <- get_new_cran_packages(lubridate::interval( + lubridate::today(), + lubridate::today() - 2 +))$name +removed_pkgs <- get_removed_cran_packages(lubridate::interval( + lubridate::today(), + lubridate::today() - 2 +))$name # filter windows packages # filter new packages from the last X days -cran_pkgs <- unique(tools::CRAN_package_db() |> - filter(`OS_type` != "windows" | is.na(`OS_type`)) |> - filter(`Package` %nin% new_packages) |> - pull(Package)) +cran_pkgs <- unique( + tools::CRAN_package_db() |> + filter(`OS_type` != "windows" | is.na(`OS_type`)) |> + filter(`Package` %nin% new_packages) |> + pull(Package) +) -arch <- "amd64" +arch <- "arm64" platform <- "redhat-8" platform <- "redhat-9" platform <- "alpine-320" @@ -28,13 +40,24 @@ platform <- "alpine-321" platform <- "ubuntu-2204" platform <- "ubuntu-2404" -data <- dbGetQuery(con, "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", params = list(platform, arch)) -removed_pkgs = get_removed_cran_packages(lubridate::interval(lubridate::today(), lubridate::today() - 2))$name +data <- dbGetQuery( + con, + "SELECT name FROM single_builds WHERE platform = $1 AND arch = $2 and removed = FALSE;", + params = list(platform, arch) +) +removed_pkgs = get_removed_cran_packages(lubridate::interval( + lubridate::today(), + lubridate::today() - 2 +))$name pkgs_db <- unique(data$name) pkgs_db = setdiff(pkgs_db, removed_pkgs) pkgs <- setdiff(cran_pkgs, pkgs_db) -formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)]))))) +formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)])))) +) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") cat(formatted_pkgs_one_line, "\n", sep = "") @@ -42,4 +65,13 @@ length(pkgs) clipr::write_clip(formatted_pkgs_one_line) -# sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs))) \ No newline at end of file +# sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs))) + +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), + aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"), + endpoint = "https://hel1.your-objectstorage.com", + region_name = "hel1", +) + +s3fs::s3_dir_delete("devxy-r-package-binaries-docs/_site") diff --git a/local/missing-cran-packages.R b/local/missing-cran-packages.R index 11d0674..ca5cad8 100644 --- a/local/missing-cran-packages.R +++ b/local/missing-cran-packages.R @@ -1,5 +1,5 @@ ### Scope: Lists packages which are missing any tarball in S3 compared to the CRAN pkg listing -library(bincraftR) +library(bincraft) library(dplyr) s3fs::s3_file_system( aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), @@ -10,30 +10,53 @@ s3fs::s3_file_system( ) s3fs::s3_dir_ls("s3://devxy-r-package-binaries") -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib" +)) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) # done -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib" +)) # done +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib" +)) -files_b=basename(files) +files_b = basename(files) files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) -cran_pkgs <- tools::CRAN_package_db() |> - filter(`Date/Publication` <= "2025-01-25") |> +cran_pkgs <- tools::CRAN_package_db() |> + filter(`Date/Publication` <= "2025-01-25") |> pull(Package) pkgs = setdiff(cran_pkgs, files_b_split) -formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900]))))) +formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs[1:900])))) +) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") cat(formatted_pkgs_one_line, "\n", sep = "") clipr::write_clip(formatted_pkgs_one_line) - diff --git a/local/missing-packages-in-index.R b/local/missing-packages-in-index.R index 85700cd..6453888 100644 --- a/local/missing-packages-in-index.R +++ b/local/missing-packages-in-index.R @@ -1,4 +1,4 @@ -library(bincraftR) +library(bincraft) library(dplyr) s3fs::s3_file_system( aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), @@ -7,45 +7,109 @@ s3fs::s3_file_system( region_name = "hel1", ) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/jammy/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib" +)) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib")) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib" +)) -files_b=basename(files) +files_b = basename(files) files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1]) -files_b_split = setdiff(files_b_split, c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive")) +files_b_split = setdiff( + files_b_split, + c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive") +) # windows-only -files_b_split = setdiff(files_b_split, c("PACKAGES", "PACKAGES.gz", "PACKAGES.rds", "PACKAGES.db", "Archive", "RInno", "KeyboardSimulator", "R2PPT", "RWinEdt", "blatr", "excel.link", "spectrino", "taskscheduleR", "MDSGUI", "BiplotGUI", "R2wd")) -pkgs_index = available.packages("https://cran.devxy.io/amd64/jammy/latest/src/contrib")[, "Package"] -pkgs_index_b=basename(pkgs_index) +files_b_split = setdiff( + files_b_split, + c( + "PACKAGES", + "PACKAGES.gz", + "PACKAGES.rds", + "PACKAGES.db", + "Archive", + "RInno", + "KeyboardSimulator", + "R2PPT", + "RWinEdt", + "blatr", + "excel.link", + "spectrino", + "taskscheduleR", + "MDSGUI", + "BiplotGUI", + "R2wd" + ) +) +pkgs_index = available.packages( + "https://cran.devxy.io/amd64/jammy/latest/src/contrib" +)[, "Package"] +pkgs_index_b = basename(pkgs_index) if (length(files_b_split) != length(pkgs_index_b)) { pkgs_missing = setdiff(files_b_split, pkgs_index_b) message("Packages missing in index but present in S3:") pkgs_missing - formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs_missing[1:900]))))) + formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs_missing[1:900])))) + ) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") - + cat(formatted_pkgs_one_line, "\n", sep = "") } -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib")) - -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib")) -files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib")) - - +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/amd64/alpine320/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/jammy/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/noble/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel8/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/rhel9/latest/src/contrib" +)) +files <- s3fs::s3_dir_ls(sprintf( + "devxy-r-package-binaries-hel1/arm64/alpine320/latest/src/contrib" +)) diff --git a/local/packages-to-build.R b/local/packages-to-build.R new file mode 100644 index 0000000..3b508fb --- /dev/null +++ b/local/packages-to-build.R @@ -0,0 +1,259 @@ +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) +suppressPackageStartupMessages(library(dplyr)) +library(DBI, quietly = TRUE) +library(purrr, quietly = TRUE) +library(progressr, quietly = TRUE) +suppressPackageStartupMessages(library(data.table)) + +# Sys.setenv("OS" = "alpine") +# Sys.setenv("OS_VERSION" = "3.22") +# Sys.setenv("ARCH" = "arm64") + +arch <- Sys.getenv("ARCH") +# target: alpine-322, ubuntu-2404, redhat-9, etc. +platform <- paste( + Sys.getenv("OS"), + gsub("[.]", "", Sys.getenv("OS_VERSION")), + sep = "-" +) +# Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10") +codename <- bincraft::set_codename(NULL) + +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) + +cran_archive <- tools::CRAN_archive_db() +cran_release <- tools::CRAN_package_db() +# Subset cran_archive to only those packages +cran_archive_in_release <- cran_archive[ + names(cran_archive) %in% cran_release$Package +] + +archive_versions <- rbindlist( + lapply(names(cran_archive), function(pkg) { + df <- as.data.table(cran_archive_in_release[[pkg]]) + file_names <- rownames(cran_archive_in_release[[pkg]]) # Get row names from the original data.frame! + versions <- sub(".*_(.*)\\.tar\\.gz$", "\\1", file_names) + data.table( + Package = pkg, + Version = versions, + mtime = df$mtime + ) + }), + fill = TRUE +) + +# Select the 4 most recent archive versions by mtime for each package +# Combined with the 1 release version = 5 versions per package +archive_versions <- archive_versions[ + order(Package, -as.numeric(mtime)) +][, + head(.SD, 4), + by = Package +][, + .(Package, Version) +] + +# Now get release versions (assuming cran_release has Package and Version columns) +release_versions <- data.table( + Package = cran_release$Package, + Version = as.character(cran_release$Version) +) + +pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE)) +setorder(pkgs_to_build, Package, Version) + +### Get all packages in S3 +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) +s3_pkgs <- s3fs::s3_dir_ls( + sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename), + recurse = TRUE +) + +file_names <- basename(s3_pkgs) + +# An object occupying a key is not proof a binary was built: a package whose +# build failed has its CRAN source published under exactly that name. Left in +# the cache, `build_binary_package()` reads it as "already built" and skips the +# package forever, which is how alpine324 accumulated ~13.5k source tarballs. +# +# bincraft stamps `Built` only on records it actually built, so the slot's own +# index distinguishes them. A slot last indexed by a bincraft that predates that +# fix stamps `Built` on everything, so the cache is then unchanged from before. +# Archived objects have no index record and are kept: unknown means binary, +# never "rebuild it". +index_url <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", + arch, + codename +) +source_served <- tryCatch( + { + con_idx <- gzcon(url(index_url, open = "rb")) + on.exit(close(con_idx), add = TRUE) + idx <- read.dcf(con_idx, fields = c("Package", "Version", "Built")) + sprintf( + "%s_%s.tar.gz", + idx[is.na(idx[, "Built"]), "Package"], + idx[is.na(idx[, "Built"]), "Version"] + ) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read %s (%s); keeping the full S3 cache\n", + index_url, + conditionMessage(e) + )) + character(0) + } +) + +binary_cache <- setdiff(file_names, source_served) +cat(sprintf( + "S3 cache: %d objects, %d served as CRAN source, %d usable binaries\n", + length(file_names), + length(file_names) - length(binary_cache), + length(binary_cache) +)) + +# Save the S3 file listing for the build step to use as s3_package_cache. +# This avoids loading s3fs/reticulate in the build container, saving memory for +# the dependency-installer subprocesses +saveRDS(binary_cache, "/mnt/cache/packages/s3_cache.rds") +# Built from the filtered listing, not the raw one: `s3_dt` is subtracted from +# the build list below, so a source fallback left in here would exclude the very +# package that needs building. +matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", binary_cache) +parts <- regmatches(binary_cache, matches) +parts <- parts[sapply(parts, length) == 3] +s3_dt <- data.table( + Package = sapply(parts, `[`, 2), + Version = sapply(parts, `[`, 3) +) + +### Get all packages with build errors + +sql_query <- paste0( + # nolint + "SELECT error_occurred FROM ", + "single_builds", + " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4" +) +# Function to query for a single package-version +query_error <- function(pkg, ver) { + purrr::insistently( + ~ DBI::dbGetQuery( + con, + sql_query, + params = list(pkg, ver, platform, arch) + ), + rate = purrr::rate_backoff( + pause_base = 1L, + pause_cap = 60L, + pause_min = 1L, + max_times = 10L, + jitter = FALSE + ), + quiet = FALSE + )() +} + +# Fetch all relevant columns from the database +errored_pkgs <- DBI::dbGetQuery( + con, + sprintf( + "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", + platform, + arch + ) +) +errored_pkgs <- as.data.table(errored_pkgs) +setkey(pkgs_to_build, Package, Version) +setnames(errored_pkgs, c("Package", "Version")) +setkey(errored_pkgs, Package, Version) + +### Final subsetting +pkgs_no_error <- pkgs_to_build[!errored_pkgs] +# Return the full (Package, Version) pairs that need building +pkgs <- pkgs_no_error[!s3_dt] +# Deduplicate +pkgs <- unique(pkgs) +setorder(pkgs, Package, Version) + +### R-minor sensitivity (classify once per package, applied to all versions) +source(file.path("local", "r-minor-helpers.R")) +risky_deps <- bincraft::abi_risky_linking_deps() + +release_meta <- data.table( + Package = cran_release$Package, + NeedsCompilation = cran_release$NeedsCompilation, + LinkingTo = cran_release$LinkingTo +) + +meta <- release_meta[Package %in% unique(pkgs$Package)] +meta[, + triage := mapply( + classify_from_metadata, + NeedsCompilation, + LinkingTo, + MoreArgs = list(risky_deps = risky_deps) + ) +] + +# Only the "ambiguous" compiled packages need a source grep. +ambiguous <- meta[triage == "ambiguous", Package] +sensitive_ambiguous <- character() +if (length(ambiguous) > 0L) { + tmp_src <- file.path(tempdir(), "abi_src") + dir.create(tmp_src, showWarnings = FALSE, recursive = TRUE) + sens <- vapply( + ambiguous, + function(pkg) { + out <- tryCatch( + { + dl <- utils::download.packages( + pkg, + destdir = tmp_src, + repos = "https://cloud.r-project.org", + quiet = TRUE + ) + isTRUE(as.logical(bincraft::needs_per_minor_recompile(dl[1L, 2L]))) + }, + error = function(e) TRUE + ) # fail safe: treat as sensitive + out + }, + logical(1L) + ) + sensitive_ambiguous <- ambiguous[sens] +} + +sensitive_pkgs <- unique(c( + meta[triage == "sensitive", Package], + sensitive_ambiguous +)) +pkgs[, r_minor_sensitive := Package %in% sensitive_pkgs] +sprintf( + "R-minor-sensitive packages: %s of %s", + length(sensitive_pkgs), + uniqueN(pkgs$Package) +) diff --git a/local/packages-without-any-binary.R b/local/packages-without-any-binary.R index a6994ad..f42196f 100644 --- a/local/packages-without-any-binary.R +++ b/local/packages-without-any-binary.R @@ -1,17 +1,24 @@ ### Lists packages without any successful binaries, i.e. pkgs for which all builds errored (per platform & arch) library(DBI) library(dplyr) -con <- DBI::dbConnect(RPostgres::Postgres(), - dbname = "build_metadata", host = "r-binaries.devxy.io", - port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"), +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "r_binaries", + password = Sys.getenv("PGPASS"), sslmode = "require" ) -cran_pkgs <- tools::CRAN_package_db() |> - filter(`Date/Publication` <= "2024-12-02") |> +cran_pkgs <- tools::CRAN_package_db() |> + filter(`Date/Publication` <= "2024-12-02") |> pull(Package) -data <- dbGetQuery(con, "SELECT name,platform,arch,removed,error_occurred FROM single_builds;") +data <- dbGetQuery( + con, + "SELECT name,platform,arch,removed,error_occurred FROM single_builds;" +) pkgs <- data |> filter(platform == "alpine-320", arch == "arm64") |> @@ -22,7 +29,11 @@ pkgs <- data |> pkgs = setdiff(cran_pkgs, pkgs) # Format, remove line breaks, and print as a single line -formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900]))))) +formatted_pkgs <- gsub( + '"', + "'", + capture.output(dput(as.character(na.omit(pkgs[1:900])))) +) formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ") diff --git a/local/patches/README.md b/local/patches/README.md new file mode 100644 index 0000000..ce60c9a --- /dev/null +++ b/local/patches/README.md @@ -0,0 +1,120 @@ +# Patch Registry + +This directory contains the curated registry of per-package build-time patches consumed by bincraft's `patches` argument. + +## Schema + +The registry is defined in `registry.json` as an array of patch entries. Each entry specifies lightweight build-time overrides (environment variables, configure arguments, Makevars) and optionally a source diff to apply before building. + +### Field semantics + +| Field | Type | Required | Description | +| ---------------- | ---------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `package` | string | yes | CRAN package name. | +| `versions` | string | yes | `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. | +| `platforms` | array of strings | yes | Matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). An entry matches if any listed token matches any build token. `["*"]` matches all platforms. | +| `env` | object | no | Environment variables exported only for this package's isolated build. | +| `configure_args` | array | no | Arguments passed as `--configure-args` to the isolated build. | +| `makevars` | object | no | Key/value pairs written into a package-local Makevars for the isolated build. | +| `patch` | string or null | no | Path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. | +| `reason` | string | yes | Human explanation, surfaced in logs and metadata. | + +## Adding an entry + +To add a new patch entry: + +1. Add an object to the array in `registry.json` with the fields documented above. + Start with lightweight overrides (environment variables, configure arguments, Makevars) before resorting to source diffs. + +2. If a source diff is needed, place it in `local/patches//.patch` and reference its path in the `patch` field. + For example, a diff for `RcppParallel` would go in `local/patches/RcppParallel/fix.patch` and be referenced as `"patch": "RcppParallel/fix.patch"`. + +3. The `reason` field should clearly explain why the patch is needed and what problem it solves. + +## Validation + +The registry is validated and applied by bincraft during the build process. +For manual validation, run the validator from the repo root: + +```bash +Rscript local/validate-patches.R +``` + +This validates the schema, referenced patch-file existence, and checks for duplicate entries across platforms and versions. + +## Triaging failures into entries + +`local/failing-builds-report.R` turns recorded build failures into triaged patch suggestions instead of hand-scraping Crow logs (issue #115, steps 1 + 2). +It is read-only: it queries `single_builds WHERE error_occurred`, groups failures by a normalised error fingerprint, classifies each group against the known signature set in `local/failing-builds-classify.R`, and prints a report. + +```bash +# All platforms/arches; needs the DB password. +PGPASS=... Rscript local/failing-builds-report.R +# Restrict scope and also emit a machine-readable report. +PGPASS=... Rscript local/failing-builds-report.R --platform alpine-321 --arch amd64 --json report.json +``` + +Each group is tagged `AUTO-PROPOSABLE` (a known env/makevars lever, or an already-curated package patch, safe to pre-fill as a `registry.json` entry) or `HUMAN TRIAGE` (unknown signature, or a fix that needs a novel source diff). +For auto-proposable groups it prints a ready-to-review registry entry; still run `validate-patches.R` and an isolated trial build before merging. +Novel source diffs and unknown signatures stay human-reviewed by design. + +Add a new signature by appending a rule to `build_signatures()` in `local/failing-builds-classify.R`; the pure helpers are covered by `local/tests/test-failing-builds-classify.R`. + +### Proposing entries (step 3: propose, do not apply) + +`local/propose-patches.R` takes the auto-proposable candidates one step further: for each classified, safe fix affecting a package with no current entry, it emits a pre-filled `registry.json` entry and validates the candidate set against a temporary merged registry (the real registry is never touched unless you ask). +The human gate stays: it never merges. + +```bash +# Default: print candidates + validation, take no action. +PGPASS=... Rscript local/propose-patches.R +# Append the candidates to registry.json + the proposals ledger (you commit + open the PR). +PGPASS=... Rscript local/propose-patches.R --write +# Or post/update a Forgejo tracking issue instead (needs FORGEJO_TOKEN). +PGPASS=... FORGEJO_TOKEN=... Rscript local/propose-patches.R --open-issue +``` + +The acceptance criteria before merging a proposal are: `validate-patches.R` passes (checked automatically), and an isolated trial build succeeds. +Run the trial build inside the failing platform's build-env image; it uploads/archives nothing and writes no metadata: + +```bash +Rscript local/trial-build-patch.R +``` + +#### Autonomous PR + trial-build gate + +`--open-pr` closes the loop: it writes the top-N candidates (by failure volume) onto the reused `auto/registry-patch-proposals` branch and opens/updates a single PR. +`.crow/auto-apply-patches.yaml` runs this on a cron. +`FORGEJO_TOKEN` is used for both the branch push and the PR (no separate write-scoped secret). + +```bash +# Bounded batch; opens/updates one PR. +PGPASS=... FORGEJO_TOKEN=... Rscript local/propose-patches.R --open-pr --limit 10 +``` + +The merge gate is `.crow/trial-build-registry.yaml`: matrixed over the build-env images, each platform trial-builds only the entries the branch **adds** (`local/trial-build-registry.R`, which diffs the registry against `main`) and is green only if every new entry builds. +The repo uses no `pull_request` triggers, so this gate runs manually or on a cron against the auto-patch branch (`--var patch_branch=...`); wiring it to fire on the PR needs `event: pull_request` enabled on the forge. + +`--write` and `--open-issue` also append to `local/patches/proposals-log.json`, a ledger of what was proposed. + +### Feedback loop (step 4) + +`local/proposal-tracking.R` reports the signature hit rate, proposed-vs-merged status (a proposal counts as merged once its package appears in the registry), and retirement candidates (registry entries whose package no longer appears in any current failure, so the upstream cause was likely fixed). +It also surfaces the classifier's **blind spots**: the unclassified failures (candidates for a new signature) and the groups blocked on a dependency build, so the unknown buckets get the same visibility as the proposals. +It is read-only on the DB, with an optional Forgejo issue as the only write. + +```bash +# Print the metrics + blind spots. +PGPASS=... Rscript local/proposal-tracking.R --json metrics.json +# Post/update a "needs signatures" tracking issue with the unclassified failures. +PGPASS=... FORGEJO_TOKEN=... Rscript local/proposal-tracking.R --open-issue +``` + +The unclassified groups are the natural place to discover which new signatures are worth adding to `build_signatures()`. +The pure metric/ledger helpers live in `local/proposal-tracking-lib.R` and are covered by `local/tests/test-proposal-tracking-lib.R`. + +### Scheduled run + +`.crow/weekly-patch-proposals.yaml` runs both steps weekly (register the `weekly-patch-proposals` cron in the crow UI). +It posts/updates two Forgejo issues -- one with the auto-proposable entries, one with the unclassified/blocked failures -- and logs the feedback-loop metrics. +It clones read-only; the only writes are the two tracking issues. diff --git a/local/patches/RcppParallel/force-bundled-tbb.patch b/local/patches/RcppParallel/force-bundled-tbb.patch new file mode 100644 index 0000000..c46a05e --- /dev/null +++ b/local/patches/RcppParallel/force-bundled-tbb.patch @@ -0,0 +1,154 @@ +diff --git a/R/aaa.R b/R/aaa.R +index 568a2aa..bfdab4c 100644 +--- a/R/aaa.R ++++ b/R/aaa.R +@@ -5,4 +5,21 @@ TBB_LIB <- "" + TBB_INC <- "" + + TBB_NAME <- "tbb" +-TBB_MALLOC_NAME <- "tbbmalloc" +\ No newline at end of file ++TBB_MALLOC_NAME <- "tbbmalloc" ++ ++# bincraft patch: our build images (and plenty of user environments) export ++# TBB_ROOT / TBB_LIB / TBB_INC. A binary we publish always carries its own ++# oneTBB in RcppParallel/lib -- see the companion changes in ++# tools/config/configure.R and src/install.libs.R -- so honouring those ++# variables at run time is actively harmful: .onLoad() would dyn.load() a ++# second, unrelated TBB into the process next to the bundled one (two copies ++# of the same symbols in the global scope, which segfaults R on load), and ++# RcppParallelLibs() / CxxFlags() would hand that system TBB to dependents ++# such as rstan, putting NEEDED libtbb.so.12 back into their binaries. Read ++# the variables only when explicitly opted back in. ++bincraftGetenv <- function(name, unset = "") { ++ if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") == "TRUE") ++ Sys.getenv(name, unset = unset) ++ else ++ unset ++} +\ No newline at end of file +diff --git a/R/tbb.R b/R/tbb.R +index 6f6a745..e407986 100644 +--- a/R/tbb.R ++++ b/R/tbb.R +@@ -17,7 +17,7 @@ tbbLibraryPath <- function(name = NULL) { + sysname <- Sys.info()[["sysname"]] + + # find root for TBB install +- tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) ++ tbbRoot <- bincraftGetenv("TBB_LIB", unset = tbbRoot()) + if (is.null(name)) + return(tbbRoot) + +@@ -58,7 +58,7 @@ tbbCxxFlags <- function() { + flags <- c("-DRCPP_PARALLEL_USE_TBB=1") + + # if TBB_INC is set, apply those library paths +- tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) ++ tbbInc <- bincraftGetenv("TBB_INC", unset = TBB_INC) + if (!file.exists(tbbInc)) { + tbbInc <- system.file("include", package = "RcppParallel") + } +@@ -117,7 +117,7 @@ tbbLdFlags <- function() { + } + + # shortcut if TBB_LIB defined +- tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) ++ tbbLib <- bincraftGetenv("TBB_LINK_LIB", bincraftGetenv("TBB_LIB", unset = TBB_LIB)) + if (nzchar(tbbLib)) { + if (R.version$os == "emscripten") { + fmt <- "-L%1$s -l%2$s" +diff --git a/src/install.libs.R b/src/install.libs.R +index 3b3cfda..c0e6f3e 100644 +--- a/src/install.libs.R ++++ b/src/install.libs.R +@@ -477,6 +477,18 @@ prependFlags <- function(prependFlags, toFlags) { + tbbLib <- Sys.getenv("TBB_LIB") + tbbInc <- Sys.getenv("TBB_INC") + ++# bincraft patch: the companion change in tools/config/configure.R stops an ++# ambient TBB_LIB / TBB_INC from selecting a system TBB, but this script is ++# also run directly by `R CMD INSTALL` (not only through the `tbb` rule in ++# src/Makevars, which passes the configured values), so at install time it ++# still sees the image's environment and would symlink the system libraries ++# into RcppParallel/lib. Drop them here for the same reason, under the same ++# opt-out. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbLib <- "" ++ tbbInc <- "" ++} ++ + args <- commandArgs(trailingOnly = TRUE) + if (identical(args, "build")) { + if (nzchar(tbbLib) && nzchar(tbbInc)) { +diff --git a/tools/config/configure.R b/tools/config/configure.R +index 6293fe1..eae4aaa 100644 +--- a/tools/config/configure.R ++++ b/tools/config/configure.R +@@ -40,6 +40,24 @@ tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) + tbbLib <- Sys.getenv("TBB_LIB", unset = NA) + tbbInc <- Sys.getenv("TBB_INC", unset = NA) + ++# bincraft patch: ignore an ambient TBB_ROOT / TBB_LIB / TBB_INC. Several of ++# our build images export these (a leftover from RcppParallel 5.x, whose ++# bundled Intel TBB would not build on musl or with modern g++), and any of ++# them switches the branches below to a system TBB. The published binary then ++# records NEEDED libtbb.so.12 (or libtbb.so.2 for the classic Intel TBB) and ++# gets a RcppParallel/lib full of absolute symlinks into the image's library ++# dir, so it cannot dyn.load on a consumer machine without that exact TBB. ++# 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, ++# so the bundled copy is always the right choice here; forcing it in the ++# package rather than relying on the image environment keeps the binary ++# correct whichever image version CI happens to pull. Set ++# BINCRAFT_ALLOW_SYSTEM_TBB=TRUE to restore the upstream behaviour. ++if (Sys.getenv("BINCRAFT_ALLOW_SYSTEM_TBB", unset = "FALSE") != "TRUE") { ++ tbbRoot <- NA ++ tbbLib <- NA ++ tbbInc <- NA ++} ++ + tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") + tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") + +@@ -186,12 +204,23 @@ define( + ) + + # set PKG_LIBS ++# ++# bincraft patch: the library directories below are passed as plain '-L', not ++# '-Wl,-L'. gcc expands its own search dirs (/usr/lib64, /usr/lib/) ++# into explicit '-L' options ahead of anything forwarded verbatim with '-Wl,', ++# so with '-Wl,-L' a system libtbb.so wins over the one named here: on a build ++# host with a distro TBB installed, '-ltbb' resolves to that library and ++# RcppParallel.so records its SONAME (libtbb.so.12, or libtbb.so.2 for the ++# classic Intel TBB) instead of the bundled 'libtbb.so'. The binary then loads ++# the system TBB rather than the copy shipped in RcppParallel/lib, and fails ++# outright on a machine that has no system TBB. gcc places a plain '-L' before ++# its built-in dirs, so the intended library is found first. + pkgLibs <- if (!is.na(tbbLib)) { + + # a TBB supplied via TBB_LIB / TBB_ROOT. an rpath is meaningless on Windows, + # where the loader has no equivalent -- see R/zzz.R for how we resolve there + c( +- "-Wl,-L\"$(TBB_LIB)\"", ++ "-L\"$(TBB_LIB)\"", + if (.Platform$OS.type != "windows") + sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), + "-l$(TBB_NAME)", +@@ -201,14 +230,14 @@ pkgLibs <- if (!is.na(tbbLib)) { + } else if (R.version$os == "emscripten") { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)" + ) + + } else { + + c( +- "-Wl,-Ltbb/build/lib_release", ++ "-Ltbb/build/lib_release", + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" + ) diff --git a/local/patches/fs/force-vendored-libuv.patch b/local/patches/fs/force-vendored-libuv.patch new file mode 100644 index 0000000..c69e8b9 --- /dev/null +++ b/local/patches/fs/force-vendored-libuv.patch @@ -0,0 +1,19 @@ +diff --git a/configure b/configure +--- a/configure ++++ b/configure +@@ -11,6 +11,15 @@ + PKG_TEST_HEADER="" + PKG_LIBS="-luv" + ++# bincraft patch: force the vendored static libuv so the resulting binary ++# is self-contained. fs configure otherwise links system libuv whenever ++# pkg-config finds libuv-devel (installed as a build-time sysreq), yielding ++# an fs.so with NEEDED libuv.so.1 that fails to dyn.load on machines lacking ++# runtime libuv (install.packages/renv do not install SystemRequirements). ++echo "Building static libuv (bincraft: forced vendored)" 1>&2 ++cp -f src/Makevars.vendor src/Makevars ++exit 0 ++ + # Use pkg-config if available + if [ `command -v pkg-config` ]; then + PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` diff --git a/local/patches/registry.json b/local/patches/registry.json new file mode 100644 index 0000000..a7e29dd --- /dev/null +++ b/local/patches/registry.json @@ -0,0 +1,22 @@ +[ + { + "package": "RcppParallel", + "versions": ">=6.0.0", + "platforms": ["*"], + "env": {}, + "configure_args": [], + "makevars": {}, + "patch": "RcppParallel/force-bundled-tbb.patch", + "reason": "RcppParallel 6.x bundles oneTBB 2022 and builds it with cmake on every platform we ship, so the system-TBB workaround needed for 5.x is gone, but two things still steer the build back to a system TBB. (1) Ambient TBB_ROOT/TBB_LIB/TBB_INC: several build images still export these (build-env-images dropped them, but the images are rebuilt only by cron/manual runs, so a stale image keeps them), and R reads ~/.Renviron *after* the process environment, so no pipeline-side env override can undo it. configure.R then takes the system-TBB branch, install.libs.R symlinks the image's libraries into RcppParallel/lib as absolute paths, and the binary records NEEDED libtbb.so.12 (libtbb.so.2 for the classic Intel TBB on el8/el9) -- it cannot dyn.load on a consumer machine without that exact TBB. Both files are patched to ignore those variables (opt out with BINCRAFT_ALLOW_SYSTEM_TBB=TRUE); install.libs.R needs it separately because R CMD INSTALL runs it outside the src/Makevars rule that passes the configured values. (2) Link order: the bundled branch passes its build dir as '-Wl,-Ltbb/build/lib_release', and gcc expands its own search dirs (/usr/lib64, /usr/lib/) into '-L' options ahead of anything forwarded with '-Wl,', so '-ltbb' would still resolve to a distro TBB when one is installed; a plain '-L' puts the bundled dir first. Verified on the current build-env-ubuntu:jammy image (which still exports TBB_INC/TBB_LIB): NEEDED libtbb.so, RUNPATH $ORIGIN/../lib, real libtbb.so.2 in RcppParallel/lib, and the package loads with every system libtbb moved away." + }, + { + "package": "fs", + "versions": "*", + "platforms": ["*"], + "env": {}, + "configure_args": [], + "makevars": {}, + "patch": "fs/force-vendored-libuv.patch", + "reason": "fs 2.x configure links system libuv whenever pkg-config finds libuv-devel (installed as a build-time sysreq), producing an fs.so with NEEDED libuv.so.1. That binary fails to dyn.load on consumer machines lacking runtime libuv, because install.packages/renv do not install SystemRequirements (only pak does, and only in the build container). The patch short-circuits configure to copy src/Makevars.vendor and build the bundled static libuv (needs cmake) so the binary is self-contained on every platform. An env/pkg-config override was tried first but the rebuilt binary still linked libuv.so.1, so a source patch is used instead." + } +] diff --git a/local/proposal-tracking-lib.R b/local/proposal-tracking-lib.R new file mode 100644 index 0000000..28f65db --- /dev/null +++ b/local/proposal-tracking-lib.R @@ -0,0 +1,230 @@ +# Pure, IO-free helpers for the proposal feedback loop (issue #115, step 4): +# a small ledger of what the classifier proposed, plus metrics derived from the +# live triage report and the registry (signature hit rate, proposed-vs-merged, +# retirement candidates). +# +# Kept free of DB/HTTP/clock so it can be sourced by the proposer, the tracker +# entrypoint, and the unit tests. Timestamps are passed in by callers. + +# Split proposal candidates into the ones safe to emit and the ambiguous ones. +# A candidate is a list with at least `package` and `signature`. A package that +# maps to more than one distinct auto-proposable signature is genuinely +# ambiguous (two conflicting fix tiers, e.g. makevars vs source patch): emitting +# both would create colliding registry entries, so those are routed to human +# triage instead of guessed at. Returns list(keep = ..., ambiguous = ...), where +# `ambiguous` is a named list of package -> the distinct signatures seen. +dedupe_candidates <- function(candidates) { + if (length(candidates) == 0L) { + return(list(keep = list(), ambiguous = list())) + } + pkgs <- vapply(candidates, function(c) as.character(c$package), character(1L)) + by_pkg <- split(candidates, pkgs) + keep <- list() + ambiguous <- list() + for (pkg in names(by_pkg)) { + cs <- by_pkg[[pkg]] + sigs <- unique(vapply( + cs, + function(c) as.character(c$signature), + character(1L) + )) + if (length(sigs) == 1L) { + keep[[length(keep) + 1L]] <- cs[[1L]] # one signature -> take the first + } else { + ambiguous[[pkg]] <- sigs + } + } + list(keep = keep, ambiguous = ambiguous) +} + +# Stable identity of a ledger record: one proposal per (package, signature). +ledger_key <- function(record) { + paste0( + if (is.null(record$package)) "?" else record$package, + "|", + if (is.null(record$signature)) "?" else record$signature + ) +} + +# Merge freshly-generated proposals into an existing ledger without clobbering +# history: a record whose (package, signature) already exists is left as-is +# (its status/PR/issue are preserved); genuinely new proposals are appended. +# Returns the combined list. Pure: callers stamp `proposed_at` before passing. +merge_ledger <- function(existing, new_records) { + if (is.null(existing)) { + existing <- list() + } + seen <- vapply(existing, ledger_key, character(1L)) + out <- existing + for (rec in new_records) { + if (!(ledger_key(rec) %in% seen)) { + out[[length(out) + 1L]] <- rec + seen <- c(seen, ledger_key(rec)) + } + } + out +} + +# Per-signature hit rate from a triage report (list of group records from +# build_triage_report) crossed with the set of already-registered packages. +# For each *matched* signature: how many failing builds/packages it explains, +# and how many of those packages are already addressed by a registry entry. +signature_hit_rate <- function(report, registered_pkgs = character(0L)) { + matched <- Filter(function(g) isTRUE(g$matched), report) + by_sig <- split( + matched, + vapply(matched, function(g) g$signature, character(1L)) + ) + lapply(names(by_sig), function(sig) { + grps <- by_sig[[sig]] + pkgs <- unique(unlist(lapply(grps, function(g) g$packages))) + addressed <- intersect(pkgs, registered_pkgs) + list( + signature = sig, + builds = sum(vapply(grps, function(g) g$build_count, integer(1L))), + packages = length(pkgs), + addressed = length(addressed), + open = length(setdiff(pkgs, registered_pkgs)), + auto_proposable = any(vapply( + grps, + function(g) isTRUE(g$auto_proposable), + logical(1L) + )) + ) + }) +} + +# Proposed-vs-merged: a ledger record counts as "merged" once its package +# appears in the registry. Returns per-record status plus a rollup. +proposed_vs_merged <- function(ledger, registered_pkgs = character(0L)) { + if (is.null(ledger)) { + ledger <- list() + } + rows <- lapply(ledger, function(rec) { + merged <- !is.null(rec$package) && rec$package %in% registered_pkgs + list( + package = rec$package, + signature = rec$signature, + status = if (merged) "merged" else (rec$status %||% "proposed"), + merged = merged + ) + }) + list( + records = rows, + total = length(rows), + merged = sum(vapply(rows, function(r) isTRUE(r$merged), logical(1L))) + ) +} + +# Registry entries whose package no longer appears in any current failure are +# retirement candidates: the upstream cause was likely fixed, so the entry can +# be reviewed for removal. `failing_pkgs` is the set of currently-failing +# package names (from the live report). +retirement_candidates <- function(registry_entries, failing_pkgs) { + if (is.null(registry_entries)) { + registry_entries <- list() + } + keep <- Filter( + function(e) !is.null(e$package) && !(e$package %in% failing_pkgs), + registry_entries + ) + vapply(keep, function(e) as.character(e$package), character(1L)) +} + +# Discovery view: the failure groups the classifier could NOT auto-propose, so +# they get the same visibility as proposals. `report` is a build_triage_report +# result. Returns the unclassified groups (unknown signature -> candidates for a +# new signature) ranked by build count, capped to `max_groups`, each with up to +# `max_pkgs` example packages. `dropped_groups`/`packages_truncated` record any +# cap so nothing is silently hidden. +unclassified_summary <- function(report, max_groups = 30L, max_pkgs = 15L) { + un <- Filter(function(g) !isTRUE(g$matched), report) + un <- un[order(-vapply(un, function(g) g$build_count, integer(1L)))] + shown <- utils::head(un, max_groups) + groups <- lapply(shown, function(g) { + pkgs <- g$packages + list( + fingerprint = g$fingerprint, + build_count = g$build_count, + n_packages = length(pkgs), + packages = utils::head(pkgs, max_pkgs), + packages_truncated = length(pkgs) > max_pkgs, + platforms = g$platforms + ) + }) + list( + groups = groups, + total_groups = length(un), + dropped_groups = max(0L, length(un) - length(shown)), + total_builds = sum(vapply(un, function(g) g$build_count, integer(1L))) + ) +} + +# Aggregate blocked packages across ALL groups by the dependency they wait on, +# so one dependency (RcppParallel, rstan, sf, ...) is a single line -- deduped +# and ranked by how many distinct dependents it blocks -- instead of repeating +# once per fingerprint group. Reads each group's `blocked_map` (package -> the +# dependency it is blocked on). Returns records sorted by dependent count desc, +# each with up to `max_pkgs` example dependents. +blocked_by_dependency <- function(report, max_pkgs = 15L) { + acc <- list() # dependency -> character vector of dependent packages + for (g in report) { + bm <- g$blocked_map + if (is.null(bm) || length(bm) == 0L) { + next + } + for (pkg in names(bm)) { + for (dep in as.character(unlist(bm[[pkg]]))) { + acc[[dep]] <- unique(c(acc[[dep]], pkg)) + } + } + } + if (length(acc) == 0L) { + return(list()) + } + out <- lapply(names(acc), function(dep) { + pkgs <- acc[[dep]] + list( + dependency = dep, + n_packages = length(pkgs), + packages = utils::head(pkgs, max_pkgs), + packages_truncated = length(pkgs) > max_pkgs + ) + }) + out[order(-vapply(out, function(x) x$n_packages, integer(1L)))] +} + +# Does a registry entry's `platforms` apply to a build on `os` (e.g. +# "ubuntu-2604")? Mirrors bincraft's token match: an entry applies if any of its +# platform tokens is "*", the OS codename, or the distro family ("ubuntu"). +entry_applies_to_os <- function(entry_platforms, os) { + toks <- as.character(unlist(entry_platforms)) + family <- sub("-.*$", "", os) # ubuntu-2604 -> ubuntu + any(toks %in% c("*", os, family)) +} + +# Registry entries present in `current` but not in `base` (matched on +# package|platforms|versions), optionally restricted to those that apply to a +# given `os`. Used by the trial-build gate to build only the entries a PR adds. +new_registry_packages <- function(current, base, os = NULL) { + key <- function(e) { + sprintf( + "%s|%s|%s", + e$package %||% "?", + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + e$versions %||% "?" + ) + } + base_keys <- vapply(base %||% list(), key, character(1L)) + added <- Filter(function(e) !(key(e) %in% base_keys), current %||% list()) + if (!is.null(os)) { + added <- Filter(function(e) entry_applies_to_os(e$platforms, os), added) + } + unique(vapply( + added, + function(e) as.character(e$package %||% ""), + character(1L) + )) +} + +`%||%` <- function(a, b) if (is.null(a)) b else a diff --git a/local/proposal-tracking.R b/local/proposal-tracking.R new file mode 100644 index 0000000..f8e1284 --- /dev/null +++ b/local/proposal-tracking.R @@ -0,0 +1,304 @@ +#!/usr/bin/env Rscript + +# Feedback loop for the failure classifier (issue #115, step 4): report the +# signature hit rate, proposed-vs-merged status, registry entries that look +# retirable, and -- so the classifier's blind spots get the same visibility as +# its proposals -- the failures it could NOT classify (candidates for new +# signatures) plus the groups blocked on a dependency build. +# Read-only on the DB/registry; the only optional write is the Forgejo issue. +# +# Usage: +# PGPASS=... Rscript local/proposal-tracking.R [--json PATH] [--open-issue] +# --open-issue post/update a Forgejo issue listing the unclassified and +# dependency-blocked failures (needs FORGEJO_TOKEN) + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(DBI, quietly = TRUE) + library(RPostgres, quietly = TRUE) + library(jsonlite, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "failing-builds-classify.R")) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +args <- commandArgs(trailingOnly = TRUE) +json_out <- local({ + i <- match("--json", args) + if (!is.na(i) && i < length(args)) args[[i + 1L]] else NA_character_ +}) +do_issue <- "--open-issue" %in% args + +if (nchar(Sys.getenv("PGPASS")) == 0L) { + stop("PGPASS env var is not set; a DB password is required.") +} + +registry_file <- file.path(script_dir, "patches", "registry.json") +ledger_file <- file.path(script_dir, "patches", "proposals-log.json") + +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(DBI::dbDisconnect(con), add = TRUE) + +failures <- DBI::dbGetQuery( + con, + paste0( + "SELECT name, tag, platform, arch, r_version, timestamp, error_text ", + "FROM single_builds WHERE error_occurred = TRUE AND removed = FALSE" + ) +) + +existing_entries <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +registered_pkgs <- unique(vapply( + existing_entries, + function(e) as.character(e$package %||% ""), + character(1L) +)) +ledger <- if (file.exists(ledger_file)) { + jsonlite::fromJSON(ledger_file, simplifyVector = FALSE) +} else { + list() +} + +report <- build_triage_report(failures, registered_pkgs) +failing_pkgs <- unique(unlist(lapply(report, function(g) g$packages))) + +hit <- signature_hit_rate(report, registered_pkgs) +pvm <- proposed_vs_merged(ledger, registered_pkgs) +retire <- retirement_candidates(existing_entries, failing_pkgs) + +cat(sprintf( + "Feedback loop over %d failing builds, %d registry entries, %d ledger records.\n", + nrow(failures), + length(existing_entries), + length(ledger) +)) + +cat("\nSignature hit rate:\n") +cat(strrep("-", 60L), "\n", sep = "") +for (h in hit) { + cat(sprintf( + " %-26s %3d builds | %2d pkgs | %2d addressed | %2d open%s\n", + h$signature, + h$builds, + h$packages, + h$addressed, + h$open, + if (isTRUE(h$auto_proposable)) " [auto]" else "" + )) +} + +cat(sprintf("\nProposed vs merged: %d / %d merged.\n", pvm$merged, pvm$total)) +for (r in pvm$records) { + cat(sprintf(" %-24s %-26s %s\n", r$package, r$signature, r$status)) +} + +cat(sprintf( + "\nRetirement candidates (%d): registry entries with no current failure.\n", + length(retire) +)) +if (length(retire) > 0L) { + cat(paste0(" - ", retire, "\n"), sep = "") +} else { + cat(" (none)\n") +} + +# --------------------------------------------------------------------------- +# Blind spots: failures the classifier could not auto-propose. +# --------------------------------------------------------------------------- +blocked <- blocked_by_dependency(report) +unmatched <- unclassified_summary(report) + +cat("\nBlocked on a dependency (fix the dependency, not each dependent):\n") +if (length(blocked) > 0L) { + for (b in blocked) { + cat(sprintf(" %-20s %5d dependent(s)\n", b$dependency, b$n_packages)) + } +} else { + cat(" (none)\n") +} + +cat(sprintf( + "\nUnclassified failures (candidates for new signatures): %d group(s), %d builds.\n", + unmatched$total_groups, + unmatched$total_builds +)) +cat(strrep("-", 60L), "\n", sep = "") +for (g in unmatched$groups) { + cat(sprintf( + " [%d builds | %d pkgs] %s\n e.g. %s%s\n", + g$build_count, + g$n_packages, + g$fingerprint, + toString(g$packages), + if (isTRUE(g$packages_truncated)) ", ..." else "" + )) +} +if (unmatched$dropped_groups > 0L) { + cat(sprintf( + " (+%d more unclassified group(s) not shown)\n", + unmatched$dropped_groups + )) +} + +if (!is.na(json_out)) { + jsonlite::write_json( + list( + signature_hit_rate = hit, + proposed_vs_merged = pvm, + retirement_candidates = retire, + blocked = blocked, + unclassified = unmatched + ), + json_out, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf("\nWrote metrics to %s\n", json_out)) +} + +# --------------------------------------------------------------------------- +# Optionally publish the blind-spots to a Forgejo tracking issue. +# --------------------------------------------------------------------------- +if (do_issue) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-issue requires FORGEJO_TOKEN.") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + issue_title <- "Unclassified build failures (needs signatures) (#115)" + now <- format(Sys.time(), "%Y-%m-%d %H:%M:%S") + + body_lines <- c( + sprintf("_Generated %s from `single_builds` failures._", now), + "", + "Failures the classifier could **not** auto-propose a fix for.", + "Each unclassified group is a candidate for a new signature in `local/failing-builds-classify.R`; the blocked groups clear once the named dependency builds.", + "" + ) + + body_lines <- c(body_lines, "## Blocked on a dependency", "") + if (length(blocked) > 0L) { + for (b in blocked) { + body_lines <- c( + body_lines, + sprintf( + "- **%s**: %d dependent(s) (e.g. %s%s)", + b$dependency, + b$n_packages, + toString(b$packages), + if (isTRUE(b$packages_truncated)) ", ..." else "" + ) + ) + } + } else { + body_lines <- c(body_lines, "_None._") + } + + body_lines <- c( + body_lines, + "", + sprintf( + "## Unclassified failures (%d groups, %d builds)", + unmatched$total_groups, + unmatched$total_builds + ), + "" + ) + if (length(unmatched$groups) > 0L) { + for (g in unmatched$groups) { + body_lines <- c( + body_lines, + sprintf( + "### %d builds / %d pkg(s)", + g$build_count, + g$n_packages + ), + "", + sprintf("Fingerprint: `%s`", g$fingerprint), + sprintf( + "Packages: %s%s", + toString(g$packages), + if (isTRUE(g$packages_truncated)) ", ..." else "" + ), + sprintf("Platforms: %s", toString(g$platforms)), + "" + ) + } + if (unmatched$dropped_groups > 0L) { + body_lines <- c( + body_lines, + sprintf("_(+%d more group(s) not shown.)_", unmatched$dropped_groups) + ) + } + } else { + body_lines <- c(body_lines, "_None -- every failure is classified._") + } + new_body <- paste(body_lines, collapse = "\n") + + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(issue_title, reserved = TRUE) + ) + existing <- httr2::request(search_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + existing, + function(x) identical(x$title, issue_title), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- existing[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("\nUpdated tracking issue #%d.\n", num)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/issues", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = issue_title, body = new_body)) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf("\nOpened tracking issue #%d.\n", created$number)) + } +} diff --git a/local/propose-patches.R b/local/propose-patches.R new file mode 100644 index 0000000..360df9f --- /dev/null +++ b/local/propose-patches.R @@ -0,0 +1,578 @@ +#!/usr/bin/env Rscript + +# Propose registry patches for classified, auto-proposable build failures +# (issue #115, step 3: "propose, do not apply"). Queries `single_builds`, +# classifies failures, and for each *known, safe* fix lever affecting a package +# with no current registry entry, emits a pre-filled `registry.json` entry. +# +# The human gate stays: this never merges. Acceptance before merge is +# 1. `local/validate-patches.R` passes (checked here against a candidate +# registry), and +# 2. an isolated trial build succeeds -- run `local/trial-build-patch.R ` +# (this script prints the exact command per candidate). +# Novel source diffs and unknown signatures are never proposed (they carry +# `auto = FALSE` in the signature table). +# +# Usage: +# PGPASS=... Rscript local/propose-patches.R [--platform P] [--arch A] [--min N] +# [--limit N] +# (default) print candidates + validation, take no action +# --write append candidates to local/patches/registry.json and +# the proposals ledger (commit + open a PR yourself) +# --open-issue post/update a Forgejo tracking issue (needs FORGEJO_TOKEN) +# --open-pr write the entries, push the `auto/registry-patch-proposals` +# branch, and open/update a PR autonomously. Uses +# FORGEJO_TOKEN for both the push and the PR API. The +# `trial-build-registry` pipeline is the merge gate. +# --limit N only act on the top-N candidates by failure volume +# (bounded batch; the rest are picked up on the next run) +# --json PATH also write the machine-readable candidate list to PATH + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(DBI, quietly = TRUE) + library(RPostgres, quietly = TRUE) + library(jsonlite, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "failing-builds-classify.R")) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +# --------------------------------------------------------------------------- +# Arguments +# --------------------------------------------------------------------------- +args <- commandArgs(trailingOnly = TRUE) +opt_val <- function(flag, default = NA_character_) { + i <- match(flag, args) + if (!is.na(i) && i < length(args)) args[[i + 1L]] else default +} +platform <- opt_val("--platform", Sys.getenv("PLATFORM", "")) +arch <- opt_val("--arch", Sys.getenv("ARCH", "")) +json_out <- opt_val("--json") +min_count <- suppressWarnings(as.integer(opt_val("--min", "1"))) +if (is.na(min_count)) { + min_count <- 1L +} +do_write <- "--write" %in% args +do_issue <- "--open-issue" %in% args +do_pr <- "--open-pr" %in% args +limit <- suppressWarnings(as.integer(opt_val("--limit", NA_character_))) + +if (nchar(Sys.getenv("PGPASS")) == 0L) { + stop("PGPASS env var is not set; a DB password is required.") +} + +registry_file <- file.path(script_dir, "patches", "registry.json") +ledger_file <- file.path(script_dir, "patches", "proposals-log.json") +# Branch the autonomous PR reuses, so re-runs update one PR instead of piling up. +pr_branch <- "auto/registry-patch-proposals" + +# --------------------------------------------------------------------------- +# Query failing builds (same shape as failing-builds-report.R) +# --------------------------------------------------------------------------- +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) +on.exit(DBI::dbDisconnect(con), add = TRUE) + +where <- "error_occurred = TRUE AND removed = FALSE" +params <- list() +if (nzchar(platform)) { + where <- paste0(where, " AND platform = $", length(params) + 1L) + params <- c(params, platform) +} +if (nzchar(arch)) { + where <- paste0(where, " AND arch = $", length(params) + 1L) + params <- c(params, arch) +} +failures <- DBI::dbGetQuery( + con, + paste0( + "SELECT name, tag, platform, arch, r_version, timestamp, error_text ", + "FROM single_builds WHERE ", + where + ), + params = if (length(params) > 0L) params else NULL +) +cat(sprintf("Queried single_builds: %d failing builds\n", nrow(failures))) + +# --------------------------------------------------------------------------- +# Existing registry + classification +# --------------------------------------------------------------------------- +existing_entries <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +registered_pkgs <- unique(vapply( + existing_entries, + function(e) as.character(e$package %||% ""), + character(1L) +)) + +report <- build_triage_report(failures, registered_pkgs) +report <- Filter(function(r) r$build_count >= min_count, report) + +# Flatten auto-proposable groups into candidate records, carrying the group's +# build volume so we can prioritise the highest-impact fixes. +candidates <- list() +for (r in report) { + if (is.null(r$proposed_entries)) { + next + } + for (pkg in names(r$proposed_entries)) { + candidates[[length(candidates) + 1L]] <- list( + package = pkg, + signature = r$signature, + confidence = r$confidence, + tier = r$tier, + build_count = r$build_count, + entry = r$proposed_entries[[pkg]] + ) + } +} + +# Packages blocked on a dependency are reported (aggregated by dependency, +# ranked by impact), not proposed: fixing the named dependency clears the batch. +blocked <- blocked_by_dependency(report) +if (length(blocked) > 0L) { + n_blocked_pkgs <- length(unique(unlist( + lapply(report, function(r) r$blocked_packages) + ))) + cat(sprintf( + "\nBlocked on a dependency (%d dependencies block %d dependents; fix the dependency, not each dependent):\n", + length(blocked), + n_blocked_pkgs + )) + for (b in blocked) { + cat(sprintf(" %-20s %5d dependent(s)\n", b$dependency, b$n_packages)) + } +} + +# A package that maps to more than one auto-proposable signature is ambiguous +# (conflicting fix tiers) and would collide on the same registry key; route it +# to human triage instead of emitting both. +split_candidates <- dedupe_candidates(candidates) +candidates <- split_candidates$keep +if (length(split_candidates$ambiguous) > 0L) { + cat("\nAmbiguous (multiple signatures) -> human triage, not proposed:\n") + for (pkg in names(split_candidates$ambiguous)) { + cat(sprintf( + " %s: %s\n", + pkg, + toString(split_candidates$ambiguous[[pkg]]) + )) + } +} + +if (length(candidates) == 0L) { + cat( + "\nNo auto-proposable candidates (nothing classified, safe, unregistered, and unambiguous).\n" + ) + q(status = 0) +} + +# Prioritise by failure volume, then apply --limit so one run tackles a bounded +# batch (the rest are picked up on the next run). +candidates <- candidates[order( + -vapply(candidates, function(c) c$build_count %||% 0L, integer(1L)) +)] +deferred <- 0L +if (!is.na(limit) && limit >= 0L && length(candidates) > limit) { + deferred <- length(candidates) - limit + candidates <- utils::head(candidates, limit) + cat(sprintf( + "\nLimiting to top %d candidate(s) by failure volume; %d deferred to a later run.\n", + limit, + deferred + )) +} + +cat(sprintf( + "\n%d candidate registry %s:\n", + length(candidates), + if (length(candidates) == 1L) "entry" else "entries" +)) +cat(strrep("=", 78L), "\n", sep = "") +for (c in candidates) { + j <- jsonlite::toJSON( + c$entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf( + "\n# %s [signature: %s, %s confidence]\n", + c$package, + c$signature, + c$confidence + )) + cat(j, "\n", sep = "") + cat(sprintf( + " acceptance gate: PGPASS=... Rscript local/trial-build-patch.R %s\n", + c$package + )) +} + +# --------------------------------------------------------------------------- +# Validate the candidate set against a temporary merged registry (never +# touches the real registry unless --write succeeds). +# --------------------------------------------------------------------------- +rscript <- file.path(R.home("bin"), "Rscript") +candidate_entries <- lapply(candidates, function(c) c$entry) +merged <- c(existing_entries, candidate_entries) + +validate_registry <- function(entries) { + tmp <- tempfile(fileext = ".json") + on.exit(unlink(tmp), add = TRUE) + jsonlite::write_json( + entries, + tmp, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + status <- system2( + rscript, + file.path(script_dir, "validate-patches.R"), + env = c( + paste0("PATCH_DIR=", file.path(script_dir, "patches")), + paste0("REGISTRY_FILE=", tmp) + ), + stdout = TRUE, + stderr = TRUE + ) + cat(paste0(" ", status, "\n"), sep = "") + identical(attr(status, "status"), NULL) # NULL status attr == exit 0 +} + +cat("\n", strrep("=", 78L), "\n", sep = "") +cat("Validating candidate registry:\n") +ok <- validate_registry(merged) +if (!ok) { + stop("Candidate registry failed validation; not writing or proposing.") +} + +if (!is.na(json_out)) { + jsonlite::write_json( + candidates, + json_out, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + cat(sprintf("Wrote candidate list to %s\n", json_out)) +} + +# --------------------------------------------------------------------------- +# Actions (default: none) +# --------------------------------------------------------------------------- +now <- format(Sys.time(), "%Y-%m-%d %H:%M:%S") +new_ledger_records <- lapply(candidates, function(c) { + list( + package = c$package, + signature = c$signature, + tier = c$tier, + confidence = c$confidence, + versions = c$entry$versions, + platforms = as.character(c$entry$platforms), + proposed_at = now, + status = "proposed" + ) +}) +load_ledger <- function() { + if (file.exists(ledger_file)) { + jsonlite::fromJSON(ledger_file, simplifyVector = FALSE) + } else { + list() + } +} +save_ledger <- function(led) { + jsonlite::write_json( + led, + ledger_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) +} + +if (do_write) { + jsonlite::write_json( + merged, + registry_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) + cat(sprintf( + "\nWrote %d entries to %s and updated %s.\n", + length(candidate_entries), + registry_file, + ledger_file + )) + cat( + "Next: run the acceptance-gate trial build(s), then commit and open a PR.\n" + ) +} else if (do_issue) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-issue requires FORGEJO_TOKEN.") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + issue_title <- "Auto-proposed registry patches (classifier #115)" + body_lines <- c( + sprintf("_Generated %s from `single_builds` failures._", now), + "", + "The failure classifier proposes these registry entries for packages with no current entry.", + "Each is a known, safe fix lever; **review, trial-build, and open a PR** -- nothing is applied automatically.", + "" + ) + for (c in candidates) { + j <- jsonlite::toJSON( + c$entry, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + body_lines <- c( + body_lines, + sprintf( + "### %s (%s, %s confidence)", + c$package, + c$signature, + c$confidence + ), + "", + "```json", + as.character(j), + "```", + sprintf( + "Acceptance gate: `PGPASS=... Rscript local/trial-build-patch.R %s`", + c$package + ), + "" + ) + } + new_body <- paste(body_lines, collapse = "\n") + + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(issue_title, reserved = TRUE) + ) + existing <- httr2::request(search_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + existing, + function(x) identical(x$title, issue_title), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- existing[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("Updated tracking issue #%d.\n", num)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/issues", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = issue_title, body = new_body)) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf("Opened tracking issue #%d.\n", created$number)) + } + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) +} else if (do_pr) { + forgejo_token <- Sys.getenv("FORGEJO_TOKEN") + if (nchar(forgejo_token) == 0L) { + stop("--open-pr requires FORGEJO_TOKEN (to open the PR).") + } + suppressPackageStartupMessages(library(httr2, quietly = TRUE)) + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + + # Write the entries + ledger, then commit them onto the reused auto branch. + jsonlite::write_json( + merged, + registry_file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) + save_ledger(merge_ledger(load_ledger(), new_ledger_records)) + + git <- function(...) { + # system2() with captured output runs via /bin/sh, so shell-quote every arg + # (commit messages contain "()", refs contain "^{}", etc.). + st <- system2("git", shQuote(c(...)), stdout = TRUE, stderr = TRUE) + if (!identical(attr(st, "status"), NULL)) { + stop(sprintf( + "git %s failed:\n%s", + paste(..., collapse = " "), + paste(st, collapse = "\n") + )) + } + invisible(st) + } + git("config", "user.name", Sys.getenv("GIT_USER", "devxy-bot")) + git( + "config", + "user.email", + Sys.getenv("GIT_EMAIL", "bot@devxy.io") + ) + git("checkout", "-B", pr_branch) + git( + "add", + file.path(script_dir, "patches", "registry.json"), + ledger_file + ) + git( + "commit", + "-m", + sprintf( + "feat(patches): auto-propose %d registry %s from classified failures", + length(candidate_entries), + if (length(candidate_entries) == 1L) "entry" else "entries" + ) + ) + # Push over HTTPS with FORGEJO_TOKEN (same token used for the PR API), so no + # separate write-scoped secret is needed. The read-only `origin` clone URL + # can't push, so build an authenticated URL explicitly. + push_target <- sprintf( + "https://%s:%s@git.devxy.io/%s.git", + Sys.getenv("GIT_REMOTE_USER", "pat-s"), + forgejo_token, + repo + ) + git("push", "-f", push_target, sprintf("HEAD:refs/heads/%s", pr_branch)) + + pr_title <- sprintf( + "feat(patches): auto-proposed registry patches (%s)", + now + ) + body_lines <- c( + sprintf( + "_Auto-generated %s by `local/propose-patches.R --open-pr` from classified `single_builds` failures._", + now + ), + "", + sprintf( + "Adds %d known-lever registry %s (top by failure volume%s).", + length(candidate_entries), + if (length(candidate_entries) == 1L) "entry" else "entries", + if (deferred > 0L) { + sprintf("; %d deferred to a later run", deferred) + } else { + "" + } + ), + "", + "**Merge gate:** the `trial-build-registry` pipeline builds each new entry in its target build-env image; merge only once it is green.", + "Novel source diffs and unknown signatures are never auto-proposed.", + "", + "| package | signature | platforms |", + "| --- | --- | --- |" + ) + for (c in candidates) { + body_lines <- c( + body_lines, + sprintf( + "| %s | %s | %s |", + c$package, + c$signature, + toString(as.character(c$entry$platforms)) + ) + ) + } + new_body <- paste(body_lines, collapse = "\n") + + # One PR per reused branch: update if open, else create. + pulls_url <- sprintf( + "%s/repos/%s/pulls?state=open&limit=50", + forgejo_base, + repo + ) + open_pulls <- httr2::request(pulls_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() |> + httr2::resp_body_json(simplifyVector = FALSE) + match_idx <- which(vapply( + open_pulls, + function(p) identical(p$head$ref, pr_branch), + logical(1L) + )) + if (length(match_idx) > 0L) { + num <- open_pulls[[match_idx[1]]]$number + httr2::request(sprintf("%s/repos/%s/pulls/%d", forgejo_base, repo, num)) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(title = pr_title, body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + cat(sprintf("\nUpdated auto-patch PR #%d (branch %s).\n", num, pr_branch)) + } else { + created <- httr2::request(sprintf( + "%s/repos/%s/pulls", + forgejo_base, + repo + )) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list( + title = pr_title, + head = pr_branch, + base = "main", + body = new_body + )) |> + httr2::req_perform() |> + httr2::resp_body_json() + cat(sprintf( + "\nOpened auto-patch PR #%d (branch %s).\n", + created$number, + pr_branch + )) + } +} else { + cat( + "\nDry run: no changes made. Re-run with --write, --open-issue, or --open-pr to act.\n" + ) +} diff --git a/local/query-pkgs-without-old-versions.R b/local/query-pkgs-without-old-versions.R new file mode 100644 index 0000000..29ba891 --- /dev/null +++ b/local/query-pkgs-without-old-versions.R @@ -0,0 +1,39 @@ +library(s3fs) + +# List all files under contrib// +all_files <- s3fs::s3_dir_ls( + "s3://devxy-r-package-binaries-hel1/arm64/alpine322/latest/src/contrib/", + recurse = TRUE, + type = "file" +) + +pattern <- ".*/src/contrib/([^/_]+)_.*" +matches <- regmatches(all_files, regexec(pattern, all_files)) +pkg_names <- unique( + vapply( + matches, + function(x) if (length(x) > 1) x[2] else NA_character_, + character(1) + ) +) +pkg_names <- pkg_names[!is.na(pkg_names)] + +# For each package, check if Archive// contains any files +no_archive_files <- character(0) +for (pkg in pkg_names) { + archive_dir1 <- sprintf( + "s3://devxy-r-package-binaries-hel1/arm64/alpine322/latest/src/contrib/Archive/%s", + pkg + ) + archive_files <- unique(c( + tryCatch( + s3fs::s3_dir_ls(archive_dir1, recurse = TRUE), + error = function(e) character(0) + ) + )) + if (length(archive_files) == 0) { + no_archive_files <- c(no_archive_files, pkg) + } +} + +print(no_archive_files) diff --git a/local/r-minor-helpers.R b/local/r-minor-helpers.R new file mode 100644 index 0000000..65aafdf --- /dev/null +++ b/local/r-minor-helpers.R @@ -0,0 +1,33 @@ +# Metadata-only ABI triage so the full build avoids downloading every source. +# Mirrors bincraft::abi_classify rules 1-2; "ambiguous" packages still need a +# source grep via bincraft::needs_per_minor_recompile(). + +classify_from_metadata <- function(needs_compilation, linking_to, risky_deps) { + nc <- if (length(needs_compilation) == 0L || is.na(needs_compilation)) { + "" + } else { + tolower(trimws(needs_compilation)) + } + if (!identical(nc, "yes")) { + return("not-sensitive") + } + lt <- if (length(linking_to) == 0L || is.na(linking_to)) "" else linking_to + linked <- trimws(unlist(strsplit(lt, "[,\n]"))) + linked <- sub("\\s*\\(.*\\)$", "", linked) # strip "(>= x)" constraints + linked <- linked[nzchar(linked)] + if (any(linked %in% risky_deps)) { + return("sensitive") + } + "ambiguous" +} + +parse_build_args <- function(args) { + sensitive_only <- "--sensitive-only" %in% args + pos <- args[!startsWith(args, "--")] + list( + sensitive_only = sensitive_only, + split_into = as.integer(pos[1L]), + split_index = as.integer(pos[2L]), + ncpus = as.integer(pos[3L]) + ) +} diff --git a/local/rebuild-missing-helpers.R b/local/rebuild-missing-helpers.R new file mode 100644 index 0000000..16a588e --- /dev/null +++ b/local/rebuild-missing-helpers.R @@ -0,0 +1,87 @@ +# Pure helpers for local/rebuild-missing.R, kept separate so local/tests can +# source them without executing a rebuild. + +# Interleaved slice of the rebuild list. +# +# The list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, +# rstan*), so contiguous thirds would be badly unbalanced. Interleaving also +# makes each shard's progress counter representative of the slot as a whole. +shard_slice <- function(pkgs, split_into, split_index) { + split_into <- as.integer(split_into) + split_index <- as.integer(split_index) + if (is.na(split_into) || is.na(split_index)) { + stop("shard_slice(): split_into and split_index must be integers") + } + if (split_into < 1L || split_index < 1L || split_index > split_into) { + stop(sprintf( + "shard_slice(): need 1 <= split_index <= split_into, got %s of %s", + split_index, + split_into + )) + } + # seq() errors on a descending range, which is what an empty list or a shard + # index past the end would produce. + if (length(pkgs) < split_index) { + return(pkgs[0L]) + } + pkgs[seq.int(split_index, length(pkgs), by = split_into)] +} + +# Packages that still need building, decided from the bucket rather than from +# remembered progress. +# +# This is bincraft's `check_s3_root_package()` evaluated in bulk: an object +# whose ETag equals CRAN's published MD5sum is byte-identical to CRAN's source, +# so the build that was supposed to replace it has not happened yet. +# +# `etag_by_file` named by `_.tar.gz`, values are unquoted ETags +# `cran_version` named by package +# `cran_md5` named by `_` +# +# Unknown always means "already a binary", never "rebuild it", so an unreadable +# CRAN index or a multipart ETag can never mass-schedule work. +outstanding_packages <- function(pkgs, etag_by_file, cran_version, cran_md5) { + if (length(pkgs) == 0L) { + return(pkgs) + } + + # An empty table indexes to zero length rather than to NA, which would + # recycle the whole result away and silently report "nothing to build". + lookup <- function(table, key) { + if (length(table) == 0L) { + return(rep(NA_character_, length(key))) + } + unname(as.character(table[key])) + } + + version <- lookup(cran_version, pkgs) + file <- sprintf("%s_%s.tar.gz", pkgs, version) + etag <- lookup(etag_by_file, file) + md5 <- lookup(cran_md5, paste(pkgs, version, sep = "_")) + + # No CRAN version means the package cannot be resolved to a tarball at all; + # leave it in and let bincraft report why. + unresolved <- is.na(version) + # No object at the key: never built, so it is outstanding by definition. + absent <- !unresolved & is.na(etag) + # A multipart upload carries a compound ETag rather than an MD5. + unknown <- !is.na(etag) & grepl("-", etag, fixed = TRUE) + + is_source <- !unresolved & + !is.na(etag) & + !unknown & + !is.na(md5) & + etag == md5 + + pkgs[unresolved | absent | is_source] +} + +parse_rebuild_args <- function(args) { + pos <- args[!startsWith(args, "--")] + budget <- as.numeric(pos[3L]) + list( + split_into = as.integer(pos[1L]), + split_index = as.integer(pos[2L]), + budget_hours = if (is.na(budget)) 20 else budget + ) +} diff --git a/local/rebuild-missing.R b/local/rebuild-missing.R new file mode 100644 index 0000000..369024e --- /dev/null +++ b/local/rebuild-missing.R @@ -0,0 +1,194 @@ +### Rebuild one shard of a slot's missing-binary list. +# +# Usage: Rscript local/rebuild-missing.R [budget_hours] +# +# The list itself comes from local/fetch-rebuild-packages-from-issue.R, which +# writes $REBUILD_PKG_LIST (default /tmp/rebuild_pkgs.txt). +# +# Two properties matter here and are the reason this is a script rather than an +# `R -q -e` argument in the pipeline: +# +# * it is restartable. The outstanding set is re-derived from the bucket on +# every start, so a shard that died resumes where it stopped without any +# progress file, and without replaying thousands of per-package HEADs. +# * it terminates. A wall-clock budget stops the loop cleanly instead of the +# run having to be killed, which is what previously skipped the re-index and +# CDN purge and left rebuilt binaries hidden behind stale edge copies. + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +library(bincraft, quietly = TRUE) + +source(file.path("local", "rebuild-missing-helpers.R")) + +args <- parse_rebuild_args(commandArgs(trailingOnly = TRUE)) +if (is.na(args$split_into) || is.na(args$split_index)) { + stop("usage: rebuild-missing.R [budget_hours]") +} + +list_file <- Sys.getenv("REBUILD_PKG_LIST", "/tmp/rebuild_pkgs.txt") +pkgs <- if (file.exists(list_file)) readLines(list_file) else character(0) +pkgs <- pkgs[nzchar(pkgs)] +if (length(pkgs) == 0L) { + cat("Nothing to rebuild\n") + q("no") +} + +excluded <- jsonlite::fromJSON("local/excluded-packages.json")[["package"]] +pkgs <- setdiff(pkgs, excluded) + +mine <- shard_slice(pkgs, args$split_into, args$split_index) +cat(sprintf( + "Shard %s/%s: %s of %s listed packages\n", + args$split_index, + args$split_into, + length(mine), + length(pkgs) +)) + +### Resume: ask the bucket what is still outstanding + +codename <- bincraft::set_codename(NULL) +local_machine <- Sys.info()[["machine"]] +arch <- if (grepl("arm64|aarch64", local_machine)) "arm64" else "amd64" +slot_dir <- sprintf( + "devxy-rpkgs-binaries/%s/%s/latest/src/contrib", + arch, + codename +) + +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region_name = "eu-central-003", + refresh = TRUE +) + +# One paginated listing instead of a HEAD per package. Not recursed: the +# rebuild passes no `is_r_minor_sensitive`, so it only ever targets the flat +# path, and the resume filter matches that scope deliberately. +info <- tryCatch(s3fs::s3_dir_info(slot_dir), error = function(e) NULL) +etag_by_file <- if (is.null(info) || nrow(info) == 0L) { + cat(sprintf( + "WARNING: could not list %s; building the whole shard\n", + slot_dir + )) + stats::setNames(character(), character()) +} else { + stats::setNames( + gsub('^"|"$', "", as.character(info$etag)), + basename(as.character(info$uri)) + ) +} + +cran <- tryCatch( + { + con <- gzcon(url( + "https://cloud.r-project.org/src/contrib/PACKAGES.gz", + open = "rb" + )) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "MD5sum")) + }, + error = function(e) { + cat(sprintf( + "WARNING: could not read CRAN's index (%s)\n", + conditionMessage(e) + )) + NULL + } +) +cran_version <- stats::setNames(character(), character()) +cran_md5 <- stats::setNames(character(), character()) +if (!is.null(cran)) { + cran_version <- stats::setNames( + as.character(cran[, "Version"]), + as.character(cran[, "Package"]) + ) + keep <- !is.na(cran[, "MD5sum"]) + cran_md5 <- stats::setNames( + as.character(cran[keep, "MD5sum"]), + paste(cran[keep, "Package"], cran[keep, "Version"], sep = "_") + ) +} + +before <- length(mine) +mine <- outstanding_packages(mine, etag_by_file, cran_version, cran_md5) +cat(sprintf( + "Resume: %s of %s already carry a binary; %s outstanding\n", + before - length(mine), + before, + length(mine) +)) + +if (length(mine) == 0L) { + cat("Nothing outstanding for this shard\n") + q("no") +} + +### Build + +options( + crayon.enabled = TRUE, + Ncpus = as.integer(Sys.getenv("NCPUS", "2")), + future.globals.onReference = NULL +) + +started <- Sys.time() +n <- length(mine) +completed <- 0L +for (i in seq_along(mine)) { + elapsed <- as.numeric(difftime(Sys.time(), started, units = "hours")) + if (elapsed > args$budget_hours) { + cat(sprintf( + "Budget of %sh reached after %d/%d packages; stopping cleanly. The next run resumes from the bucket.\n", + args$budget_hours, + completed, + n + )) + break + } + + x <- mine[i] + cat(sprintf("[%d/%d] %s\n", i, n, x)) + tryCatch( + bincraft::build_binary_package( + x, + tag_limit = 1L, + patches = "local/patches", + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + upload = TRUE, + store_build_metadata = TRUE + ), + error = function(e) { + cat(sprintf("ERROR building %s - %s\n", x, conditionMessage(e))) + } + ) + completed <- completed + 1L +} + +cat(sprintf( + "Shard %s/%s finished: %d/%d packages processed in %.1fh\n", + args$split_index, + args$split_into, + completed, + n, + as.numeric(difftime(Sys.time(), started, units = "hours")) +)) diff --git a/local/repair-built-stamp.R b/local/repair-built-stamp.R new file mode 100644 index 0000000..06f73be --- /dev/null +++ b/local/repair-built-stamp.R @@ -0,0 +1,175 @@ +#!/usr/bin/env Rscript + +### Rewrite a broken `Built` stamp across one arch/codename slot. +### +### A slot's index can end up advertising a stamp whose platform triple is +### unusable, e.g. `Built: R 4.5.0; NA; ...`. uvr picks binary vs source by +### matching that triple plus the R minor, so no client matches it and the whole +### slot silently reverts to source-only, which makes uvr compile everything and +### fail wherever a system `-dev` library is missing. +### +### Rewriting it is not a matter of re-running the normal index update. +### `upload_package_index()` reuses the slot's remote `PACKAGES.db`, and +### cranlike's `update_db()` only reparses files whose md5 changed, so entries +### already in the database keep the stamp they were written with. Dropping +### `PACKAGES.db` to force a full reparse does work, but for an S3 repo cranlike +### reads each package's metadata from the CRAN *source* mirror on GitHub, so a +### 25k-entry slot means 25k requests to raw.githubusercontent.com and a real +### risk of being rate-limited part-way through. +### +### Only the `Built` column is wrong, so correct it in place instead: patch the +### column in `PACKAGES.db`, put the database back, and let +### `upload_package_index()` re-emit `PACKAGES*` from it. `update_db()` always +### rewrites the index files even when nothing was reparsed, so no tarball is +### re-read and nothing is fetched from GitHub. +### +### The replacement comes from `bincraft::built_stamp()` under the R running +### this script, so run it under the R version the slot should advertise (the +### `R_VERSION` its entry in `.crow/process-updates.yaml` uses). That is what a +### healthy `upload_package_index()` run would have written. +### +### Usage, inside the platform's build image: +### Rscript local/repair-built-stamp.R [--apply] +### +### Without `--apply` it reports what it would change and touches nothing. + +suppressPackageStartupMessages({ + library(bincraft) +}) + +args <- commandArgs(trailingOnly = TRUE) +arch <- args[1L] +apply_changes <- "--apply" %in% args + +if (is.na(arch) || !nzchar(arch)) { + stop( + "Usage: Rscript local/repair-built-stamp.R [--apply]", + call. = FALSE + ) +} + +bucket <- "devxy-rpkgs-binaries" +endpoint <- "https://s3.eu-central-003.backblazeb2.com" +region <- "eu-central-003" + +codename <- bincraft::set_codename(NULL) +if (is.null(codename) || is.na(codename) || !nzchar(codename)) { + stop( + "Could not detect a codename from /etc/os-release; run this in a build image.", + call. = FALSE + ) +} + +# built_stamp() refuses an unusable platform, so a broken build image fails here +# rather than writing a second bad stamp over the first one. +stamp <- bincraft::built_stamp() +message(sprintf("Slot: %s/%s", arch, codename)) +message(sprintf("New stamp: %s", stamp)) + +s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + endpoint = endpoint, + region_name = region, + refresh = TRUE +) + +base_dir <- file.path(bucket, arch, codename, "latest", "src", "contrib") + +# A stamp is broken when its platform component is absent or literally "NA". +broken_stamp_where <- paste( + "Built IS NULL", + "OR Built LIKE '%; NA;%'", + "OR Built LIKE '%; ;%'" +) + +# The generic slot plus every per-minor sub-slot, which carry the same stamp and +# are poisoned by the same run. +r_minors <- sub( + "^.*/R/([0-9]+\\.[0-9]+)\\.[0-9]+$", + "\\1", + list.dirs("/opt/R", recursive = FALSE) +) +r_minors <- unique(grep("^[0-9]+\\.[0-9]+$", r_minors, value = TRUE)) +slots <- c(base_dir, file.path(base_dir, r_minors)) + +repair_slot <- function(slot) { + db_remote <- file.path(slot, "PACKAGES.db") + if (!s3fs::s3_file_exists(db_remote)) { + message(sprintf(" %s: no PACKAGES.db, skipping", slot)) + return(invisible(NULL)) + } + + db_local <- tempfile(fileext = ".db") + s3fs::s3_file_download(db_remote, db_local, overwrite = TRUE) + + con <- DBI::dbConnect(RSQLite::SQLite(), db_local) + on.exit(DBI::dbDisconnect(con), add = TRUE) + + total <- DBI::dbGetQuery(con, "SELECT COUNT(*) AS n FROM packages")$n + broken <- DBI::dbGetQuery( + con, + sprintf("SELECT COUNT(*) AS n FROM packages WHERE %s", broken_stamp_where) + )$n + + message(sprintf(" %s: %s entries, %s broken", slot, total, broken)) + if (broken == 0L) { + return(invisible(NULL)) + } + if (!apply_changes) { + message(" (dry run, pass --apply to rewrite)") + return(invisible(NULL)) + } + + DBI::dbExecute( + con, + sprintf("UPDATE packages SET Built = ? WHERE %s", broken_stamp_where), + params = list(stamp) + ) + DBI::dbDisconnect(con) + on.exit() + + s3fs::s3_file_upload(db_local, db_remote, overwrite = TRUE) + message(sprintf(" rewrote %s entries and uploaded PACKAGES.db", broken)) + invisible(NULL) +} + +invisible(lapply(slots, repair_slot)) + +if (!apply_changes) { + message("Dry run complete; nothing was changed.") + quit(save = "no") +} + +# Re-emit PACKAGES/PACKAGES.gz/PACKAGES.rds from the corrected database. Nothing +# is reparsed, because no tarball's md5 changed. +message("Re-emitting index files from the corrected database...") +bincraft::upload_package_index( + codename = codename, + s3_endpoint = endpoint, + s3_region = region, + s3_bucket = bucket, + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") +) +for (minor in r_minors) { + try( + bincraft::upload_package_index( + codename = codename, + r_minor = minor, + s3_endpoint = endpoint, + s3_region = region, + s3_bucket = bucket, + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY") + ), + silent = FALSE + ) +} + +message("Done. Verify with:") +message(sprintf( + " curl -sS https://cran.devxy.io/%s/%s/latest/src/contrib/PACKAGES | grep '^Built:' | sort | uniq -c", + arch, + codename +)) diff --git a/local/test-package-loading.R b/local/test-package-loading.R new file mode 100644 index 0000000..a3b7a28 --- /dev/null +++ b/local/test-package-loading.R @@ -0,0 +1,337 @@ +# Installs every CRAN package one by one and checks that it loads. Dependencies +# go through uvr via local/uvr-install.sh, which bootstraps the uvr binary on +# first use and installs into .libPaths()[1]. +uvr_install <- function(pkg) { + Sys.setenv( + UVR_R_BIN = file.path(R.home("bin"), "R"), + UVR_TARGET_LIB = .libPaths()[1L] + ) + status <- system2("local/uvr-install.sh", shQuote(pkg)) + if (!identical(status, 0L)) { + stop( + sprintf("uvr failed to install %s (exit %s)", pkg, status), + call. = FALSE + ) + } +} + +all_pkgs <- rownames(available.packages()) + +to_skip <- c("ABRSQOL", "ACA", "ACE.CoCo") +all_pkgs <- setdiff(all_pkgs, to_skip) + +for (i in all_pkgs) { + message(sprintf("\nInstalling %s", i)) + uvr_install(i) + library(i, character.only = TRUE) +} + + +# Example data +all_pkgs <- rownames(available.packages()) +to_skip <- c( + "ABRSQOL", + "ACA", + "ACE.CoCo", + "ACEsimFit", + "ACNE", + "absorber", + "adapt4pv", + "adaptMCMC", + "addhaz", + "ADDT", + "ahaz", + "arm", + "arules", + "arulesCBA", + "aster2", + "BayesFactor", + "bc3net", + "bgsmtr", + "biglasso", + "BinNonNor", + "BinNor", + "bioassayR", + "birankr", + "BiRewire", + "bolasso", + "Boptbd", + "Brobdingnag", + "BSW", + "BTLLasso", + "bvartools", + "cAIC4", + "Category", + "celda", + "centiserve", + "cjoint", + "clinical", + "clipper", + "CodataGS", + "conos", + "CopulaInference", + "covEB", + "cplm", + "CRTgeeDR", + "cthreshER", + "ctmcmove", + "curephEM", + "CVST", + "dcGSA", + "dclone", + "dcsvm", + "DelayedArray", + "dglars", + "dhglm", + "disordR", + "distrom", + "dmm", + "DNABarcodes", + "DoubleCone", + "DRR", + "DTRlearn2", + "DWDLargeR", + "eds", + "EMCluster", + "EMMREML", + "evalITR", + "EventPointer", + "evola", + "excursions", + "expm", + "fanc", + "FAS", + "fastadi", + "fastPLS", + "fastRG", + "fdaPDE", + "flare", + "FoReco", + "frailtyHL", + "freebird", + "FSTpackage", + "gamlr", + "gamlss.lasso", + "gamm4", + "gbmt", + "gdim", + "gdistance", + "GeDS", + "geeM", + "genlasso", + "GenOrd", + "GenoScan", + "geomorph", + "geostatsp", + "GhostKnockoff", + "GIGSEA", + "GlarmaVarSel", + "glmm", + "glmmrBase", + "glmmrOptim", + "glmnet", + "glober", + "GPvam", + "graphpcor", + "gremlin", + "growthrate", + "grpCox", + "GSD", + "HelpersMG", + "hglm", + "hglm.data", + "hibayes", + "hierSDR", + "HMTL", + "hsem", + "ibmdbR", + "inca", + "INLAspacetime", + "INLAtools", + "invertiforms", + "irlba", + "islasso", + "ISLET", + "isotonic.pen", + "jordan", + "kinship2", + "KnockoffScreen", + "lcpm", + "leidenAlg", + "lfe", + "lingmatch", + "LKT", + "lme4", + "lme4breeding", + "lme4GS", + "logcondiscr", + "LPmerge", + "LRMF3", + "MAP", + "marcox", + "markovchain", + "MatrixExtra", + "matter", + "MBC", + "mcen", + "mclogit", + "MCMCglmm", + "mdhglm", + "MDPtoolbox", + "mediation", + "mefa4", + "metafor", + "mgwrsar", + "mi", + "midasml", + "mind", + "monocle", + "msda", + "MuData", + "MultiGlarmaVarSel", + "MultiOrd", + "MultiVarSel", + "mvglmmRank", + "N2R", + "nadiv", + "NBtsVarSel", + "NegBinBetaBinreg", + "NetworkRiskMeasures", + "neuroim2", + "NOISeq", + "numbat", + "optbdmaeAT", + "optimbase", + "OptimModel", + "optrcdmaeAT", + "OrdNor", + "pagoda2", + "PCovR", + "pedgene", + "pedigree", + "pedigreemm", + "pense", + "PERMANOVA", + "phateR", + "PhylogeneticEM", + "pleio", + "POINT", + "PoisBinNonNor", + "PoisBinOrd", + "PoisBinOrdNonNor", + "PoisBinOrdNor", + "PoisNonNor", + "PoisNor", + "PRISMA", + "ProbitSpatial", + "prodest", + "psqn", + "qlcMatrix", + "qpcR", + "QRM", + "quadrupen", + "QZ", + "ramps", + "randnet", + "randPedPCA", + "rBMF", + "RCBR", + "RealVAMS", + "REBayes", + "recommenderlab", + "Rediscover", + "reglogit", + "RESET", + "RGE", + "RGENERATEPREC", + "riemtan", + "RNewsflow", + "robustlmm", + "rsparse", + "rSPDE", + "rwc", + "S4Arrays", + "saeMSPE", + "sbw", + "scITD", + "scoup", + "sdwd", + "SEAGLE", + "sensory", + "serrsBayes", + "sglasso", + "sharpPen", + "SiPSiC", + "SKAT", + "snpReady", + "snpStats", + "softImpute", + "sommer", + "soptdmaeA", + "SOR", + "SparseArray", + "SparseChol", + "sparseLRMatrix", + "sparsenet", + "sparsenetgls", + "sparsestep", + "spatialprobit", + "spatialreg", + "spatstat.sparse", + "speedglm", + "sRDA", + "sSDR", + "ssfa", + "stcos", + "StratifiedSampling", + "sureLDA", + "survey", + "surveyvoi", + "svydiags", + "systemfit", + "TargetScore", + "text2map", + "textir", + "textmineR", + "textTinyR", + "tmvtnorm", + "TPEA", + "triversity", + "tsapp", + "tvReg", + "uwot", + "vagam", + "VAM", + "WaveSampling", + "WGScan", + "wordspace", + "workflowsets", + "ACSSpack", + "ADDT", + "AER" +) + +# Find the position of the last package in to_skip within all_pkgs +last_skip <- tail(to_skip, 1) + +# Find its position in all_pkgs (returns NA if not found) +start_pos <- match(last_skip, all_pkgs) + +# If not found, start from the beginning; else, start after last_skip +if (is.na(start_pos)) { + to_process <- all_pkgs +} else { + to_process <- all_pkgs[(start_pos + 1):length(all_pkgs)] +} + +if (length(to_process) == 0) { + message("All packages have been processed.") +} else { + for (i in to_process) { + message(sprintf("\nInstalling %s", i)) + uvr_install(i) + library(i, character.only = TRUE) + } + # Update to_skip to include all up to the last processed + to_skip <- all_pkgs[1:(last_skip_pos + length(to_process))] +} diff --git a/local/tests/test-build-all-args.R b/local/tests/test-build-all-args.R new file mode 100644 index 0000000..3308fc3 --- /dev/null +++ b/local/tests/test-build-all-args.R @@ -0,0 +1,13 @@ +source(file.path("..", "r-minor-helpers.R")) + +test_that("parse_build_args splits flags from positionals", { + a <- parse_build_args(c("--sensitive-only", "4", "2", "8")) + expect_true(a$sensitive_only) + expect_identical(a$split_into, 4L) + expect_identical(a$split_index, 2L) + expect_identical(a$ncpus, 8L) + + b <- parse_build_args(c("4", "2", "8")) + expect_false(b$sensitive_only) + expect_identical(b$split_into, 4L) +}) diff --git a/local/tests/test-failing-builds-classify.R b/local/tests/test-failing-builds-classify.R new file mode 100644 index 0000000..d4df9f5 --- /dev/null +++ b/local/tests/test-failing-builds-classify.R @@ -0,0 +1,165 @@ +source(file.path("..", "failing-builds-classify.R")) + +test_that("normalise_error collapses temp paths, versions, and package tokens", { + a <- normalise_error( + "In file /tmp/RtmpAb12/foo.c: RcppParallel 5.1.9 failed at 0xdeadbeef", + package = "RcppParallel" + ) + expect_false(grepl("RtmpAb12", a)) + expect_false(grepl("5\\.1\\.9", a)) + expect_false(grepl("0xdeadbeef", a)) + expect_true(grepl("", a)) + expect_true(grepl("", a)) + + # Same root error across two versions collapses to one fingerprint. + e1 <- "StanHeaders 2.32.1: tbb/tbb_stddef.h: No such file or directory" + e2 <- "StanHeaders 2.33.0: tbb/tbb_stddef.h: No such file or directory" + expect_identical( + fingerprint_error(e1, "StanHeaders"), + fingerprint_error(e2, "StanHeaders") + ) +}) + +test_that("normalise_error is safe on NA/empty input", { + expect_identical(normalise_error(NA_character_), "") + expect_identical(normalise_error(""), "") + expect_identical(fingerprint_error(NA_character_), "") +}) + +test_that("fingerprint_error picks the salient error line, not the last line", { + txt <- paste( + "* installing *source* package 'foo' ...", + "error: bar.h: No such file or directory", + "* removing '/tmp/lib/foo'", + sep = "\n" + ) + fp <- fingerprint_error(txt, "foo") + expect_true(grepl("no such file", fp)) + expect_false(grepl("removing", fp)) +}) + +test_that("classify_error matches the removed TBB header signature (makevars, auto)", { + sig <- classify_error( + "fatal error: tbb/tbb_stddef.h: No such file or directory" + ) + expect_identical(sig$id, "tbb-stddef-removed") + expect_identical(sig$tier, "makevars") + expect_true(sig$auto) + expect_true(sig$matched) +}) + +test_that("classify_error matches RcppParallel bundled-TBB failures", { + sig <- classify_error( + "Error: USE_TBB=Linux is not supported on this toolchain" + ) + expect_identical(sig$id, "rcppparallel-bundled-tbb") + expect_identical(sig$tier, "patch") + expect_true(sig$auto) +}) + +test_that("system libuv link leak is classified but stays human-only (novel diff)", { + sig <- classify_error("cannot open shared object file: libuv.so.1") + expect_identical(sig$id, "system-libuv-link-leak") + expect_true(sig$matched) + expect_false(sig$auto) +}) + +test_that("unknown signatures are never guessed at", { + sig <- classify_error("segfault: memory not mapped at address") + expect_identical(sig$id, "unclassified") + expect_false(sig$matched) + expect_false(sig$auto) + expect_true(is.na(sig$tier)) +}) + +test_that("propose_registry_entry fills a schema-valid entry for a known lever", { + sig <- classify_error("tbb/tbb_stddef.h: No such file") + json <- propose_registry_entry(sig, "StanHeaders", c("alpine", "ubuntu-2604")) + entry <- jsonlite::fromJSON(json, simplifyVector = FALSE) + expect_identical(entry$package, "StanHeaders") + expect_identical(entry$versions, "*") + expect_identical(entry$platforms, list("alpine", "ubuntu-2604")) + expect_identical(entry$makevars$CPPFLAGS, "-DTBB_INTERFACE_NEW") + expect_true(nzchar(entry$reason)) + + # No template -> no proposal (unclassified path). + expect_null(propose_registry_entry(classify_error("weird"), "x", "alpine")) +}) + +test_that("RcppParallel dependents are blocked, not proposed a per-package patch", { + # Hundreds of packages fail on ubuntu-2604 only because their RcppParallel + # dependency fails to build, so they carry RcppParallel's TBB error text. + failures <- data.frame( + name = c("ACEsimFit", "AovBay", "AdaptGauss"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + grp <- Filter(function(g) g$signature == "rcppparallel-bundled-tbb", report)[[ + 1L + ]] + # No bogus per-dependent entries (the patch targets RcppParallel's source). + expect_null(grp$proposed_entries) + expect_identical(grp$blocked_on, "RcppParallel") +}) + +test_that("failing_dependency names the dependency that actually failed", { + # A leaf package (BFpack) that fails building its rstan dependency. + txt <- paste( + "Error in installing dependencies for package BFpack with tag 1.6.1", + "Failed to build source package rstan.", + "ERROR: compilation failed for package ‘rstan’", + sep = "\n" + ) + expect_identical(failing_dependency(txt, "BFpack"), "rstan") + # A package failing in its OWN compilation is not a cascade. + own <- "ERROR: compilation failed for package ‘BFpack’" + expect_true(is.na(failing_dependency(own, "BFpack"))) + expect_true(is.na(failing_dependency(NA_character_, "x"))) + expect_true(is.na(failing_dependency("some unrelated error", "x"))) +}) + +test_that("Stan packages blocked on rstan are not proposed a per-package entry", { + # BFpack/GMLTM fail building rstan; the tbb error is in rstan's compile. + cascade <- paste( + "Failed to build source package rstan.", + "fatal error: tbb/tbb_stddef.h: No such file or directory", + sep = "\n" + ) + failures <- data.frame( + name = c("BFpack", "GMLTM", "someOwnPkg"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = c( + cascade, + cascade, + # someOwnPkg fails in its OWN compile on the same header -> fixable. + "someOwnPkg.cpp: fatal error: tbb/tbb_stddef.h: No such file or directory" + ), + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + grp <- Filter(function(g) g$signature == "tbb-stddef-removed", report)[[1L]] + # Only the own-compile package is proposed; the rstan cascades are blocked. + expect_identical(names(grp$proposed_entries), "someOwnPkg") + expect_setequal(grp$blocked_packages, c("BFpack", "GMLTM")) + expect_identical(grp$blocked_on, "rstan") +}) + +test_that("RcppParallel itself is still proposed when it is the failing package", { + failures <- data.frame( + name = "RcppParallel", + platform = "ubuntu-2604", + arch = "amd64", + error_text = "Error: USE_TBB=Linux is not supported on this toolchain", + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + grp <- report[[1L]] + expect_identical(grp$signature, "rcppparallel-bundled-tbb") + expect_false(is.null(grp$proposed_entries)) + expect_true("RcppParallel" %in% names(grp$proposed_entries)) + expect_null(grp$blocked_on) # RcppParallel is the target, not a dependent +}) diff --git a/local/tests/test-proposal-tracking-lib.R b/local/tests/test-proposal-tracking-lib.R new file mode 100644 index 0000000..869d2b3 --- /dev/null +++ b/local/tests/test-proposal-tracking-lib.R @@ -0,0 +1,246 @@ +source(file.path("..", "proposal-tracking-lib.R")) +source(file.path("..", "failing-builds-classify.R")) + +mk_failures <- function() { + data.frame( + name = c("StanHeaders", "rstan", "RcppParallel", "somepkg"), + platform = c("alpine-321", "alpine-321", "alpine-320", "redhat-9"), + arch = c("amd64", "arm64", "amd64", "amd64"), + error_text = c( + "fatal error: tbb/tbb_stddef.h: No such file or directory", + "In file: tbb/tbb_stddef.h: No such file or directory", + "Error: USE_TBB=Linux is not supported; bundled TBB on musl", + "some unmatched failure" + ), + stringsAsFactors = FALSE + ) +} + +test_that("merge_ledger appends new proposals and preserves existing history", { + existing <- list(list( + package = "fs", + signature = "system-libuv-link-leak", + status = "merged" + )) + new <- list( + list( + package = "fs", + signature = "system-libuv-link-leak", + status = "proposed" + ), + list( + package = "StanHeaders", + signature = "tbb-stddef-removed", + status = "proposed" + ) + ) + merged <- merge_ledger(existing, new) + expect_length(merged, 2L) # fs is deduped, StanHeaders added + fs <- Filter(function(r) r$package == "fs", merged)[[1L]] + expect_identical(fs$status, "merged") # existing status preserved, not clobbered +}) + +test_that("merge_ledger handles an empty/NULL starting ledger", { + new <- list(list(package = "x", signature = "s")) + expect_length(merge_ledger(NULL, new), 1L) + expect_length(merge_ledger(list(), new), 1L) +}) + +test_that("dedupe_candidates keeps single-signature pkgs, routes conflicts to triage", { + candidates <- list( + list(package = "StanHeaders", signature = "tbb-stddef-removed"), + list(package = "hmmTMB", signature = "tbb-stddef-removed"), + list(package = "hmmTMB", signature = "rcppparallel-bundled-tbb") # conflict + ) + out <- dedupe_candidates(candidates) + kept <- vapply(out$keep, function(c) c$package, character(1L)) + expect_identical(sort(kept), "StanHeaders") # hmmTMB dropped as ambiguous + expect_true("hmmTMB" %in% names(out$ambiguous)) + expect_setequal( + out$ambiguous$hmmTMB, + c("tbb-stddef-removed", "rcppparallel-bundled-tbb") + ) +}) + +test_that("dedupe_candidates collapses a package repeated under one signature", { + candidates <- list( + list(package = "rstan", signature = "tbb-stddef-removed"), + list(package = "rstan", signature = "tbb-stddef-removed") + ) + out <- dedupe_candidates(candidates) + expect_length(out$keep, 1L) + expect_length(out$ambiguous, 0L) +}) + +test_that("dedupe_candidates handles the empty list", { + out <- dedupe_candidates(list()) + expect_length(out$keep, 0L) + expect_length(out$ambiguous, 0L) +}) + +test_that("signature_hit_rate splits addressed vs open per signature", { + report <- build_triage_report(mk_failures(), registered_pkgs = "RcppParallel") + hit <- signature_hit_rate(report, registered_pkgs = "RcppParallel") + tbb <- Filter(function(h) h$signature == "tbb-stddef-removed", hit)[[1L]] + expect_identical(tbb$packages, 2L) # StanHeaders + rstan + expect_identical(tbb$addressed, 0L) + expect_identical(tbb$open, 2L) + expect_true(tbb$auto_proposable) + + rcpp <- Filter(function(h) h$signature == "rcppparallel-bundled-tbb", hit)[[ + 1L + ]] + expect_identical(rcpp$addressed, 1L) # already registered + expect_identical(rcpp$open, 0L) + + # Unclassified failures never appear as a signature. + expect_false( + "unclassified" %in% vapply(hit, function(h) h$signature, character(1L)) + ) +}) + +test_that("proposed_vs_merged marks a package merged once it is registered", { + ledger <- list( + list( + package = "StanHeaders", + signature = "tbb-stddef-removed", + status = "proposed" + ), + list( + package = "rstan", + signature = "tbb-stddef-removed", + status = "proposed" + ) + ) + pvm <- proposed_vs_merged(ledger, registered_pkgs = "StanHeaders") + expect_identical(pvm$total, 2L) + expect_identical(pvm$merged, 1L) + stan <- Filter(function(r) r$package == "StanHeaders", pvm$records)[[1L]] + expect_identical(stan$status, "merged") +}) + +test_that("unclassified_summary ranks unknown groups and caps output", { + failures <- data.frame( + name = c("a", "b", "c", "d", "solo"), + platform = "ubuntu-2604", + arch = "amd64", + error_text = c( + # 4 builds share one unknown fingerprint; 1 build a different unknown. + rep("mystery linker meltdown at stage 3", 4L), + "a totally different unknown boom" + ), + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + s <- unclassified_summary(report, max_groups = 30L, max_pkgs = 2L) + expect_identical(s$total_groups, 2L) + expect_identical(s$total_builds, 5L) + # Largest group first, and its example packages are capped at max_pkgs. + expect_identical(s$groups[[1L]]$build_count, 4L) + expect_length(s$groups[[1L]]$packages, 2L) + expect_true(s$groups[[1L]]$packages_truncated) + + # max_groups cap is reported, not silently dropped. + capped <- unclassified_summary(report, max_groups = 1L) + expect_length(capped$groups, 1L) + expect_identical(capped$dropped_groups, 1L) +}) + +test_that("blocked_by_dependency aggregates across groups, deduped and ranked", { + # RcppParallel dependents split across platforms/fingerprints -> separate + # groups, but one aggregated line; a dependent seen twice is counted once. + failures <- data.frame( + name = c("ACEsimFit", "AovBay", "AdaptGauss", "ACEsimFit", "loner"), + platform = c( + "ubuntu-2604", + "ubuntu-2604", + "alpine-324", + "alpine-324", + "ubuntu-2604" + ), + arch = "amd64", + error_text = c( + rep("Error: USE_TBB=Linux is not supported on this toolchain", 4L), + "Failed to build source package rstan.\nfatal error: tbb/tbb_stddef.h" + ), + stringsAsFactors = FALSE + ) + report <- build_triage_report(failures, registered_pkgs = character(0L)) + agg <- blocked_by_dependency(report, max_pkgs = 2L) + deps <- vapply(agg, function(b) b$dependency, character(1L)) + expect_true("RcppParallel" %in% deps && "rstan" %in% deps) + rcpp <- Filter(function(b) b$dependency == "RcppParallel", agg)[[1L]] + # ACEsimFit appears in two groups -> counted once (3 distinct dependents). + expect_identical(rcpp$n_packages, 3L) + expect_true(rcpp$packages_truncated) # capped at max_pkgs = 2 + # Ranked by dependent count: RcppParallel (3) before rstan (1). + expect_identical(deps[[1L]], "RcppParallel") +}) + +test_that("entry_applies_to_os matches codename, family, and wildcard", { + expect_true(entry_applies_to_os(list("ubuntu-2604"), "ubuntu-2604")) + expect_true(entry_applies_to_os(list("ubuntu"), "ubuntu-2604")) # family + expect_true(entry_applies_to_os(list("*"), "ubuntu-2604")) + expect_true(entry_applies_to_os(list("alpine", "ubuntu-2604"), "ubuntu-2604")) + expect_false(entry_applies_to_os(list("alpine-324"), "ubuntu-2604")) + expect_false(entry_applies_to_os(list("ubuntu-2404"), "ubuntu-2604")) # other codename +}) + +test_that("new_registry_packages returns only added entries for the platform", { + base <- list( + list( + package = "RcppParallel", + platforms = list("alpine", "ubuntu-2604"), + versions = "*" + ) + ) + current <- list( + base[[1L]], # unchanged -> not "new" + list(package = "BFpack", platforms = list("ubuntu-2604"), versions = "*"), + list( + package = "someAlpinePkg", + platforms = list("alpine-324"), + versions = "*" + ) + ) + # For ubuntu-2604: only the newly-added BFpack (RcppParallel is unchanged, + # someAlpinePkg does not apply to this OS). + expect_identical( + new_registry_packages(current, base, os = "ubuntu-2604"), + "BFpack" + ) + # For alpine-324: the alpine package is new and applies. + expect_identical( + new_registry_packages(current, base, os = "alpine-324"), + "someAlpinePkg" + ) + # Without an OS filter, both additions are returned. + expect_setequal( + new_registry_packages(current, base), + c("BFpack", "someAlpinePkg") + ) + # A changed platform set on the same package counts as a new entry. + widened <- list(list( + package = "RcppParallel", + platforms = list("*"), + versions = "*" + )) + expect_identical(new_registry_packages(widened, base), "RcppParallel") +}) + +test_that("retirement_candidates flags entries whose package no longer fails", { + entries <- list( + list(package = "RcppParallel"), + list(package = "oldpkg") + ) + # RcppParallel still fails; oldpkg does not -> only oldpkg is retirable. + out <- retirement_candidates( + entries, + failing_pkgs = c("RcppParallel", "StanHeaders") + ) + expect_identical(out, "oldpkg") + expect_length( + retirement_candidates(entries, failing_pkgs = c("RcppParallel", "oldpkg")), + 0L + ) +}) diff --git a/local/tests/test-r-minor-helpers.R b/local/tests/test-r-minor-helpers.R new file mode 100644 index 0000000..ac7b0a6 --- /dev/null +++ b/local/tests/test-r-minor-helpers.R @@ -0,0 +1,28 @@ +source(file.path("..", "r-minor-helpers.R")) + +test_that("pure-r (NeedsCompilation != yes) is not sensitive", { + expect_identical(classify_from_metadata("no", NA, c("Rcpp")), "not-sensitive") + expect_identical( + classify_from_metadata("", "Rcpp", c("Rcpp")), + "not-sensitive" + ) +}) + +test_that("LinkingTo a risky dep is sensitive (version constraints stripped)", { + expect_identical( + classify_from_metadata("yes", "Rcpp (>= 1.0)", c("Rcpp")), + "sensitive" + ) + expect_identical( + classify_from_metadata("yes", "R6,\n cpp11", c("Rcpp", "cpp11")), + "sensitive" + ) +}) + +test_that("compiled but no risky LinkingTo is ambiguous (needs source)", { + expect_identical( + classify_from_metadata("yes", "R6", c("Rcpp", "cpp11")), + "ambiguous" + ) + expect_identical(classify_from_metadata("yes", NA, c("Rcpp")), "ambiguous") +}) diff --git a/local/tests/test-rebuild-missing.R b/local/tests/test-rebuild-missing.R new file mode 100644 index 0000000..a5e70b9 --- /dev/null +++ b/local/tests/test-rebuild-missing.R @@ -0,0 +1,94 @@ +source(file.path("..", "rebuild-missing-helpers.R")) + +test_that("shard_slice partitions the list without gaps or overlap", { + pkgs <- letters[1:10] + parts <- lapply(1:3, function(i) shard_slice(pkgs, 3, i)) + + expect_identical(parts[[1]], c("a", "d", "g", "j")) + expect_identical(parts[[2]], c("b", "e", "h")) + expect_identical(parts[[3]], c("c", "f", "i")) + + expect_identical(sort(unlist(parts)), sort(pkgs)) + expect_identical(anyDuplicated(unlist(parts)), 0L) +}) + +test_that("shard_slice is deterministic and survives short lists", { + expect_identical( + shard_slice(letters[1:10], 3, 2), + shard_slice(letters[1:10], 3, 2) + ) + expect_identical(shard_slice(character(0), 3, 1), character(0)) + # more shards than packages: the tail shards get nothing rather than erroring + expect_identical(shard_slice(c("a"), 3, 1), "a") + expect_identical(shard_slice(c("a"), 3, 2), character(0)) +}) + +test_that("shard_slice rejects an out-of-range index", { + expect_error(shard_slice(letters, 3, 4), "split_index") + expect_error(shard_slice(letters, 3, 0), "split_index") +}) + +test_that("outstanding_packages keeps source fallbacks and drops real binaries", { + cran_version <- c(httr = "1.4.8", R6 = "2.6.1", curl = "7.1.0") + cran_md5 <- c( + httr_1.4.8 = "8756015b94a9cff6f410ca4de8557f12", + R6_2.6.1 = "f01b1787f12797c29194d63c9afd5d70", + curl_7.1.0 = "8af2ccbf5d85dc18866f45f1f26f348d" + ) + etag <- c( + # byte-identical to CRAN: the build never happened + "httr_1.4.8.tar.gz" = "8756015b94a9cff6f410ca4de8557f12", + # a real binary was published + "R6_2.6.1.tar.gz" = "9d6087ee9adda3f0a3b8067cfc652c05" + # curl has no object at all + ) + + out <- outstanding_packages( + c("httr", "R6", "curl"), + etag, + cran_version, + cran_md5 + ) + expect_identical(out, c("httr", "curl")) +}) + +test_that("outstanding_packages treats unknowns as already built", { + cran_version <- c(a = "1.0", b = "1.0") + cran_md5 <- c(a_1.0 = "aaaa") + + # a multipart ETag carries no MD5, and `b` is missing from CRAN's index: + # neither may schedule a rebuild + etag <- c("a_1.0.tar.gz" = "abc-3", "b_1.0.tar.gz" = "bbbb") + + expect_identical( + outstanding_packages(c("a", "b"), etag, cran_version, cran_md5), + character(0) + ) +}) + +test_that("outstanding_packages keeps a package CRAN has no version for", { + out <- outstanding_packages( + "ghost", + c(), + c(other = "1.0"), + c(other_1.0 = "aaaa") + ) + expect_identical(out, "ghost") +}) + +test_that("outstanding_packages handles an empty list", { + expect_identical( + outstanding_packages(character(0), c(), c(), c()), + character(0) + ) +}) + +test_that("parse_rebuild_args defaults the budget", { + a <- parse_rebuild_args(c("3", "2")) + expect_identical(a$split_into, 3L) + expect_identical(a$split_index, 2L) + expect_identical(a$budget_hours, 20) + + b <- parse_rebuild_args(c("3", "2", "1.5")) + expect_identical(b$budget_hours, 1.5) +}) diff --git a/local/trial-build-patch.R b/local/trial-build-patch.R new file mode 100644 index 0000000..2229074 --- /dev/null +++ b/local/trial-build-patch.R @@ -0,0 +1,68 @@ +#!/usr/bin/env Rscript + +# Acceptance gate for a proposed registry patch (issue #115, step 3): build one +# package in isolation with the current `local/patches` registry applied, and +# report whether it succeeds. Nothing is uploaded, archived, or written to the +# metadata DB -- `patchhash` keeps the trial binary out of the real cache. +# +# Must run inside a build-env image (the same image the failing platform uses), +# because it invokes the real compiler toolchain via bincraft. +# +# Usage: +# Rscript local/trial-build-patch.R [tag] +# +# Exit status: 0 if the patched build succeeds, 1 otherwise -- so it can gate a +# CI step or a manual pre-merge check. + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 1L) { + stop("usage: Rscript local/trial-build-patch.R [tag]") +} +package <- args[[1L]] +tag <- if (length(args) >= 2L) args[[2L]] else NULL + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +patches_dir <- file.path( + if (is.na(script_path)) "local" else dirname(script_path), + "patches" +) + +suppressPackageStartupMessages(library(bincraft, quietly = TRUE)) + +cat(sprintf( + "Trial build: %s%s with registry %s (no upload/archive/metadata)\n", + package, + if (is.null(tag)) "" else sprintf(" @ %s", tag), + patches_dir +)) + +# build_binary_package() catches build failures internally and RETURNS "error" +# for the failed tag rather than throwing (bincraft >= v4.4.7), so inspect the +# return value -- checking only for a thrown exception reports a broken build as +# passing (false green). +res <- tryCatch( + bincraft::build_binary_package( + package, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ), + error = function(e) { + cat(sprintf("Trial build FAILED (threw): %s\n", conditionMessage(e))) + "error" + } +) +flat <- as.character(unlist(res)) +ok <- length(flat) > 0L && !("error" %in% flat) + +if (ok) { + cat(sprintf("Trial build OK: %s builds with the proposed patch.\n", package)) + q(status = 0) +} +q(status = 1) diff --git a/local/trial-build-registry.R b/local/trial-build-registry.R new file mode 100644 index 0000000..9708408 --- /dev/null +++ b/local/trial-build-registry.R @@ -0,0 +1,163 @@ +#!/usr/bin/env Rscript + +# Merge gate for the auto-patch PR (issue #115, step 3): for every registry +# entry the PR ADDS that applies to this platform, trial-build the package with +# the registry applied, in this platform's own build-env image. Nothing is +# uploaded, archived, or written to the metadata DB. +# +# Exit 0 only if every new entry's package builds; exit 1 if any fails, so it +# gates the PR. A platform with no new entries is a fast no-op. +# +# Usage (inside a build-env image): +# PLATFORM=ubuntu-2604 Rscript local/trial-build-registry.R [base_ref] +# base_ref git ref to diff the registry against (default: origin/main) + +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + q(status = 1) +}) + +suppressPackageStartupMessages({ + library(jsonlite, quietly = TRUE) + library(bincraft, quietly = TRUE) +}) + +script_path <- local({ + a <- commandArgs(trailingOnly = FALSE) + f <- sub("^--file=", "", a[grepl("^--file=", a)]) + if (length(f) == 1L && nzchar(f)) normalizePath(f) else NA_character_ +}) +script_dir <- if (is.na(script_path)) "local" else dirname(script_path) +source(file.path(script_dir, "proposal-tracking-lib.R")) + +args <- commandArgs(trailingOnly = TRUE) +base_ref <- if (length(args) >= 1L) { + args[[1L]] +} else { + Sys.getenv("BASE_REF", "origin/main") +} +os <- Sys.getenv("PLATFORM", "") +if (!nzchar(os)) { + stop("PLATFORM env var is not set (e.g. ubuntu-2604).") +} + +patches_dir <- file.path(script_dir, "patches") +registry_file <- file.path(patches_dir, "registry.json") +current <- if (file.exists(registry_file)) { + jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +} else { + list() +} +# Read the registry at base_ref. Fail loud if the ref or file can't be read: +# silently treating the base as empty would trial-build the WHOLE registry +# instead of just the entries the branch adds. +registry_rel <- "local/patches/registry.json" +# system2() with captured output runs via /bin/sh, so shell-quote the git args +# (refs contain "^{}" and ":" that the shell would otherwise mangle). +ref_ok <- suppressWarnings(system2( + "git", + shQuote(c( + "rev-parse", + "--verify", + "--quiet", + sprintf("%s^{commit}", base_ref) + )), + stdout = TRUE, + stderr = FALSE +)) +if (!is.null(attr(ref_ok, "status"))) { + stop(sprintf("base ref %s does not resolve to a commit.", base_ref)) +} +in_base <- suppressWarnings(system2( + "git", + shQuote(c("ls-tree", base_ref, "--", registry_rel)), + stdout = TRUE, + stderr = FALSE +)) +file_in_base <- length(in_base) > 0L && any(nzchar(in_base)) +base_json <- suppressWarnings(system2( + "git", + shQuote(c("show", sprintf("%s:%s", base_ref, registry_rel))), + stdout = TRUE, + stderr = FALSE +)) +show_ok <- is.null(attr(base_json, "status")) +if (file_in_base && !show_ok) { + stop(sprintf( + "could not read %s at %s; refusing to build the whole registry.", + registry_rel, + base_ref + )) +} +base <- if (show_ok && length(base_json) > 0L) { + jsonlite::fromJSON(paste(base_json, collapse = "\n"), simplifyVector = FALSE) +} else { + list() # file genuinely absent at base -> every entry is new +} + +pkgs <- new_registry_packages(current, base, os = os) +if (length(pkgs) == 0L) { + cat(sprintf( + "No new registry entries apply to %s; nothing to trial-build.\n", + os + )) + q(status = 0) +} + +cat(sprintf( + "Trial-building %d new registry %s on %s (vs %s):\n %s\n", + length(pkgs), + if (length(pkgs) == 1L) "entry" else "entries", + os, + base_ref, + toString(pkgs) +)) + +results <- vapply( + pkgs, + function(pkg) { + cat(sprintf("\n=== trial build: %s ===\n", pkg)) + # bincraft::build_binary_package() catches build failures internally and + # RETURNS "error" for the failed tag rather than throwing, so a green gate + # must inspect the return value -- checking only for a thrown exception + # reports a broken build as passing. + res <- tryCatch( + bincraft::build_binary_package( + pkg, + tag_limit = 1L, + patches = patches_dir, + archive = FALSE, + upload = FALSE, + store_build_metadata = FALSE + ), + error = function(e) { + cat(sprintf("FAILED %s (threw): %s\n", pkg, conditionMessage(e))) + "error" + } + ) + flat <- as.character(unlist(res)) + ok <- length(flat) > 0L && !("error" %in% flat) + if (!ok) { + cat(sprintf( + "FAILED %s: build did not succeed (result: %s)\n", + pkg, + if (length(flat) > 0L) toString(flat) else "" + )) + } + ok + }, + logical(1L) +) + +failed <- pkgs[!results] +cat(sprintf( + "\n%d/%d passed on %s.%s\n", + sum(results), + length(results), + os, + if (length(failed) > 0L) sprintf(" Failed: %s", toString(failed)) else "" +)) +if (length(failed) > 0L) { + q(status = 1) +} +q(status = 0) diff --git a/local/uvr-install.sh b/local/uvr-install.sh new file mode 100755 index 0000000..3966e85 --- /dev/null +++ b/local/uvr-install.sh @@ -0,0 +1,120 @@ +#!/bin/sh +# Install R packages into the CI library with uvr (https://github.com/nbafrank/uvr). +# +# Usage: +# local/uvr-install.sh httr2 jsonlite +# local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft@v4.4.3 +# +# Replaces `pak::pak(...)`. uvr is project-scoped: `uvr add` refuses to run +# outside a project and always writes to `.uvr/library/`, and only +# `uvr sync --library` can target an existing library. The project is therefore +# minted in a scratch directory under TMPDIR and thrown away afterwards; that +# also keeps `uvr init`'s `.Rprofile` out of the repo checkout, where it would +# hijack `.libPaths()` for every other R call in the pipeline. +# +# Pruning is a no-op here: uvr disables it whenever `--library` is passed, +# precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds +# bincraft and its dependencies alongside whatever this script installs). +# +# System dependencies come from uvr's vendored r-system-requirements rules, so +# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed. +# +# Environment: +# UVR_R_BIN R interpreter to install for; set by install-bincraft.R so +# the per-R-minor passes target their own R, not the primary +# R_VERSION fallback interpreter selector (/opt/R//bin/R) +# UVR_TARGET_LIB target library; defaults to R_LIBS_USER, then to the +# active R's .libPaths()[1] (which is where pak wrote) +# UVR_INSTALL_DIR where the uvr binary lands (default /usr/local/bin) + +set -eu + +# renovate: datasource=github-releases depName=nbafrank/uvr +UVR_PIN="v0.4.6" + +if [ "$#" -eq 0 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi + +# The build images keep R under /opt/R/ and off PATH. uvr resolves the +# interpreter via PATH and never downloads one unless `uvr r install` is run, so +# put the requested R first. +r_bin="${UVR_R_BIN:-}" +if [ -z "$r_bin" ] && [ -n "${R_VERSION:-}" ] && [ -x "/opt/R/${R_VERSION}/bin/R" ]; then + r_bin="/opt/R/${R_VERSION}/bin/R" +fi +if [ -n "$r_bin" ]; then + PATH="$(dirname "$r_bin"):$PATH" + export PATH +else + r_bin="$(command -v R)" +fi + +target_lib="${UVR_TARGET_LIB:-${R_LIBS_USER:-}}" +if [ -z "$target_lib" ]; then + target_lib="$("$r_bin" --no-echo --no-save -e 'cat(.libPaths()[1])')" +fi +if [ -z "$target_lib" ]; then + echo "error: could not determine a target library; set UVR_TARGET_LIB" >&2 + exit 2 +fi +mkdir -p "$target_lib" + +# Pin the manifest to the active R so the lockfile's R stays in step with the +# library's R sentinel. Without that, uvr can decide the library is ABI-stale +# and wipe it -- and this target is shared with bincraft. uvr only discovers R +# via PATH/R_HOME (it does not scan /opt/R), so the R put on PATH above is the +# only candidate this constraint can resolve to. +# shellcheck disable=SC2016 # $major/$minor are R expressions, not shell vars +r_full="$("$r_bin" --no-echo --no-save -e 'cat(paste(R.version$major, R.version$minor, sep = "."))')" + +install_dir="${UVR_INSTALL_DIR:-/usr/local/bin}" +uvr_bin="${install_dir}/uvr" +if [ ! -x "$uvr_bin" ]; then + echo "Bootstrapping uvr ${UVR_PIN} into ${install_dir}" + UVR_INSTALL_DIR="$install_dir" UVR_VERSION="$UVR_PIN" \ + sh -c 'curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | sh' +fi + +project_dir="${TMPDIR:-/tmp}/uvr-ci-$$" +rm -rf "$project_dir" +mkdir -p "$project_dir" +trap 'rm -rf "$project_dir"' EXIT +cd "$project_dir" + +"$uvr_bin" init --here --r-version "$r_full" +# --no-install resolves and locks only; retry because concurrent shards can +# expose short-lived DNS or CRAN-index failures and uvr rolls the manifest back +# cleanly after an unsuccessful resolution. +add_attempt=1 +while ! "$uvr_bin" add --no-install "$@"; do + if [ "$add_attempt" -ge 4 ]; then + echo "error: uvr add failed after ${add_attempt} attempts" >&2 + exit 1 + fi + add_delay=$((add_attempt * 10)) + echo "warning: uvr add attempt ${add_attempt} failed; retrying in ${add_delay}s" >&2 + sleep "$add_delay" + add_attempt=$((add_attempt + 1)) +done + +# TEMPORARY (drop once the images ship a uvr above v0.4.5): the sync below runs +# `apt-get install` for every resolved system dependency without refreshing the +# index first, and the ubuntu build images end their apt layers with +# `rm -rf /var/lib/apt/lists/*`. With no index apt cannot resolve a package that +# exists and is enabled, so `igraph needs: libglpk-dev` fails the whole build +# with `E: Unable to locate package libglpk-dev` on ubuntu 26.04. +# +# Fixed upstream in `e491b2e`, tagged one day after v0.4.5 (nbafrank/uvr#250), +# and the images pick it up via build-env-images#23 — but only after an image +# rebuild is triggered, which is why this runs here too. +# +# apt only: `apk add` fetches its index implicitly and dnf refreshes expired +# metadata on its own. Non-fatal, since a refresh failure still leaves whatever +# index is already there, and the install's own error is the more actionable one. +if command -v apt-get >/dev/null 2>&1; then + apt-get update -qq || echo "warning: apt-get update failed; continuing" >&2 +fi + +"$uvr_bin" sync --library "$target_lib" --install-system-deps diff --git a/local/validate-patches.R b/local/validate-patches.R new file mode 100644 index 0000000..b20aea1 --- /dev/null +++ b/local/validate-patches.R @@ -0,0 +1,82 @@ +#!/usr/bin/env Rscript +# Validate local/patches/registry.json: schema, referenced patch files, and +# ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. +# +# Defaults to local/patches/registry.json. To validate a candidate registry +# without touching the real one (e.g. from the patch proposer), set: +# PATCH_DIR directory patch-file paths resolve against (default local/patches) +# REGISTRY_FILE registry.json to validate (default /registry.json) + +dir <- Sys.getenv("PATCH_DIR", unset = "local/patches") +registry_file <- Sys.getenv( + "REGISTRY_FILE", + unset = file.path(dir, "registry.json") +) +if (!file.exists(registry_file)) { + cat("No registry.json found; nothing to validate.\n") + quit(status = 0L) +} + +or_q <- function(x) if (is.null(x)) "?" else x + +reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +required <- c("package", "versions", "platforms", "reason") +errs <- character(0L) + +for (i in seq_along(reg)) { + e <- reg[[i]] + missing <- setdiff(required, names(e)) + if (length(missing) > 0L) { + errs <- c( + errs, + sprintf( + "entry %d (%s): missing %s", + i, + if (is.null(e$package)) "?" else e$package, + toString(missing) + ) + ) + } + if (!is.null(e$patch)) { + p <- file.path(dir, e$patch) + if (!file.exists(p)) { + errs <- c( + errs, + sprintf("entry %d (%s): patch file '%s' missing", i, e$package, p) + ) + } + } +} + +# Ambiguous overlap: two entries for the same package with identical platforms +# and versions. +keys <- vapply( + reg, + function(e) { + sprintf( + "%s|%s|%s", + or_q(e$package), + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + or_q(e$versions) + ) + }, + character(1L) +) +dups <- keys[duplicated(keys)] +if (length(dups) > 0L) { + errs <- c( + errs, + sprintf("ambiguous duplicate entries: %s", toString(unique(dups))) + ) +} + +if (length(errs) > 0L) { + cat("Patch registry validation FAILED:\n") + cat(paste0(" - ", errs, "\n")) + quit(status = 1L) +} +cat(sprintf( + "Patch registry OK (%d %s).\n", + length(reg), + if (length(reg) == 1L) "entry" else "entries" +)) diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R new file mode 100644 index 0000000..e7ca776 --- /dev/null +++ b/local/weekly-missing-binaries-audit.R @@ -0,0 +1,474 @@ +options(error = function() { + cat("ERROR:", geterrmessage(), "\n", file = stdout()) + traceback(2) + q(status = 1) +}) + +suppressPackageStartupMessages(library(data.table)) +library(DBI, quietly = TRUE) +library(RPostgres, quietly = TRUE) +library(jsonlite, quietly = TRUE) +library(httr2, quietly = TRUE) + +# --------------------------------------------------------------------------- +# Environment / config +# --------------------------------------------------------------------------- +platform <- Sys.getenv("PLATFORM") +arch <- Sys.getenv("ARCH") + +if (nchar(platform) == 0) { + stop("PLATFORM env var is not set") +} +if (nchar(arch) == 0) { + stop("ARCH env var is not set") +} + +# Map platform names to S3 codenames +s3_codename <- gsub("-", "", platform) +s3_codename <- sub("^redhat", "rhel", s3_codename) +s3_codename <- sub("^ubuntu2204$", "jammy", s3_codename) +s3_codename <- sub("^ubuntu2404$", "noble", s3_codename) + +# Derive OS family for the Forgejo issue title +os_family <- if (grepl("^ubuntu", platform)) { + "Ubuntu" +} else if (grepl("^alpine", platform)) { + "Alpine" +} else if (grepl("^redhat", platform)) { + "Red Hat" +} else { + platform +} + +cat(sprintf( + "Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n", + platform, + arch, + s3_codename, + os_family +)) + +# --------------------------------------------------------------------------- +# 1. Query PostgreSQL for known build failures (before anything that uses curl, +# to avoid C++ pointer conflicts between curl and RPostgres/libpq) +# --------------------------------------------------------------------------- +cat("Connecting to PostgreSQL...\n") +con <- DBI::dbConnect( + RPostgres::Postgres(), + dbname = "build_metadata", + host = "r-binaries.devxy.io", + port = 15432, + user = "rpkgs", + password = Sys.getenv("PGPASS"), + sslmode = "require" +) + +errored_pkgs <- DBI::dbGetQuery( + con, + sprintf( + "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE AND platform = '%s' AND arch = '%s'", + platform, + arch + ) +) +DBI::dbDisconnect(con) + +errored_dt <- as.data.table(errored_pkgs) +if (nrow(errored_dt) > 0) { + setnames(errored_dt, c("Package", "Version")) + setkey(errored_dt, Package, Version) +} +cat(sprintf( + "Found %d known build failures for %s/%s\n", + nrow(errored_dt), + platform, + arch +)) + +# --------------------------------------------------------------------------- +# 2. CRAN release packages (uses curl internally - must come after PG) +# --------------------------------------------------------------------------- +cat("Fetching CRAN package database...\n") +cran_release <- tools::CRAN_package_db() +cran_dt <- data.table( + Package = as.character(cran_release$Package), + Version = as.character(cran_release$Version) +) + +# --------------------------------------------------------------------------- +# 3. Published binaries +# --------------------------------------------------------------------------- +# Read the slot's own index rather than listing the bucket. An object being +# present does not mean a binary was built: when a build fails, bincraft +# publishes the CRAN source tarball in its place so the package stays +# installable, and a bucket listing cannot tell the two apart. That is how +# amd64/alpine324 came to hold 13,547 CRAN sources that this audit never +# reported. bincraft stamps `Built` only on records it actually built, so the +# index answers the question a listing cannot. +# +# A slot last indexed by a bincraft that predates the source-fallback fix +# stamps `Built` on every record, including the fallbacks, so this reports +# exactly what it used to until that slot is re-indexed. +index_url <- sprintf( + "https://cran.rpkgs.com/%s/%s/latest/src/contrib/PACKAGES.gz", + arch, + s3_codename +) +cat(sprintf("Reading package index: %s\n", index_url)) + +index <- tryCatch( + { + con <- gzcon(url(index_url, open = "rb")) + on.exit(close(con), add = TRUE) + read.dcf(con, fields = c("Package", "Version", "Built")) + }, + error = function(e) { + cat(sprintf( + "WARNING: Could not read %s: %s\n", + index_url, + conditionMessage(e) + )) + NULL + } +) + +if (is.null(index) || nrow(index) == 0) { + binary_dt <- data.table(Package = character(0), Version = character(0)) +} else { + built <- !is.na(index[, "Built"]) + binary_dt <- data.table( + Package = as.character(index[built, "Package"]), + Version = as.character(index[built, "Version"]) + ) + cat(sprintf( + "Index holds %d records, %d of them built binaries (%d served as CRAN source)\n", + nrow(index), + sum(built), + sum(!built) + )) +} + +cat(sprintf( + "S3 contains %d binaries for %s/%s\n", + nrow(binary_dt), + arch, + s3_codename +)) + +# --------------------------------------------------------------------------- +# 4. Find missing packages (CRAN release version without a binary in S3) +# --------------------------------------------------------------------------- +setkey(cran_dt, Package, Version) +setkey(binary_dt, Package, Version) +missing_dt <- cran_dt[!binary_dt] +cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) + +# --------------------------------------------------------------------------- +# 5. Remove excluded packages +# --------------------------------------------------------------------------- +script_dir <- tryCatch( + dirname(normalizePath( + if (exists("ofile", envir = sys.frame(1), inherits = FALSE)) { + sys.frame(1)$ofile + } else { + commandArgs(trailingOnly = FALSE)[grepl( + "--file=", + commandArgs(trailingOnly = FALSE) + )] |> + sub("--file=", "", x = _) + }, + mustWork = FALSE + )), + error = function(e) "local" +) +excluded_path <- file.path(script_dir, "excluded-packages.json") + +excluded_pkgs <- data.table(package = character(0), reason = character(0)) +if (file.exists(excluded_path)) { + excluded_raw <- jsonlite::fromJSON(excluded_path) + excluded_pkgs <- as.data.table(excluded_raw) + cat(sprintf( + "Loaded %d excluded packages from %s\n", + nrow(excluded_pkgs), + excluded_path + )) + missing_dt <- missing_dt[!Package %in% excluded_pkgs$package] + cat(sprintf( + "%d packages remain after removing exclusions\n", + nrow(missing_dt) + )) +} else { + cat(sprintf( + "No excluded-packages.json found at %s -- skipping exclusion step\n", + excluded_path + )) +} + +# --------------------------------------------------------------------------- +# 6. Split into rebuildable vs known failures +# --------------------------------------------------------------------------- +if (nrow(errored_dt) > 0) { + known_failures_dt <- missing_dt[errored_dt, nomatch = 0] + rebuildable_dt <- missing_dt[!errored_dt] +} else { + known_failures_dt <- missing_dt[0] + rebuildable_dt <- missing_dt +} + +cat(sprintf( + "Rebuildable: %d | Known failures: %d\n", + nrow(rebuildable_dt), + nrow(known_failures_dt) +)) + +# --------------------------------------------------------------------------- +# 7. Write rebuildable package names to cache RDS +# --------------------------------------------------------------------------- +cache_dir <- "/mnt/cache/packages" +cache_file <- file.path( + cache_dir, + sprintf("weekly_rebuild_%s_%s.rds", platform, arch) +) + +if (dir.exists(cache_dir)) { + saveRDS(rebuildable_dt$Package, cache_file) + cat(sprintf( + "Wrote %d rebuildable packages to %s\n", + nrow(rebuildable_dt), + cache_file + )) +} else { + cat(sprintf("Cache dir %s does not exist -- skipping RDS write\n", cache_dir)) +} + +# --------------------------------------------------------------------------- +# 8. Update Forgejo issue +# --------------------------------------------------------------------------- +forgejo_token <- Sys.getenv("FORGEJO_TOKEN") +if (nchar(forgejo_token) == 0) { + cat("FORGEJO_TOKEN not set -- skipping issue update\n") +} else { + issue_title <- sprintf( + "Missing package binaries for latest version (%s)", + os_family + ) + + n_missing <- nrow(missing_dt) + n_rebuild <- nrow(rebuildable_dt) + + arch_lines <- sprintf( + "### %s (%d missing, %d to rebuild)", + arch, + n_missing, + n_rebuild + ) + if (nrow(rebuildable_dt) > 0) { + arch_lines <- c( + arch_lines, + paste0("- ", rebuildable_dt$Package, " (", rebuildable_dt$Version, ")") + ) + } else { + arch_lines <- c(arch_lines, "_None_") + } + if (nrow(known_failures_dt) > 0) { + arch_lines <- c( + arch_lines, + "", + "#### Known build failures", + paste0( + "- ", + known_failures_dt$Package, + " (", + known_failures_dt$Version, + ")" + ) + ) + } + + build_excluded_footer <- function() { + if (nrow(excluded_pkgs) == 0) { + return(character(0)) + } + entries <- paste( + paste0(excluded_pkgs$package, " (", excluded_pkgs$reason, ")"), + collapse = ", " + ) + c("---", "", "## Excluded packages", entries) + } + + forgejo_base <- "https://git.devxy.io/api/v1" + repo <- "devxy/build-cran-binaries" + + search_url <- sprintf( + "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", + forgejo_base, + repo, + utils::URLencode(issue_title, reserved = TRUE) + ) + search_resp <- httr2::request(search_url) |> + httr2::req_headers(Authorization = paste("token", forgejo_token)) |> + httr2::req_perform() + + existing_issues <- httr2::resp_body_json(search_resp, simplifyVector = FALSE) + + match_idx <- which( + sapply(existing_issues, function(x) x$title) == issue_title + ) + + today_str <- format(Sys.Date(), "%Y-%m-%d") + + if (length(match_idx) > 0) { + # ---- Update existing issue ---- + issue_number <- existing_issues[[match_idx[1]]]$number + old_body <- existing_issues[[match_idx[1]]]$body + if (is.null(old_body)) { + old_body <- "" + } + + lines <- strsplit(old_body, "\n", fixed = TRUE)[[1]] + + # Update timestamp + ts_idx <- which(grepl("^_Last updated:", lines)) + if (length(ts_idx) > 0) { + lines[ts_idx[1]] <- sprintf("_Last updated: %s_", today_str) + } else { + lines <- c(sprintf("_Last updated: %s_", today_str), "", lines) + } + + plat_header <- sprintf("## %s", platform) + + plat_idx <- which(lines == plat_header) + + if (length(plat_idx) == 0) { + # Platform section missing -- insert before --- footer + footer_idx <- which(lines == "---") + insert_at <- if (length(footer_idx) > 0) { + footer_idx[length(footer_idx)] + } else { + length(lines) + 1 + } + + new_plat_block <- c(sprintf("## %s", platform), "", arch_lines, "") + lines <- c( + lines[seq_len(insert_at - 1)], + new_plat_block, + lines[seq(insert_at, length(lines))] + ) + } else { + pi <- plat_idx[1] + + # End of platform section: next ## or --- at a higher level, or EOF + next_section <- which(grepl("^## |^---", lines) & seq_along(lines) > pi) + plat_end <- if (length(next_section) > 0) { + next_section[1] - 1 + } else { + length(lines) + } + + plat_lines <- lines[seq(pi, plat_end)] + + # Arch subsection headers within the platform block (### arm64 / ### amd64). + # Match by prefix: headers carry a " (N missing, M to rebuild)" suffix, so + # exact-equality matching never found the existing block and silently + # appended a duplicate on every run. Remove *all* blocks for this arch + # (collapsing any previously accumulated duplicates), then write one fresh + # block, so the issue holds a single current subsection per arch. + sub_hdr <- which(grepl("^### ", plat_lines)) + arch_re <- sprintf("^### %s( |$)", arch) + + if (length(sub_hdr) == 0) { + # No arch subsections yet -- append after the platform header/preamble. + new_plat_lines <- c(plat_lines, "", arch_lines) + } else { + preamble <- plat_lines[seq_len(sub_hdr[1] - 1)] + # Each subsection runs from its ### header to the line before the next + # ### header (#### known-failures stays inside its own block). + sub_end <- c(sub_hdr[-1] - 1, length(plat_lines)) + kept <- character(0) + for (k in seq_along(sub_hdr)) { + block <- plat_lines[seq(sub_hdr[k], sub_end[k])] + if (!grepl(arch_re, block[1])) { + kept <- c(kept, block) + } + } + new_plat_lines <- c(preamble, kept, "", arch_lines) + } + + tail_lines <- if (plat_end < length(lines)) { + lines[seq(plat_end + 1, length(lines))] + } else { + character(0) + } + lines <- c(lines[seq_len(pi - 1)], new_plat_lines, tail_lines) + } + + # Rebuild excluded footer + excl_hdr_idx <- which(lines == "## Excluded packages") + if (length(excl_hdr_idx) > 0) { + pre_dash <- which(lines == "---" & seq_along(lines) < excl_hdr_idx[1]) + remove_from <- if (length(pre_dash) > 0) { + pre_dash[length(pre_dash)] + } else { + excl_hdr_idx[1] + } + lines <- lines[seq_len(remove_from - 1)] + } + footer <- build_excluded_footer() + if (length(footer) > 0) { + lines <- c(lines, "", footer) + } + + new_body <- paste(lines, collapse = "\n") + + patch_url <- sprintf( + "%s/repos/%s/issues/%d", + forgejo_base, + repo, + issue_number + ) + httr2::request(patch_url) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list(body = new_body)) |> + httr2::req_method("PATCH") |> + httr2::req_perform() + + cat(sprintf("Updated Forgejo issue #%d: %s\n", issue_number, issue_title)) + } else { + # ---- Create new issue ---- + body_lines <- c( + sprintf("_Last updated: %s_", today_str), + "", + sprintf("## %s", platform), + "", + arch_lines + ) + footer <- build_excluded_footer() + if (length(footer) > 0) { + body_lines <- c(body_lines, "", footer) + } + + post_url <- sprintf("%s/repos/%s/issues", forgejo_base, repo) + create_resp <- httr2::request(post_url) |> + httr2::req_headers( + Authorization = paste("token", forgejo_token), + `Content-Type` = "application/json" + ) |> + httr2::req_body_json(list( + title = issue_title, + body = paste(body_lines, collapse = "\n") + )) |> + httr2::req_perform() + + new_issue <- httr2::resp_body_json(create_resp) + cat(sprintf( + "Created Forgejo issue #%d: %s\n", + new_issue$number, + issue_title + )) + } +} + +cat("Done.\n") diff --git a/plans/2026-06-30-package-patching-implementation.md b/plans/2026-06-30-package-patching-implementation.md new file mode 100644 index 0000000..3a743d7 --- /dev/null +++ b/plans/2026-06-30-package-patching-implementation.md @@ -0,0 +1,1477 @@ +# Package Patching Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let a curated registry of packages be "patched" (env/configure/Makevars overrides and/or source diffs) before `pak` installs them, including when they are transitive dependencies, so compiler-/OS-specific failures like `RcppParallel` stop cascading. + +**Architecture:** The mechanism lives in `bincraft`. Before `pak::local_install_deps()` runs, bincraft pre-builds each registry-matched package as a binary (reusing `pkgbuild::build(binary=TRUE)` + `cranlike::add_PACKAGES`), caches it, and serves it from a local `file://` repo prepended to `options("repos")` so `pak` installs the patched binary — direct or transitive — without recompiling. The curated registry (`registry.json` + diff files) lives in this repo and is passed in via a new `patches` argument. + +**Tech Stack:** R, `pak`, `pkgbuild`, `cranlike` (fork `pat-s/cranlike@s3`), `pkgsearch`, `withr`, `jsonlite`, `testthat` (3e), `mockery`. Two repos: `bincraft` at `/Users/pjs/git/codefloe.com/rpkgs/bincraftr`, and this repo `build-cran-binaries`. + +## Global Constraints + +- bincraft package name is `bincraft`; repo dir is `…/rpkgs/bincraftr`. Current version `4.2.2.9999`. +- Logging uses bincraft's wrappers only: `log_info`, `log_warn`, `log_success`, `log_debug`, `log_error`, `log_header` (never bare `message`/`cat` in package code). +- `cli`-style inline markup is allowed in log messages (e.g. `{.pkg %s}`, `{.path %s}`); curly braces in dynamic/error text must be escaped as already done in `install_helpers.R`. +- Tests: `testthat` 3rd edition, files at `tests/testthat/test-.R`, network/build tests guarded with `skip_on_cran()` / `skip_if_offline()`; end-to-end build tests guarded behind `skip_if_not(nzchar(Sys.getenv("BINCRAFT_PATCH_E2E")))`. +- New exported functions need roxygen with `@keywords internal` for non-user helpers; run `devtools::document()` after adding roxygen. +- Patched-binary cache dir default: `/mnt/cache/patched-binaries`. +- Platform tokens for matching = `c(, , )`, e.g. `ubuntu-2604` → `c("ubuntu-2604","ubuntu","amd64")`. +- Registry entry required fields: `package`, `versions`, `platforms`, `reason`. Optional: `env`, `configure_args`, `makevars`, `patch`. +- One sentence per line in prose/commit messages; do not hard-wrap at 80 columns. +- Use `fj -H codefloe.com` for any PR operations (Forgejo), not `gh`. + +--- + +## Phase A — bincraft mechanism + +All Phase A paths are relative to `/Users/pjs/git/codefloe.com/rpkgs/bincraftr`. + +### Task A0: Proof of mechanism — pak installs a patched binary from a prepended `file://` repo + +This de-risks the core assumption before building anything on top: that `pak` installs a binary from a local `file://` repo in preference to CRAN for an equal version, and does so without recompiling. If this fails, the contingency (documented in Step 4) is to serve patched *source* and rely on `pkgcache` build-caching — the rest of the plan changes only inside `build_patched_binary()`. + +**Files:** +- Create: `tools/verify-patch-mechanism.R` + +**Interfaces:** +- Produces: a runnable script proving `pak::pkg_install()` resolves a local patched binary over CRAN. No package API. + +- [ ] **Step 1: Write the verification script** + +```r +# tools/verify-patch-mechanism.R +# Proves pak installs a patched binary from a prepended file:// repo instead of +# CRAN's, without recompiling. Run inside a Linux build-env container: +# Rscript tools/verify-patch-mechanism.R +# Exits 0 on success, 1 on failure. + +pkg <- "glue" # small, pure-R CRAN package +sentinel <- "PatchMechanismProof" + +work <- tempfile("verify_") +repo <- file.path(work, "repo", "src", "contrib") +lib <- file.path(work, "lib") +dir.create(repo, recursive = TRUE) +dir.create(lib, recursive = TRUE) + +# 1. Download CRAN source for the current version. +ap <- available.packages(repos = "https://cloud.r-project.org") +ver <- ap[pkg, "Version"] +src <- file.path(work, sprintf("%s_%s.tar.gz", pkg, ver)) +download.file( + sprintf("https://cloud.r-project.org/src/contrib/%s_%s.tar.gz", pkg, ver), + src, mode = "wb" +) + +# 2. Unpack, inject a sentinel field into DESCRIPTION, build a binary. +untar(src, exdir = work) +desc <- file.path(work, pkg, "DESCRIPTION") +writeLines(c(readLines(desc), sprintf("%s: yes", sentinel)), desc) +pkgbuild::build( + file.path(work, pkg), binary = TRUE, vignettes = FALSE, + dest_path = repo, quiet = TRUE +) +built <- list.files(repo, pattern = sprintf("^%s_.*\\.tar\\.gz$", pkg), full.names = TRUE) +file.rename(built[1L], file.path(repo, sprintf("%s_%s.tar.gz", pkg, ver))) +cranlike::add_PACKAGES(sprintf("%s_%s.tar.gz", pkg, ver), repo) + +# 3. Install with the local repo prepended; assert our patched build won. +withr::with_options( + list(repos = c(patched = sprintf("file://%s", dirname(dirname(repo))), + CRAN = "https://cloud.r-project.org")), + pak::pkg_install(pkg, lib = lib, ask = FALSE, upgrade = FALSE) +) + +installed_desc <- file.path(lib, pkg, "DESCRIPTION") +ok <- file.exists(installed_desc) && + any(grepl(sentinel, readLines(installed_desc))) + +if (ok) { + cat("PROOF PASSED: pak installed the patched local binary.\n") + quit(status = 0L) +} else { + cat("PROOF FAILED: pak did not install the patched local binary.\n") + quit(status = 1L) +} +``` + +- [ ] **Step 2: Run the proof in a build-env container** + +Run (amd64 example; use any supported build-env image): + +```bash +just build-single ubuntu 2604 amd64 4.5.0 glue 1.0.0 1 || true # warms the env +docker run --rm -v "$PWD":/work -w /work reg.devxy.io/rpkgs/build-env-ubuntu:2604 \ + Rscript tools/verify-patch-mechanism.R +``` + +Expected: final line `PROOF PASSED: pak installed the patched local binary.` and exit status 0. + +- [ ] **Step 3: Commit** + +```bash +git add tools/verify-patch-mechanism.R +git commit -m "test(patches): prove pak installs a patched binary from a local file:// repo" +``` + +- [ ] **Step 4: Record the outcome / contingency** + +If the proof PASSED, proceed to Task A1 unchanged. +If it FAILED (pak recompiled or picked CRAN's), the mechanism switches to serving patched *source*: in Task A4 `build_patched_binary()` skips `pkgbuild::build()` and instead repackages the patched source tree with `pkgbuild::build(binary = FALSE)`; everything else (registry, matching, cache, repo prepend) is unchanged because `pak` build-caches the compiled result via `pkgcache`. Note the chosen path in the commit message and continue. + +--- + +### Task A1: Registry loading and normalization + +**Files:** +- Create: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Produces: `load_patch_registry(patches_dir)` → `list()` of normalized entries; each entry is a named list with `package`, `versions`, `platforms` (character vector), `env` (named list), `configure_args` (character), `makevars` (named list), `reason`, and `patch_path` (absolute path or `NULL`). `normalize_patch_entry(entry, patches_dir)` → one normalized entry; errors on missing required field or missing patch file. + +- [ ] **Step 1: Write the failing test** + +```r +# tests/testthat/test-patches.R +test_that("load_patch_registry parses and normalizes entries", { + dir <- withr::local_tempdir() + writeLines("--- a patch ---", file.path(dir, "fix.patch")) + jsonlite::write_json( + list(list( + package = "RcppParallel", versions = "*", + platforms = list("alpine", "ubuntu-2604"), + env = list(RCPP_PARALLEL_USE_TBB = "0"), + patch = "fix.patch", reason = "bundled TBB fails" + )), + file.path(dir, "registry.json"), auto_unbox = TRUE + ) + + reg <- load_patch_registry(dir) + + expect_length(reg, 1L) + expect_identical(reg[[1L]]$package, "RcppParallel") + expect_identical(reg[[1L]]$platforms, c("alpine", "ubuntu-2604")) + expect_identical(reg[[1L]]$env$RCPP_PARALLEL_USE_TBB, "0") + expect_identical(reg[[1L]]$configure_args, character(0L)) + expect_true(file.exists(reg[[1L]]$patch_path)) +}) + +test_that("load_patch_registry returns empty list when no registry", { + expect_identical(load_patch_registry(NULL), list()) + expect_identical(load_patch_registry(withr::local_tempdir()), list()) +}) + +test_that("normalize_patch_entry errors on missing required field", { + expect_error( + normalize_patch_entry(list(package = "x"), tempdir()), + "missing required field" + ) +}) + +test_that("normalize_patch_entry errors on missing patch file", { + expect_error( + normalize_patch_entry( + list(package = "x", versions = "*", platforms = "alpine", + reason = "r", patch = "nope.patch"), + tempdir() + ), + "does not exist" + ) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function load_patch_registry". + +- [ ] **Step 3: Write minimal implementation** + +```r +# R/patches.R + +#' Load and validate the patch registry +#' +#' Reads `registry.json` from `patches_dir` and returns normalized entries. +#' +#' @param patches_dir Directory containing `registry.json` and any patch files, +#' or `NULL` to disable patching. +#' @return A list of normalized patch entries (possibly empty). +#' @keywords internal +load_patch_registry <- function(patches_dir) { + if (is.null(patches_dir)) { + return(list()) + } + registry_file <- file.path(patches_dir, "registry.json") + if (!file.exists(registry_file)) { + log_warn(sprintf( + "Patch directory {.path %s} has no registry.json; patching disabled.", + patches_dir + )) + return(list()) + } + raw <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) + lapply(raw, normalize_patch_entry, patches_dir = patches_dir) +} + +#' Normalize and validate a single patch registry entry +#' +#' @param entry A list parsed from `registry.json`. +#' @param patches_dir Directory used to resolve a relative `patch` path. +#' @return The entry with defaults filled and `patch_path` resolved. +#' @keywords internal +normalize_patch_entry <- function(entry, patches_dir) { + required <- c("package", "versions", "platforms", "reason") + missing <- setdiff(required, names(entry)) + if (length(missing) > 0L) { + stop( + sprintf("Patch entry is missing required field(s): %s", toString(missing)), + call. = FALSE + ) + } + entry$platforms <- as.character(unlist(entry$platforms)) + entry$env <- if (is.null(entry$env)) list() else entry$env + entry$configure_args <- if (is.null(entry$configure_args)) { + character(0L) + } else { + as.character(unlist(entry$configure_args)) + } + entry$makevars <- if (is.null(entry$makevars)) list() else entry$makevars + if (!is.null(entry$patch)) { + patch_path <- file.path(patches_dir, entry$patch) + if (!file.exists(patch_path)) { + stop( + sprintf( + "Patch file '%s' for package '%s' does not exist.", + patch_path, entry$package + ), + call. = FALSE + ) + } + entry$patch_path <- patch_path + } else { + entry$patch_path <- NULL + } + entry +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): load and validate the patch registry" +``` + +--- + +### Task A2: Platform matching and version-constraint satisfaction + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: normalized entries from Task A1. +- Produces: `build_platform_tokens(platform, arch)` → character vector; `entry_matches_platform(entry, tokens)` → logical; `match_patch_entries(registry, platform, arch)` → filtered list; `version_satisfies(version, constraint)` → logical (constraint forms: `"*"` handled by caller, `"x.y.z"` exact, `">=x"`, `"<=x"`, `">x"`, `"=5.1.0")) + expect_false(version_satisfies("5.0.0", ">=5.1.0")) + expect_true(version_satisfies("5.1.11-2", "<=5.1.11-2")) + expect_false(version_satisfies("5.1.12", "<=5.1.11-2")) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function build_platform_tokens". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Build platform tokens for patch matching +#' @keywords internal +build_platform_tokens <- function(platform, arch) { + family <- sub("-.*$", "", platform) + unique(c(platform, family, arch)) +} + +#' Does a patch entry apply to the current platform tokens? +#' @keywords internal +entry_matches_platform <- function(entry, tokens) { + any(entry$platforms == "*") || + length(intersect(entry$platforms, tokens)) > 0L +} + +#' Filter registry entries applicable to the current build +#' @keywords internal +match_patch_entries <- function(registry, platform, arch) { + if (length(registry) == 0L) { + return(list()) + } + tokens <- build_platform_tokens(platform, arch) + Filter(function(e) entry_matches_platform(e, tokens), registry) +} + +#' Test whether a version satisfies a single constraint +#' +#' @param version A version string (CRAN style, may contain `-`). +#' @param constraint One of `"x.y.z"`, `"==x"`, `">=x"`, `"<=x"`, `">x"`, `"=|<=|==|>|<)?\\s*(.+)$", constraint) + )[[1L]] + op <- parts[2L] + target <- parts[3L] + v <- package_version(version) + t <- package_version(target) + if (op == "" || op == "==") { + return(v == t) + } + switch( + op, + ">=" = v >= t, + "<=" = v <= t, + ">" = v > t, + "<" = v < t, + FALSE + ) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): platform matching and version-constraint checks" +``` + +--- + +### Task A3: Cache key and version resolution + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: normalized entries. +- Produces: `patch_cache_key(entry, version, platform, arch, r_minor)` → string `"_____"`, where `hash12` covers `env`/`configure_args`/`makevars`/patch bytes; `resolve_patch_version(entry)` → latest CRAN version satisfying `entry$versions`, or `NA_character_`; `describe_patch(entry)` → short human label. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("patch_cache_key is stable and sensitive to env/patch changes", { + e1 <- list(package = "P", env = list(A = "1"), + configure_args = character(0L), makevars = list(), + patch_path = NULL) + e2 <- e1; e2$env <- list(A = "2") + + k1 <- patch_cache_key(e1, "1.0", "alpine-324", "amd64", "4.5") + expect_identical(k1, patch_cache_key(e1, "1.0", "alpine-324", "amd64", "4.5")) + expect_false(identical( + k1, patch_cache_key(e2, "1.0", "alpine-324", "amd64", "4.5") + )) + expect_match(k1, "^P_1.0_alpine-324_amd64_4.5_[0-9a-f]{12}$") +}) + +test_that("resolve_patch_version returns latest for wildcard, NA when unmet", { + local_mocked_bindings( + cran_package = function(pkg) list(Version = "5.1.12"), + .package = "pkgsearch" + ) + expect_identical( + resolve_patch_version(list(package = "RcppParallel", versions = "*")), + "5.1.12" + ) + expect_identical( + resolve_patch_version(list(package = "RcppParallel", versions = ">=9.0")), + NA_character_ + ) +}) + +test_that("describe_patch summarizes the active overrides", { + expect_match( + describe_patch(list(env = list(RCPP_PARALLEL_USE_TBB = "0"), + configure_args = character(0L), makevars = list(), + patch_path = NULL)), + "env: RCPP_PARALLEL_USE_TBB=0" + ) + expect_match( + describe_patch(list(env = list(), configure_args = character(0L), + makevars = list(), patch_path = "/x/fix.patch")), + "source patch" + ) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function patch_cache_key". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Compute the cache key for a patched binary +#' @keywords internal +patch_cache_key <- function(entry, version, platform, arch, r_minor) { + payload <- list( + env = entry$env, + configure_args = entry$configure_args, + makevars = entry$makevars, + patch = if (!is.null(entry$patch_path)) { + readBin(entry$patch_path, "raw", file.size(entry$patch_path)) + } else { + raw(0L) + } + ) + tmp <- tempfile() + on.exit(unlink(tmp), add = TRUE) + saveRDS(payload, tmp) + hash <- substr(unname(tools::md5sum(tmp)), 1L, 12L) + sprintf( + "%s_%s_%s_%s_%s_%s", + entry$package, version, platform, arch, r_minor, hash + ) +} + +#' Resolve the CRAN version to build for a patch entry +#' +#' Returns the latest CRAN version satisfying the entry's `versions` constraint, +#' or `NA_character_` when CRAN's latest does not satisfy it or lookup fails. +#' @keywords internal +resolve_patch_version <- function(entry) { + latest <- tryCatch( + pkgsearch::cran_package(entry$package)$Version, + error = function(e) NA_character_ + ) + if (is.na(latest)) { + return(NA_character_) + } + if (identical(entry$versions, "*") || version_satisfies(latest, entry$versions)) { + return(latest) + } + NA_character_ +} + +#' Short human label describing a patch entry's overrides +#' @keywords internal +describe_patch <- function(entry) { + bits <- character(0L) + if (length(entry$env) > 0L) { + bits <- c(bits, sprintf( + "env: %s", + paste( + names(entry$env), + unlist(entry$env), + sep = "=", collapse = "," + ) + )) + } + if (length(entry$configure_args) > 0L) { + bits <- c(bits, sprintf("configure: %s", toString(entry$configure_args))) + } + if (length(entry$makevars) > 0L) { + bits <- c(bits, "makevars") + } + if (!is.null(entry$patch_path)) { + bits <- c(bits, "source patch") + } + if (length(bits) == 0L) "no-op" else paste(bits, collapse = "; ") +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. (Note: `local_mocked_bindings` requires testthat >= 3.1.7; bincraft uses 3e.) + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): cache key, version resolution, and patch description" +``` + +--- + +### Task A4: Build a patched binary in isolation + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: a normalized entry, a resolved `version`, a `dest_dir`. +- Produces: `download_cran_source(package, version, dest_dir, cran)` → path or `NULL`; `apply_source_patch(patch_path, pkg_src)` → logical; `configure_args_to_build_args(configure_args)` → character; `build_patched_binary(entry, version, dest_dir)` → path to built binary tarball or `NULL`. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("configure_args_to_build_args formats configure args", { + expect_identical(configure_args_to_build_args(character(0L)), character(0L)) + expect_identical( + configure_args_to_build_args(c("--with-foo", "--no-bar")), + "--configure-args=--with-foo --no-bar" + ) +}) + +test_that("apply_source_patch returns FALSE when patch does not apply", { + src <- withr::local_tempdir() + writeLines("unrelated content", file.path(src, "file.txt")) + bad_patch <- tempfile(fileext = ".patch") + writeLines(c( + "--- a/missing.txt", "+++ b/missing.txt", + "@@ -1 +1 @@", "-nope", "+nope2" + ), bad_patch) + expect_false(apply_source_patch(bad_patch, src)) +}) + +test_that("build_patched_binary returns NULL when download fails", { + local_mocked_bindings(download_cran_source = function(...) NULL) + expect_null( + build_patched_binary( + list(package = "P", env = list(), configure_args = character(0L), + makevars = list(), patch_path = NULL), + "1.0", withr::local_tempdir() + ) + ) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function configure_args_to_build_args". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Download a CRAN source tarball for an exact version +#' @keywords internal +download_cran_source <- function( + package, + version, + dest_dir, + cran = "https://cloud.r-project.org" +) { + fname <- sprintf("%s_%s.tar.gz", package, version) + urls <- c( + sprintf("%s/src/contrib/%s", cran, fname), + sprintf("%s/src/contrib/Archive/%s/%s", cran, package, fname) + ) + dest <- file.path(dest_dir, fname) + for (u in urls) { + ok <- tryCatch( + { + utils::download.file(u, dest, mode = "wb", quiet = TRUE) + file.exists(dest) && file.size(dest) > 0L + }, + error = function(e) FALSE + ) + if (isTRUE(ok)) { + return(dest) + } + } + log_warn(sprintf( + "Could not download CRAN source for {.pkg %s} %s.", + package, version + )) + NULL +} + +#' Apply a unified diff to an unpacked source tree +#' +#' Uses `patch -p1 --forward` so an already-applied or non-applying patch fails +#' cleanly (returns FALSE) instead of corrupting the tree. +#' @keywords internal +apply_source_patch <- function(patch_path, pkg_src) { + status <- system2( + "patch", + args = c( + "-p1", "--forward", "--batch", + "-d", shQuote(pkg_src), + "-i", shQuote(patch_path) + ), + stdout = FALSE, stderr = FALSE + ) + identical(status, 0L) +} + +#' Format configure args for `pkgbuild::build(args = ...)` +#' @keywords internal +configure_args_to_build_args <- function(configure_args) { + if (length(configure_args) == 0L) { + return(character(0L)) + } + sprintf("--configure-args=%s", paste(configure_args, collapse = " ")) +} + +#' Build a patched binary for one registry entry, in isolation +#' +#' Downloads CRAN source for `version`, applies the source patch (if any), and +#' builds a binary with the entry's env / configure / Makevars overrides scoped +#' to this build only. Returns the built tarball path, or `NULL` on any failure. +#' @keywords internal +build_patched_binary <- function(entry, version, dest_dir) { + workdir <- tempfile("patch_build_") + dir.create(workdir, recursive = TRUE, showWarnings = FALSE) + on.exit(unlink(workdir, recursive = TRUE, force = TRUE), add = TRUE) + + src_tarball <- download_cran_source(entry$package, version, workdir) + if (is.null(src_tarball)) { + return(NULL) + } + + utils::untar(src_tarball, exdir = workdir) + pkg_src <- file.path(workdir, entry$package) + + if (!is.null(entry$patch_path)) { + if (!apply_source_patch(entry$patch_path, pkg_src)) { + log_warn(sprintf( + "Patch for {.pkg %s} %s did not apply cleanly; skipping patched build.", + entry$package, version + )) + return(NULL) + } + } + + build_env <- entry$env + if (length(entry$makevars) > 0L) { + mk <- tempfile(fileext = ".mk") + writeLines( + vapply( + names(entry$makevars), + function(k) sprintf("%s=%s", k, entry$makevars[[k]]), + character(1L) + ), + mk + ) + build_env$R_MAKEVARS_USER <- mk + } + + tryCatch( + withr::with_envvar(build_env, { + pkgbuild::build( + path = pkg_src, + binary = TRUE, + vignettes = FALSE, + dest_path = dest_dir, + args = configure_args_to_build_args(entry$configure_args), + quiet = TRUE + ) + }), + error = function(e) { + log_warn(sprintf( + "Isolated patched build of {.pkg %s} %s failed: %s", + entry$package, version, conditionMessage(e) + )) + NULL + } + ) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add R/patches.R tests/testthat/test-patches.R +git commit -m "feat(patches): build a patched binary in isolation from CRAN source" +``` + +--- + +### Task A5: Orchestrate the local patched repo (cache + index) + +**Files:** +- Modify: `R/patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: all helpers above. +- Produces: `prepare_patched_repo(patches_dir, platform, arch, r_minor, cache_dir, repo_dir)` → path to a `src/contrib`-style dir containing patched binaries + a `PACKAGES` index, or `NULL` when nothing matched/built. On a cache hit it copies the cached tarball into `repo_dir`; on a miss it builds, then writes the result into `cache_dir`. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("prepare_patched_repo returns NULL when no entries match", { + dir <- withr::local_tempdir() + jsonlite::write_json( + list(list(package = "A", versions = "*", platforms = list("redhat"), + reason = "r")), + file.path(dir, "registry.json"), auto_unbox = TRUE + ) + expect_null( + prepare_patched_repo(dir, "ubuntu-2604", "amd64", "4.5", + cache_dir = withr::local_tempdir(), + repo_dir = withr::local_tempdir()) + ) +}) + +test_that("prepare_patched_repo serves a cached binary and writes an index", { + dir <- withr::local_tempdir() + jsonlite::write_json( + list(list(package = "glue", versions = "*", platforms = list("*"), + env = list(A = "1"), reason = "r")), + file.path(dir, "registry.json"), auto_unbox = TRUE + ) + cache <- withr::local_tempdir() + repo <- withr::local_tempdir() + + local_mocked_bindings( + resolve_patch_version = function(entry) "1.0.0", + build_patched_binary = function(entry, version, dest_dir) { + f <- file.path(dest_dir, sprintf("%s_%s.tar.gz", entry$package, version)) + writeLines("fake binary", f) + f + } + ) + + out <- prepare_patched_repo(dir, "ubuntu-2604", "amd64", "4.5", + cache_dir = cache, repo_dir = repo) + + expect_identical(out, repo) + expect_true(file.exists(file.path(repo, "glue_1.0.0.tar.gz"))) + expect_true(file.exists(file.path(repo, "PACKAGES"))) + # The build result was cached under the key. + expect_length(list.files(cache, pattern = "^glue_1.0.0_.*\\.tar\\.gz$"), 1L) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL with "could not find function prepare_patched_repo". + +- [ ] **Step 3: Write minimal implementation (append to `R/patches.R`)** + +```r +#' Prepare a local repo of patched binaries for the current build +#' +#' For each registry entry matching the current platform, ensures a patched +#' binary is present in `repo_dir` (from `cache_dir` if available, else built +#' and then cached) and writes a `PACKAGES` index over them. +#' +#' @param patches_dir Directory with `registry.json`, or `NULL`. +#' @param platform Build platform, e.g. `"ubuntu-2604"`. +#' @param arch Build arch, e.g. `"amd64"`. +#' @param r_minor R `"major.minor"` string, e.g. `"4.5"`. +#' @param cache_dir Persistent cache for patched binaries. +#' @param repo_dir Directory to assemble the local repo in. +#' @return `repo_dir` if at least one patched binary was produced, else `NULL`. +#' @keywords internal +prepare_patched_repo <- function( + patches_dir, + platform, + arch, + r_minor, + cache_dir = file.path("/mnt", "cache", "patched-binaries"), + repo_dir = tempfile("patched_repo_") +) { + entries <- match_patch_entries( + load_patch_registry(patches_dir), platform, arch + ) + if (length(entries) == 0L) { + return(NULL) + } + + dir.create(repo_dir, recursive = TRUE, showWarnings = FALSE) + dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE) + + produced <- 0L + for (entry in entries) { + version <- resolve_patch_version(entry) + if (is.na(version)) { + log_warn(sprintf( + "No CRAN version of {.pkg %s} satisfies '%s'; patch skipped.", + entry$package, entry$versions + )) + next + } + + key <- patch_cache_key(entry, version, platform, arch, r_minor) + cached <- file.path(cache_dir, sprintf("%s.tar.gz", key)) + target <- file.path( + repo_dir, sprintf("%s_%s.tar.gz", entry$package, version) + ) + + if (file.exists(cached)) { + log_info(sprintf( + "Using cached patched binary for {.pkg %s} %s.", + entry$package, version + )) + file.copy(cached, target, overwrite = TRUE) + } else { + log_info(sprintf( + "Applying patch to {.pkg %s} %s [%s]: %s", + entry$package, version, describe_patch(entry), entry$reason + )) + built <- build_patched_binary(entry, version, repo_dir) + if (is.null(built)) { + next + } + if (!identical(normalizePath(built), normalizePath(target))) { + file.copy(built, target, overwrite = TRUE) + } + file.copy(target, cached, overwrite = TRUE) + } + produced <- produced + 1L + } + + if (produced == 0L) { + return(NULL) + } + cranlike::add_PACKAGES( + list.files(repo_dir, pattern = "\\.tar\\.gz$"), + repo_dir + ) + repo_dir +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 5: Update imports and document** + +Add to `DESCRIPTION` `Imports:` (alphabetical) `jsonlite` if not present, and ensure `pkgsearch`, `pkgbuild`, `cranlike`, `withr` are listed (they are). Then: + +```bash +Rscript -e 'devtools::document()' +git add R/patches.R tests/testthat/test-patches.R DESCRIPTION NAMESPACE +git commit -m "feat(patches): orchestrate local patched-binary repo with caching" +``` + +--- + +### Task A6: Wire patched repo into the pak install path + +**Files:** +- Modify: `R/install_helpers.R:333-389` (`run_pak_install_with_mutex`) +- Modify: `R/install-deps.R:23-75` (`install_pkg_sys_deps`) +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Consumes: `prepare_patched_repo()`. +- Produces: `run_pak_install_with_mutex(local_clone_dir_single, env_vars, patched_repo = NULL)` — prepends `file://` to `options("repos")` for the install; `install_pkg_sys_deps(package_name, tag, local_clone_dir, platform, aggressive_cleanup = FALSE, patches = NULL, arch = NULL)` — builds the patched repo before installing. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("run_pak_install_with_mutex prepends the patched repo to repos", { + seen <- NULL + local_mocked_bindings( + acquire_pak_mutex = function(...) tempfile(), + release_pak_mutex = function(...) invisible(NULL), + retry_with_backoff = function(func, ...) func() + ) + local_mocked_bindings( + local_install_deps = function(...) { + seen <<- getOption("repos") + invisible(TRUE) + }, + .package = "pak" + ) + + run_pak_install_with_mutex( + tempfile(), list(), patched_repo = "/tmp/patched" + ) + + expect_true(any(grepl("file:///tmp/patched", seen))) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL — `patched_repo` argument not yet accepted / repos not prepended. + +- [ ] **Step 3: Edit `run_pak_install_with_mutex` in `R/install_helpers.R`** + +Change the signature line: + +```r +run_pak_install_with_mutex <- function(local_clone_dir_single, env_vars) { +``` + +to: + +```r +run_pak_install_with_mutex <- function( + local_clone_dir_single, + env_vars, + patched_repo = NULL +) { +``` + +Replace the inner `retry_with_backoff(...)` block (the one wrapping `pak::local_install_deps`) with: + +```r + retry_with_backoff(function() { + withr::with_envvar(env_vars, { + repos <- getOption("repos") + if (!is.null(patched_repo)) { + repos <- c( + patched = sprintf("file://%s", patched_repo), + repos + ) + } + withr::with_options(list(repos = repos), { + # Default to non-verbose (suppressed messages) + suppressMessages(pak::local_install_deps(sprintf( + "%s", + local_clone_dir_single + ))) + }) + }) + }) +``` + +- [ ] **Step 4: Edit `install_pkg_sys_deps` in `R/install-deps.R`** + +Change the signature to add `patches` and `arch`: + +```r +install_pkg_sys_deps <- function( + package_name, + tag, + local_clone_dir, + platform = platform, + aggressive_cleanup = FALSE, + patches = NULL, + arch = NULL +) { +``` + +Immediately before the `run_pak_install_with_mutex(...)` call, insert: + +```r + # Build a local repo of patched binaries (if any apply) and serve it to pak. + r_minor <- paste( + R.version$major, + strsplit(R.version$minor, ".", fixed = TRUE)[[1L]][1L], + sep = "." + ) + patched_repo <- tryCatch( + prepare_patched_repo(patches, platform, arch, r_minor), + error = function(e) { + log_warn(sprintf("Patch preparation failed: %s", conditionMessage(e))) + NULL + } + ) +``` + +and change the call from: + +```r + run_pak_install_with_mutex( + local_clone_dir_single, + env_vars + ) +``` + +to: + +```r + run_pak_install_with_mutex( + local_clone_dir_single, + env_vars, + patched_repo = patched_repo + ) +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add R/install_helpers.R R/install-deps.R tests/testthat/test-patches.R +git commit -m "feat(patches): serve patched binaries to pak during dep install" +``` + +--- + +### Task A7: Thread `patches` through the public build API + +**Files:** +- Modify: `R/build_binaries.R` (`build_binary_package`, `execute_package_builds`, `build_single_tag`, `handle_system_dependencies`) +- Create: `man-roxygen/param-patches.R` +- Test: `tests/testthat/test-patches.R` + +**Interfaces:** +- Produces: `build_binary_package(..., patches = NULL)` and the internal chain each carry `patches` down to `install_pkg_sys_deps()`. `handle_system_dependencies(..., patches = NULL)` passes `patches` and `arch` through. + +- [ ] **Step 1: Write the failing test** + +```r +test_that("handle_system_dependencies forwards patches and arch", { + captured <- list() + local_mocked_bindings( + install_pkg_sys_deps = function(package_name, tag, local_clone_dir_single, + platform, patches = NULL, arch = NULL) { + captured <<- list(patches = patches, arch = arch) + invisible(TRUE) + } + ) + handle_system_dependencies( + "RcppParallel", "5.1.11-2", "ubuntu-2604", tempfile(), "amd64", + NULL, NULL, NULL, NULL, NULL, NULL, NULL, + patches = "local/patches" + ) + expect_identical(captured$patches, "local/patches") + expect_identical(captured$arch, "amd64") +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")'` +Expected: FAIL — `handle_system_dependencies` has no `patches` argument. + +- [ ] **Step 3: Create the roxygen template** + +```r +# man-roxygen/param-patches.R +#' @param patches Optional path to a patch registry directory containing a +#' `registry.json` (and any referenced diff files). When set, matching +#' packages are pre-built as patched binaries and served to `pak` during +#' dependency installation. Defaults to `NULL` (no patching). +``` + +- [ ] **Step 4: Edit the four functions in `R/build_binaries.R`** + +In `build_single_tag()`'s call to `handle_system_dependencies(...)`, add `patches = patches` as the final argument, and add `patches = NULL` to `build_single_tag`'s own signature plus `#' @template param-patches` to its roxygen block. + +Change `handle_system_dependencies` signature to end with `metadata_db_sslmode,` then add `patches = NULL`, and change its inner `install_pkg_sys_deps(...)` call from: + +```r + install_pkg_sys_deps( + package_name, + tag, + local_clone_dir_single, + platform + ) +``` + +to: + +```r + install_pkg_sys_deps( + package_name, + tag, + local_clone_dir_single, + platform, + patches = patches, + arch = arch + ) +``` + +In `execute_package_builds()`, add `patches = NULL` to the signature and pass `patches = patches` into its `build_single_tag(...)` call inside `worker_function`. + +In `build_binary_package()`, add `patches = NULL` to the signature (after `s3_package_cache`), add `#' @template param-patches` to its roxygen, and pass `patches = patches` into the `execute_package_builds(...)` call. + +- [ ] **Step 5: Document, test, and run package check** + +Run: + +```bash +Rscript -e 'devtools::document()' +Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")' +``` + +Expected: PASS for the new test; `man/build_binary_package.Rd` etc. regenerated. + +- [ ] **Step 6: Commit** + +```bash +git add R/build_binaries.R man-roxygen/param-patches.R man NAMESPACE tests/testthat/test-patches.R +git commit -m "feat(patches): thread patches argument through build_binary_package" +``` + +--- + +### Task A8: End-to-end patch test (guarded) and version bump + +**Files:** +- Modify: `tests/testthat/test-patches.R` +- Modify: `DESCRIPTION` (version), `NEWS.md` + +**Interfaces:** +- Produces: a guarded e2e test proving a dependent package builds when its failing dependency is patched. + +- [ ] **Step 1: Add the guarded e2e test** + +```r +test_that("a patched dependency unblocks a dependent build (e2e)", { + skip_if_not(nzchar(Sys.getenv("BINCRAFT_PATCH_E2E"))) + skip_if_offline() + + patches_dir <- withr::local_tempdir() + jsonlite::write_json( + list(list( + package = "RcppParallel", versions = "*", platforms = list("*"), + env = list(RCPP_PARALLEL_USE_TBB = "0"), + reason = "bundled TBB fails on this toolchain" + )), + file.path(patches_dir, "registry.json"), auto_unbox = TRUE + ) + + out <- withr::local_tempdir() + result <- build_binary_package( + "rts2", tag = "latest", local_output_dir_root = out, + upload = FALSE, archive = FALSE, patches = patches_dir + ) + expect_true(isTRUE(result) || identical(result, "skipped")) +}) +``` + +- [ ] **Step 2: Run the e2e test in a container** + +Run: + +```bash +docker run --rm -e BINCRAFT_PATCH_E2E=1 -v "$PWD":/work -w /work \ + reg.devxy.io/rpkgs/build-env-ubuntu:2604 \ + Rscript -e 'devtools::load_all("."); testthat::test_file("tests/testthat/test-patches.R")' +``` + +Expected: the e2e test runs (not skipped) and PASSES; build log shows `Applying patch to RcppParallel`. + +- [ ] **Step 3: Bump version and changelog** + +In `DESCRIPTION`, bump `Version:` to `4.3.0.9999`. Prepend to `NEWS.md`: + +```markdown +# bincraft 4.3.0 + +* `build_binary_package()` gains a `patches` argument: a registry of + per-package env / configure / Makevars overrides and source diffs that are + pre-built into patched binaries and served to pak, fixing compiler- and + OS-specific failures (e.g. RcppParallel) including for transitive deps. +``` + +- [ ] **Step 4: Commit** + +```bash +git add tests/testthat/test-patches.R DESCRIPTION NEWS.md +git commit -m "test(patches): guarded end-to-end test; bump to 4.3.0.9999" +``` + +--- + +## Phase B — build-cran-binaries registry and wiring + +All Phase B paths are relative to this repo (`build-cran-binaries`). Phase B depends on a bincraft build that includes Phase A (install the updated bincraft in the build containers, or bump the pinned version in `.crow/build-all-versions-install-deps.yaml`). + +### Task B1: Create the patch registry with the RcppParallel entry + +**Files:** +- Create: `local/patches/registry.json` +- Create: `local/patches/README.md` + +**Interfaces:** +- Produces: the curated registry consumed by bincraft's `patches` argument. + +- [ ] **Step 1: Write the registry** + +```json +[ + { + "package": "RcppParallel", + "versions": "*", + "platforms": ["alpine", "ubuntu-2604"], + "env": { "RCPP_PARALLEL_USE_TBB": "0" }, + "configure_args": [], + "makevars": {}, + "patch": null, + "reason": "bundled Intel TBB fails to build on musl and on newer toolchains (e.g. g++ 15 on ubuntu-2604); disabling TBB falls back to TinyThread" + } +] +``` + +- [ ] **Step 2: Write the README** + +`local/patches/README.md` documents the schema (copy the field table from `specs/2026-06-30-package-patching-design.md`), how to add an entry, and that source diffs go in `local/patches//.patch` referenced by the `patch` field. + +- [ ] **Step 3: Verify it parses** + +Run: `Rscript -e 'x <- jsonlite::fromJSON("local/patches/registry.json", simplifyVector = FALSE); stopifnot(length(x) == 1L, x[[1]]$package == "RcppParallel"); cat("ok\n")'` +Expected: `ok`. + +- [ ] **Step 4: Commit** + +```bash +git add local/patches/registry.json local/patches/README.md +git commit -m "feat(patches): add patch registry with RcppParallel TBB workaround" +``` + +--- + +### Task B2: Registry validator script + +**Files:** +- Create: `local/validate-patches.R` + +**Interfaces:** +- Consumes: `local/patches/registry.json`. +- Produces: a script that exits non-zero on schema violations, missing patch files, or ambiguous overlapping entries. + +- [ ] **Step 1: Write the validator** + +```r +#!/usr/bin/env Rscript +# Validate local/patches/registry.json: schema, referenced patch files, and +# ambiguous overlaps. Exits 1 on any problem. Used by pre-commit and CI. + +dir <- "local/patches" +registry_file <- file.path(dir, "registry.json") +if (!file.exists(registry_file)) { + cat("No registry.json found; nothing to validate.\n") + quit(status = 0L) +} + +reg <- jsonlite::fromJSON(registry_file, simplifyVector = FALSE) +required <- c("package", "versions", "platforms", "reason") +errs <- character(0L) + +for (i in seq_along(reg)) { + e <- reg[[i]] + missing <- setdiff(required, names(e)) + if (length(missing) > 0L) { + errs <- c(errs, sprintf( + "entry %d (%s): missing %s", i, + if (is.null(e$package)) "?" else e$package, toString(missing) + )) + } + if (!is.null(e$patch)) { + p <- file.path(dir, e$patch) + if (!file.exists(p)) { + errs <- c(errs, sprintf("entry %d (%s): patch file '%s' missing", + i, e$package, p)) + } + } +} + +# Ambiguous overlap: two entries for the same package with identical platforms +# and versions. +keys <- vapply(reg, function(e) { + sprintf("%s|%s|%s", e$package, + paste(sort(as.character(unlist(e$platforms))), collapse = ","), + e$versions) +}, character(1L)) +dups <- keys[duplicated(keys)] +if (length(dups) > 0L) { + errs <- c(errs, sprintf("ambiguous duplicate entries: %s", toString(unique(dups)))) +} + +if (length(errs) > 0L) { + cat("Patch registry validation FAILED:\n") + cat(paste0(" - ", errs, "\n")) + quit(status = 1L) +} +cat(sprintf("Patch registry OK (%d entrie(s)).\n", length(reg))) +``` + +- [ ] **Step 2: Run it (expect success on the B1 registry)** + +Run: `Rscript local/validate-patches.R` +Expected: `Patch registry OK (1 entrie(s)).` and exit 0. + +- [ ] **Step 3: Run it against a broken registry (expect failure)** + +Run: + +```bash +cp local/patches/registry.json /tmp/reg.bak +Rscript -e 'writeLines("[{\"package\":\"X\"}]", "local/patches/registry.json")' +Rscript local/validate-patches.R; echo "exit=$?" +cp /tmp/reg.bak local/patches/registry.json +``` + +Expected: prints `validation FAILED` with a missing-field message and `exit=1`. + +- [ ] **Step 4: Commit** + +```bash +git add local/validate-patches.R +git commit -m "feat(patches): add registry validator script" +``` + +--- + +### Task B3: Hook the validator into pre-commit + +**Files:** +- Modify: `.pre-commit-config.yaml` + +**Interfaces:** +- Produces: a local hook that runs `local/validate-patches.R` when the registry or patch files change. + +- [ ] **Step 1: Add the hook** + +Add a `repo: local` hook entry to `.pre-commit-config.yaml`: + +```yaml + - repo: local + hooks: + - id: validate-patches + name: validate patch registry + entry: Rscript local/validate-patches.R + language: system + files: ^local/patches/ + pass_filenames: false +``` + +- [ ] **Step 2: Verify the hook runs** + +Run: `pre-commit run validate-patches --all-files` +Expected: hook passes (`Patch registry OK`). + +- [ ] **Step 3: Commit** + +```bash +git add .pre-commit-config.yaml +git commit -m "ci(patches): validate patch registry in pre-commit" +``` + +--- + +### Task B4: Pass `patches` through the build entry points + +**Files:** +- Modify: `local/build-one.R:99-119` (the `build_binary_package` call) +- Modify: `local/build-all.R:125-145` (the `build_binary_package` call) + +**Interfaces:** +- Consumes: the new bincraft `patches` argument (Phase A) and `local/patches/`. +- Produces: both entry points pass `patches = "local/patches"`. + +- [ ] **Step 1: Edit `local/build-one.R`** + +In the `bincraft::build_binary_package(` call, add as a new argument (e.g. after `archive = TRUE,`): + +```r + patches = "local/patches", +``` + +- [ ] **Step 2: Edit `local/build-all.R`** + +In the `bincraft::build_binary_package(` call, add: + +```r + patches = "local/patches", +``` + +- [ ] **Step 3: Verify the scripts still parse** + +Run: `Rscript -e 'invisible(parse("local/build-one.R")); invisible(parse("local/build-all.R")); cat("parse ok\n")'` +Expected: `parse ok`. + +- [ ] **Step 4: Commit** + +```bash +git add local/build-one.R local/build-all.R +git commit -m "feat(patches): pass patch registry to bincraft build calls" +``` + +--- + +### Task B5: Document the feature in the README + +**Files:** +- Modify: `README.md` + +**Interfaces:** +- Produces: a short "Patching packages" section explaining the registry and linking the design spec. + +- [ ] **Step 1: Add a README section** + +Add a `## Patching packages` section after the "Build Process" section describing: why patching exists (compiler/OS-specific failures cascading via shared deps like RcppParallel), where the registry lives (`local/patches/registry.json`), the two tiers (env/configure/Makevars overrides vs source diffs), and that bincraft pre-builds patched binaries served to pak. Link `specs/2026-06-30-package-patching-design.md`. + +- [ ] **Step 2: Commit** + +```bash +git add README.md +git commit -m "docs(patches): document the package patching workflow" +``` + +--- + +## Self-Review + +**Spec coverage:** +- Registry in this repo, passed to bincraft → Task A7 (`patches` arg), B1 (registry), B4 (wiring). ✓ +- Mechanism in bincraft (resolve → pre-build → prepend repo → install) → A4–A6. ✓ +- Both tiers, env-first → A4 (`env`, `configure_args`, `makevars`, `patch` all applied; env is the cheap default and the RcppParallel entry uses only env). ✓ +- Pre-built patched binary served from a prepended local repo → A0 proof + A5/A6. ✓ +- Cache keyed by pkg/version/platform/arch/rminor/patchhash → A3 (`patch_cache_key`) + A5 (cache use). ✓ +- S3 upload of patched binaries → **partial**: A5 caches locally only. Cross-machine S3 reuse is deferred (see note below) to keep the first cut shippable; local `/mnt/cache` reuse already removes per-dependent rebuilds within a container. Recorded as out-of-first-cut, not dropped. +- Error handling (diff fails → skip+warn; build fails → skip+warn; no match → skip; ambiguous overlap → validation error) → A4 (`apply_source_patch` FALSE path, build tryCatch), A5 (`resolve_patch_version` NA path), B2 (overlap validation). ✓ +- Observability (log line per applied patch) → A5 (`log_info` with `describe_patch`). DB metadata recording of applied patches is deferred with S3 (same note). +- Testing (registry parse/match, cache key, integration, failure path) → A1–A5 unit tests, A8 e2e, A4 patch-fail test. ✓ + +**Deferred from spec (call out to user):** S3 upload/reuse of patched binaries and recording applied patches in the Postgres build-metadata row. The local `/mnt/cache/patched-binaries` cache already prevents repeated rebuilds within a container; S3 reuse across CI jobs is a follow-up. If you want it in the first cut, add a Task A5b (upload `cached` tarball to a `…/patched/` S3 slot and check there before building) and a metadata column — say so and I'll insert them. + +**Placeholder scan:** No TBD/TODO; every code step shows complete code. README/registry-README prose steps describe exact content to write (acceptable for docs). + +**Type consistency:** `prepare_patched_repo` signature is identical across A5 (definition) and A6 (call site, via defaults). `patch_cache_key(entry, version, platform, arch, r_minor)` argument order matches between A3 and A5. `run_pak_install_with_mutex(..., patched_repo)` matches between A6 definition and the A6 test. `handle_system_dependencies(..., patches)` matches between A7 edit and A7 test. `build_patched_binary(entry, version, dest_dir)` matches A4 and A5. + +**Open risk:** Task A0 gates the whole approach; if it fails, the documented source-serving contingency keeps every later task valid with a localized change in A4. diff --git a/plans/2026-08-07-per-minor-edge-routing.md b/plans/2026-08-07-per-minor-edge-routing.md new file mode 100644 index 0000000..3c3e320 --- /dev/null +++ b/plans/2026-08-07-per-minor-edge-routing.md @@ -0,0 +1,166 @@ +# Per-R-minor edge routing implementation plan + +Spec: `specs/2026-08-07-per-minor-edge-routing-design.md` + +**Goal:** let a stock `install.packages()` see the per-minor packages by routing `PACKAGES*` requests to `…/src/contrib//`, where `bincraft` publishes a union index. + +**Architecture:** the union is built in `bincraft`; the edge script only redirects index requests, gated on a `UNION_SLOTS` script variable; the script lives in this repo and is applied by OpenTofu. + +**Tech stack:** Deno / TypeScript (Bunny Edge Scripting, SDK 0.12), OpenTofu with `BunnyWay/bunnynet` 0.17, R (bincraft). + +## Global constraints + +- Redirect only `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds`; never a tarball, because the union index already carries the correct tarball URL for both classes of package. +- Every redirect carries `Cache-Control: no-store`; redirect targets stay UA-independent. +- `UNION_SLOTS` is empty by default, so deploying the script is a no-op until a slot is backfilled. +- A slot is `/`, e.g. `amd64/alpine324`. +- Verified prerequisites: `PACKAGES*` is served `cdn-cache: BYPASS`, so the script sees every index request; `Deno.env.get()` reads script variables; the SDK local server listens on `127.0.0.1:8080`. + +--- + +## Task 1: Edge script and its test matrix + +**Files:** + +- Create: `edge/rpkgs-router.ts` +- Create: `edge/rpkgs-router.test.ts` +- Modify: `justfile` (add `edge-test`) + +**Produces:** a single-file script deployable as `bunnynet_compute_script.content`, reading `UNION_SLOTS` from the environment. + +- [ ] **Step 1: write the test matrix first** + +`edge/rpkgs-router.test.ts` spawns `deno run -A edge/rpkgs-router.ts` with `UNION_SLOTS=amd64/alpine324`, waits for `127.0.0.1:8080`, and issues requests with `redirect: "manual"`. + +Cases, asserted on the `location` header (or its absence): + +| # | path | User-Agent | expectation | +| --- | ------------------------------------------------------ | --------------------------------------------- | ------------------------------------------------------------ | +| 1 | `/amd64/alpine324/latest/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)` | 302 → `…/src/contrib/4.5/PACKAGES.gz` | +| 2 | same | `R (4.6.0 …)` | 302 → `…/src/contrib/4.6/PACKAGES.gz` | +| 3 | same, but slot `amd64/noble` | `R (4.5.3 …)` | no redirect (slot not in `UNION_SLOTS`) | +| 4 | `…/src/contrib/curl_7.1.0.tar.gz` | `R (4.5.3 …)` | no redirect | +| 5 | `…/src/contrib/4.5/PACKAGES.gz` | `R (4.5.3 …)` | no redirect (loop guard) | +| 6 | `…/src/contrib/PACKAGES.gz` | `curl/8.0` | no redirect (no R minor) | +| 7 | `/src/contrib/PACKAGES.gz` | alpine UA with `Alpine Linux … 3.24` | 302 → `/amd64/alpine324/latest/src/contrib/4.5/PACKAGES.gz` | +| 8 | `/src/contrib/PACKAGES.gz` | `R (4.5.3 x86_64-pc-linux-musl …)`, no distro | 302 → `cran.r-project.org`, **not** a `linux-musl` slot | +| 9 | `/src/contrib/foo_1.0.tar.gz` | `R (4.5.1 aarch64-apple-darwin20 …)` | 302 → `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | +| 10 | `/bin/macosx/big-sur-arm64/contrib/4.5/foo_1.0.tar.gz` | any | 302 → `cran.r-project.org` | +| 11 | any redirect above | — | `cache-control: no-store` | + +- [ ] **Step 2: run the tests and watch them fail** + +`just edge-test` → every case fails, because `edge/rpkgs-router.ts` does not exist. + +- [ ] **Step 3: write `edge/rpkgs-router.ts`** + +Order of evaluation in `onOriginRequest`: + +1. normalise `//` runs in the path +2. darwin `/src/contrib/*` → `/bin/macosx//contrib//` +3. `/bin/macosx/**` → CRAN +4. `/{arch}/{os}/latest/src/contrib/`: pass through if `rest` already starts with `/`, or is not an index file, or the slot is not in `UNION_SLOTS`, or the UA has no R minor; otherwise redirect into `/` +5. `/`, `/src/contrib`, `/src/contrib/**`: resolve arch+os from the UA, redirect to CRAN when the distro is unidentifiable, otherwise redirect to the qualified path, adding `/` under the same index-file rule +6. anything else: pass through + +The R minor comes from either `R/4.5.3` or `R (4.5.3 …)`, so a stock UA is enough. The `linux-gnu` / `linux-musl` fallback in `parseUserAgent` is deleted: those are not slot names. + +- [ ] **Step 4: run the tests until they pass** + +`just edge-test` + +- [ ] **Step 5: commit** + +```bash +git add edge/rpkgs-router.ts edge/rpkgs-router.test.ts justfile +git commit -m "feat(edge): route PACKAGES requests to the per-R-minor slot" +``` + +--- + +## Task 2: Manage the script from OpenTofu + +**Files:** + +- Modify: `cdn.tf` + +**Consumes:** `edge/rpkgs-router.ts` from Task 1. + +- [ ] **Step 1: add the resources** + +```terraform +resource "bunnynet_compute_script" "rpkgs_router" { + type = "middleware" + name = "rpkgs-router" + content = file("${path.module}/edge/rpkgs-router.ts") +} + +resource "bunnynet_compute_script_variable" "rpkgs_router_union_slots" { + script = bunnynet_compute_script.rpkgs_router.id + name = "UNION_SLOTS" + default_value = "" + required = false +} +``` + +and replace `middleware_script = 29277` with `middleware_script = bunnynet_compute_script.rpkgs_router.id`. + +- [ ] **Step 2: validate** + +`tofu init -backend=false && tofu validate` + +- [ ] **Step 3: import the existing script (needs `BUNNYNET_API_KEY`)** + +```bash +tofu import bunnynet_compute_script.rpkgs_router 29277 +tofu plan +``` + +The plan must show an in-place `content` update and **no** replacement of the pull zone. A replacement means the import did not take. + +- [ ] **Step 4: commit** + +```bash +git add cdn.tf +git commit -m "feat(cdn): manage the edge middleware script from this repo" +``` + +--- + +## Task 3: Union index writer in bincraft + +**Files (repo `codefloe.com/rpkgs/bincraft`):** + +- Modify: `R/package_index.R` +- Test: `tests/testthat/test-package_index.R` + +**Produces:** `write_union_index(flat_records, minor_records)` returning the merged records, called from `upload_package_index()` when `r_minor` is set. + +- [ ] **Step 1: write the failing tests** + +- a package present in both slots keeps the per-minor record, with `Path = "4.5"` +- a package only in the flat slot survives with no `Path` +- a package only in the per-minor slot survives with `Path = "4.5"` +- a union smaller than the flat input raises an error rather than returning + +- [ ] **Step 2: run them and watch them fail** + +`Rscript -e 'testthat::test_file("tests/testthat/test-package_index.R")'` + +- [ ] **Step 3: implement `write_union_index()` and call it from `upload_package_index()`** + +After `update_PACKAGES()` has written the per-minor index, read the flat slot's `PACKAGES.rds`, set `Path = ` on the per-minor records, drop the flat records for packages the per-minor slot already has, and rewrite `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` in the per-minor slot. + +- [ ] **Step 4: run the tests until they pass** + +- [ ] **Step 5: commit and open the PR against bincraft** + +--- + +## Task 4: Roll out slot by slot + +- [ ] Re-index one slot (`amd64/alpine324`, R 4.5) and confirm the union index lists both `curl` (per-minor, `Path: 4.5`) and `jsonlite` (flat, no `Path`). +- [ ] Set `UNION_SLOTS = "amd64/alpine324"` and confirm in `reg.devxy.io/r/r-alpine:4.5-3.24` that `available.packages()` returns the union count and `"curl" %in% rownames(...)`. +- [ ] Add `arm64/alpine324`, then the remaining slots. + +`install.packages("curl")` will still fail to build on `alpine324` until that slot's source tarballs are replaced with real binaries. That is tracked separately. diff --git a/provider.tf b/provider.tf index 768220b..d4f2564 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.7" + version = "~> 0.18" } } } diff --git a/renovate.json b/renovate.json index 36c06ff..7b2c3b3 100644 --- a/renovate.json +++ b/renovate.json @@ -1,9 +1,31 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "local>devxy/renovate-config" - ], + "extends": ["local>devxy/renovate-config"], "ignorePaths": [ - "docker/**" + "docker/**", + ".crow/process-updates.yaml", + ".crow/build-all-versions.yaml", + ".crow/weekly-rebuild-missing.yaml" + ], + "customManagers": [ + { + "customType": "regex", + "fileMatch": ["^\\.crow/.*\\.yaml$"], + "matchStrings": [ + "packageVersion\\(\"bincraft\"\\) != \"(?[0-9.]+)\"", + "bincraft\\.git@v(?[0-9.]+)" + ], + "depNameTemplate": "bincraft", + "packageNameTemplate": "https://codefloe.com/rpkgs/bincraft.git", + "datasourceTemplate": "git-tags", + "extractVersionTemplate": "^v?(?.+)$" + }, + { + "customType": "regex", + "fileMatch": ["^local/uvr-install\\.sh$"], + "matchStrings": [ + "# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+UVR_PIN=\"(?[^\"]+)\"" + ] + } ] } diff --git a/scripts/purge_cdn_cache.sh b/scripts/purge_cdn_cache.sh index a4e34c3..b525a44 100644 --- a/scripts/purge_cdn_cache.sh +++ b/scripts/purge_cdn_cache.sh @@ -1,64 +1,44 @@ -#!/bin/bash +#!/usr/bin/env bash +# +# Purge BunnyCDN edge cache for the index files of one ARCH+OS_ID +# combination across one or more domains. +# +# Usage: +# purge_cdn_cache.sh [...] +# +set -euo pipefail -set -o pipefail -set -x +if (( $# < 4 )); then + echo "usage: $0 [...]" >&2 + exit 2 +fi -# Function to URL-encode a string using Python -urlencode() { - python3 -c "import urllib.parse, sys; print(urllib.parse.quote(sys.argv[1], safe=''))" "$1" +api_key="$1" +arch="$2" +os_id="$3" +shift 3 + +resources=( + "src/contrib/PACKAGES" + "src/contrib/PACKAGES.gz" + "src/contrib/PACKAGES.rds" + "src/contrib/PACKAGES.db" + "src/contrib/Meta/archive.rds" +) + +purge_one() { + local target="$1" + # --data-urlencode encodes value of `url=` for us; -G forces GET. + curl -sS -G -X POST \ + -H "AccessKey: ${api_key}" \ + --data-urlencode "url=${target}" \ + --data "async=false" \ + "https://api.bunny.net/purge" + echo } -# Function to send the purge request using the provided API key -purge() { - local api_key="$1" - local url="$2" - curl -vL -X POST -H "AccessKey: $api_key" "$url" -} - -# Function to generate the purge URLs for a given domain, arch, and OS ID -generate_urls() { - local domain="$1" - local arch="$2" - local os_id="$3" - - # Array of resource paths (can be modified or extended as needed) - local resources=( - "src/contrib/PACKAGES" - "src/contrib/PACKAGES.gz" - "src/contrib/PACKAGES.rds" - "src/contrib/PACKAGES.db" - ) - - local urls=() +for domain in "$@"; do for resource in "${resources[@]}"; do - local target_url="https://${domain}/${arch}/${os_id}/latest/${resource}" - local encoded - encoded=$(urlencode "$target_url") - urls+=( "https://api.bunny.net/purge?url=${encoded}&async=false" ) + purge_one "https://${domain}/${arch}/${os_id}/latest/${resource}" done - - echo "${urls[@]}" -} - -# Function to process multiple domains. -# Usage: process_domains domain1 [domain2 ... domainN] -purge_cdn_cache() { - local api_key="$1" - local arch="$2" - local os_id="$3" - shift 3 # Remove the first three arguments, leaving the domains - - for domain in "$@"; do - # Generate the list of URLs for the current domain - local urls - urls=($(generate_urls "$domain" "$arch" "$os_id")) - - # Purge each URL - for url in "${urls[@]}"; do - purge "$api_key" "$url" - done - done -} - -# Example usage: -# purge_cdn_cache "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "domain1.com" "domain2.com" +done diff --git a/scripts/purge_cdn_zone.sh b/scripts/purge_cdn_zone.sh new file mode 100755 index 0000000..648dfc3 --- /dev/null +++ b/scripts/purge_cdn_zone.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Purge the entire BunnyCDN pull zone. +# +# `purge_cdn_cache.sh` purges the five index files by URL, which is right after +# a normal update: new packages arrive at new URLs, so only the index is stale. +# +# A rebuild is different. It replaces an object *in place*: a package whose +# build failed was published as its CRAN source, and the rebuilt binary takes +# exactly the same URL. The zone caches tarballs for ~370 days +# (`cache_expiration_time` in cdn.tf), so without a purge every client keeps +# receiving the source tarball for up to a year, and nothing about it looks +# wrong from the outside. +# +# Purging per URL would mean one API call per replaced package -- ~13.5k per +# arch against a rate-limited endpoint, where a single missed call leaves a +# silently stale package. One zone purge is a single call regardless of how many +# objects were replaced. The cost is a cold cache for everything else, which is +# why this is not used by the daily update path. +# +# The public hostnames currently use separate pull zones, so callers must pass +# every zone that serves the repository. A zone can be identified by its +# numeric ID or by one of its hostnames; hostname lookup avoids persisting IDs +# that change when a zone is recreated. +# +# Usage: +# purge_cdn_zone.sh [...] +# +set -euo pipefail + +if (($# < 2)); then + echo "usage: $0 [...]" >&2 + exit 2 +fi + +api_key="$1" +shift + +resolve_zone_id() { + local zone="$1" + local response_file + local zone_id + + if [[ "${zone}" =~ ^[0-9]+$ ]]; then + echo "${zone}" + return + fi + + response_file=$(mktemp) + curl -sS -o "${response_file}" \ + -H "AccessKey: ${api_key}" \ + "https://api.bunny.net/pullzone" + zone_id=$( + jq -r --arg hostname "${zone}" \ + '(.Items // .)[] | select(any(.Hostnames[]?; .Value == $hostname)) | .Id' \ + "${response_file}" + ) + rm -f "${response_file}" + + if [[ -z "${zone_id}" ]]; then + echo "Could not find BunnyCDN pull zone for hostname ${zone}" >&2 + exit 1 + fi + + echo "${zone_id}" +} + +for zone in "$@"; do + zone_id=$(resolve_zone_id "${zone}") + echo "Purging BunnyCDN pull zone ${zone_id}" + + response_file="/tmp/purge_zone_response_${zone_id}.txt" + status=$( + curl -sS -o "${response_file}" -w '%{http_code}' -X POST \ + -H "AccessKey: ${api_key}" \ + -H "Content-Length: 0" \ + "https://api.bunny.net/pullzone/${zone_id}/purgeCache" + ) + + if [[ "${status}" != "200" && "${status}" != "204" ]]; then + echo "Purge of pull zone ${zone_id} failed with HTTP ${status}:" >&2 + cat "${response_file}" >&2 + exit 1 + fi + + echo "Purged pull zone ${zone_id} (HTTP ${status})" +done diff --git a/specs/2026-06-30-package-patching-design.md b/specs/2026-06-30-package-patching-design.md new file mode 100644 index 0000000..1344173 --- /dev/null +++ b/specs/2026-06-30-package-patching-design.md @@ -0,0 +1,157 @@ +# Design: Dynamic per-package patching during binary builds + +Date: 2026-06-30 +Status: Approved (pending spec review) + +## Problem + +Some CRAN packages fail to compile on specific build platforms due to compiler- or OS-specific issues that have nothing to do with the package being built. +The canonical example is `RcppParallel`: its bundled Intel TBB sources fail to build on musl (Alpine) and on newer OS/compiler combinations. +Observed failure on `ubuntu-2604` ("resolute") with `g++ 15.2.0`: + +``` +../build/common.inc:74: *** "" is not supported. Add build/.inc file with os-specific settings . Stop. +make: *** [Makevars:163: tbb] Error 2 +ERROR: compilation failed for package 'RcppParallel' +``` + +Because `RcppParallel` is a dependency of many packages, a single such failure cascades: every dependent package (e.g. `rts2`) also fails, even though nothing is wrong with the dependent itself. + +Today there is no way to intervene. +A package can only be **excluded** (`local/excluded-packages.json`), which is all-or-nothing and does not help dependents. + +## Goal + +Allow a curated set of packages to be "patched" — via lightweight build-time overrides or, when necessary, real source diffs — **before** they are installed, whether the package is a direct build target or a transitive dependency pulled in by `pak`. + +## Key constraint that drives the design + +When `RcppParallel` fails here, it is being installed as a **transitive dependency** by `pak`, inside `bincraft::build_binary_package()`. +`pak` downloads, configures, and compiles it in one subprocess; this repo never touches that source. +For a fix to reach a dependency-of-a-dependency, the fixed package must be visible to `pak` itself, where `pak`'s repositories/sources are configured — which is inside `bincraft`. + +Decisions taken during brainstorming: + +- **Mechanism lives in `bincraft`** (the engine), because only there can transitive deps be influenced. +- **Patch tiers: both, env-overrides first.** Support cheap per-package env vars / configure args / Makevars (version-independent) *and* true source diffs (version-pinned), preferring the lightweight override. +- **Registry data lives in this repo** (`build-cran-binaries`) and is passed into `bincraft`, keeping `bincraft` as pure mechanism and the frequently-changing policy data with operational config. + +## Approaches considered + +| Approach | How pak sees the fix | Verdict | +|---|---|---| +| A. Patched **source** repo — drop patched `.tar.gz` source into a local repo, prepend it | pak recompiles from your source | Simple, but env-tier overrides leak globally (one subprocess builds everything) and the dep recompiles on every dependent build | +| **B. Pre-built patched binary repo (chosen)** | pak installs a ready binary by repo priority | Per-package scoping is free; no recompile; the binary is a cacheable/uploadable artifact that fits the existing system | +| C. pkgdepends per-build hook | intercept each build | No clean per-package pre-compile hook exists; fragile | + +Chosen: **B**. + +## Architecture + +### Registry (this repo) + +``` +local/patches/ + registry.json # the manifest + RcppParallel/ + fix.patch # optional source diff, referenced by an entry +``` + +`registry.json` is an array of entries: + +```json +[ + { + "package": "RcppParallel", + "versions": "*", + "platforms": ["alpine", "ubuntu-2604"], + "env": { "RCPP_PARALLEL_USE_TBB": "0" }, + "configure_args": [], + "makevars": {}, + "patch": null, + "reason": "bundled TBB fails to build on musl / newer compilers" + } +] +``` + +Field semantics: + +- `package` (string, required): CRAN package name. +- `versions` (string, required): `"*"` for any, a constraint such as `">=5.1.0"`, or an exact version `"5.1.11-2"`. + Env-tier fixes are typically `"*"`; source diffs are normally exact or lower-bounded because a diff is pinned to the source it was generated against. +- `platforms` (array of strings, required): matched against the running build's platform tokens — distro family (`alpine`, `ubuntu`, `redhat`), codename (`ubuntu-2604`, `alpine-324`), and arch (`amd64`, `arm64`). + An entry matches if any listed token matches any build token. + `["*"]` matches all platforms. +- `env` (object, optional): environment variables exported only for this package's isolated build. +- `configure_args` (array, optional): passed as `--configure-args` to the isolated build. +- `makevars` (object, optional): key/value pairs written into a package-local Makevars for the isolated build. +- `patch` (string or null, optional): path (relative to `local/patches/`) to a unified diff applied to the unpacked CRAN source before building. +- `reason` (string, required): human explanation, surfaced in logs and metadata. + +A fix is any combination of `env`, `configure_args`, `makevars`, and `patch`. +"Env-first" is an authoring guideline (prefer the lightweight override) and an ordering of effort, not a runtime branch — all present fields are applied together for the isolated build. + +### Flow (inside bincraft, around existing pak resolution) + +1. **Resolve** the dependency set (dry-run) to learn the concrete versions `pak` will install. + Reuse bincraft's existing resolution where possible (e.g. a `pkgdepends` proposal: `$resolve()` → inspect resolution → ... → `$solve()` / `$install()` after the local repo is prepended). +2. For each resolved package that matches a registry entry (name + `versions` + `platforms`): obtain a **patched binary** for the exact `version × platform × arch × R-minor`: + - **Cache hit** (local `/mnt/cache/patched-binaries/` or S3): fetch it into the local repo. + - **Cache miss**: download the CRAN **source** for that version, apply the source `patch` (if any) to the unpacked tree, build the binary in isolation with `env` / `configure_args` / `makevars` applied, then place the binary in the local repo and write it to the cache (and S3 if uploading is enabled). +3. **Prepend** the local binary repo (`file://…`) to `pak`'s repo list, and regenerate its `PACKAGES` index. +4. Run the **normal install**. + `pak` resolves the patched binary for the matched package — direct or transitive — because it wins on repo priority for an equal version, and installs it without recompiling. + +### Caching (essential) + +`RcppParallel` is a dependency of dozens of packages; without caching the fix would be rebuilt on every dependent build. +Patched binaries are keyed by: + +``` +_____ +``` + +`patchhash` is a hash of the normalized registry entry plus the referenced diff file contents. +Editing a patch therefore changes the hash and auto-invalidates stale cached binaries. + +- Local cache: `/mnt/cache/patched-binaries/`. +- Optional S3 cache for cross-build reuse: a dedicated `…/patched/` slot under the existing arch/codename structure, mirroring how normal binaries are stored. + +### S3 upload + +Patched binaries **are** uploaded to S3 (in addition to the local cache) so they are reused across CI jobs and machines, not just within one container. +They live in a separate `patched/` slot and are not published into the user-facing `src/contrib` index — they are an internal build accelerator, not a distributed artifact. + +## Error handling + +- **Source diff fails to apply** (CRAN moved past the pinned version): log a clear warning, skip that entry, and proceed. + The package builds unpatched (status quo) and may fail. + The skipped/failed-to-apply patch is surfaced in build metadata. +- **Pre-build of the patched binary fails**: log a warning, skip, proceed. +- **No version or platform match**: skip silently (the entry simply does not apply to this build). +- **Overlapping entries for one package**: the most specific entry wins (a concrete `platforms`/`versions` beats `"*"`). + Genuine ambiguity (two equally specific, conflicting entries) is a validation error reported before the build. + +## Observability + +- One log line per applied patch, e.g.: `Applying patch to RcppParallel 5.1.11-2 [env: RCPP_PARALLEL_USE_TBB=0]: bundled TBB fails on musl / newer compilers` +- The set of applied patches (package, version, `patchhash`) is recorded in the Postgres build-metadata row for the build, so it is queryable later. + +## Testing + +- **Unit (registry):** parsing and matching — version constraints, platform token matching, precedence/specificity, and detection of ambiguous overlaps. +- **Unit (cache key):** `patchhash` changes when the entry or diff changes; is stable otherwise. +- **Integration:** `RcppParallel` on `resolute` (and/or Alpine) fails to build without a registry entry and succeeds with one; a dependent package such as `rts2` succeeds once the dependency is patched. +- **Failure path:** an entry pinned to an old version against a newer CRAN release → graceful skip with a warning, build continues. + +## Out of scope + +- Shipping a default registry inside `bincraft` (registry is repo-local for now; a baseline-in-engine + repo-override model can come later if needed). +- Publishing patched binaries into the public `src/contrib` index. +- Automatic detection of which packages need patches — entries are curated by hand. + +## Split of work + +- **bincraft:** the mechanism — registry ingestion, resolution hook, isolated patched-binary build, caching/upload, local-repo prepend, logging, metadata recording. + A new `patches` argument on `build_binary_package()`. +- **build-cran-binaries (this repo):** the `local/patches/` registry and diffs, passing `patches = "local/patches"` through `build-one.R` / `build-all.R`, and documentation. diff --git a/specs/2026-08-07-per-minor-edge-routing-design.md b/specs/2026-08-07-per-minor-edge-routing-design.md new file mode 100644 index 0000000..b55812c --- /dev/null +++ b/specs/2026-08-07-per-minor-edge-routing-design.md @@ -0,0 +1,174 @@ +# Design: Routing clients to per-R-minor binary slots + +Date: 2026-08-07 +Status: Approved (pending spec review) + +## Problem + +`bincraft` routes ABI-"risky" packages to a per-minor slot `…/latest/src/contrib//` and indexes every directory independently (`upload_package_index()` calls `cranlike::update_PACKAGES()` on one prefix at a time). +Nothing unions those indices, and `contrib.url()` only ever yields `/src/contrib`, so no value of `options(repos)` can address a per-minor slot. +Only `uvr` resolves per-minor URLs, which means the per-minor slots are invisible to `install.packages()` by construction. + +Measured on 2026-08-07: + +| slot | flat `src/contrib` | `src/contrib/4.5` | unique packages only in the per-minor slot | +| ----------------- | ------------------ | ----------------- | ------------------------------------------ | +| `amd64/alpine324` | 21 640 | 3 310 | 2 886 | +| `amd64/noble` | 24 495 | 398 | 23 | + +This is what issue #63 records as "missing binaries" on `alpine324`. +The packages are not missing; they are in a directory base R cannot reach. +The user-visible symptom in `reg.devxy.io/r/r-alpine:4.5-3.24` is: + +``` +> install.packages("curl") +Warning message: +package 'curl' is not available for this version of R +``` + +A second, unrelated defect exists on the same slot and is **out of scope here**: many `alpine324` tarballs are byte-identical CRAN _source_ tarballs that the index nevertheless stamps `Built: R 4.5.3; …-linux-musl`. +Routing exposes `curl`; only a rebuild of that slot makes it install. + +## Goal + +Let a stock `install.packages()` see one complete package list for its own R minor, without duplicating tarballs and without an R-version-varying cache key anywhere in the CDN. + +## Key constraint that drives the design + +R resolves a package's download URL from the index, not from the request path, and it keeps the `contriburl` it _asked for_ rather than the one it was redirected to. +Measured with `options(repos = …/latest)` against a middleware that redirects the index into `4.5/`: + +``` +curl available: TRUE +curl repo: …/latest/src/contrib # the flat URL, not the 4.5 one it was served from +``` + +So the union index is always addressed relative to the **flat** directory, whatever path it was fetched from. +`available.packages()` honours a `Path:` field and folds it into the `Repository` column, which gives the whole routing for free: + +- a per-minor record carries `Path: `, so its tarball is fetched from `…/src/contrib//` +- a flat record carries no `Path`, so its tarball is fetched from `…/src/contrib/` + +Verified end to end against the live CDN with a locally built union index for `amd64/alpine324` (31 507 records): + +``` +curl: 7.1.0 -> …/latest/src/contrib/4.5 -> curl_7.1.0.tar.gz 717 725 B +jsonlite: 2.0.0 -> …/latest/src/contrib -> jsonlite_2.0.0.tar.gz 1 055 849 B +``` + +The corollary is that the edge script must **not** rewrite tarball requests: every tarball URL is already correct when it leaves the client, and redirecting one into `/` would break exactly the flat packages the union is meant to preserve. + +The complementary trick does not work: R's `gzcon()` reads only the first member of a concatenated gzip stream (10 291 of an expected 31 931 records), so an edge-side merge would have to fully decompress and recompress both indices and additionally 404 `PACKAGES.rds` to stop R preferring it. +That is why the union is produced in `bincraft`, not at the edge. + +## Approaches considered + +| Approach | Where the union lives | Verdict | +| ----------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- | +| **A. Union index written by `bincraft` (chosen)** | per-minor `PACKAGES*`, per-minor entries carry `Path: ` | Edge does one redirect; `PACKAGES.rds` stays correct; no duplication | +| B. Merge at the edge | middleware fetches both indices, recompresses | ~2 MB decompress/recompress per cache fill, cache key must include the R minor, breaks R's `.rds` fast path | +| C. Move the minor up the path (`latest//src/contrib/`) | addressable by `options(repos)` directly | No edge logic at all, but a full layout migration and breaks the published URL contract | + +Chosen: **A**. + +## Architecture + +### bincraft: union index (separate PR) + +After writing a per-minor index, republish it as a union of that slot and the flat slot: + +1. Read the flat slot's `PACKAGES.rds` and the per-minor slot's own records. +2. Set `Path: ` on every per-minor record, so its tarball resolves into the per-minor directory. +3. Drop every flat record whose package is already present in the per-minor slot, so the per-minor build always wins, and leave the survivors without a `Path`. +4. Write the merged `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` into `…/src/contrib//`. + +Guard: refuse to publish a union with fewer records than the flat index it was built from. +A truncated union is worse than no union, because it silently removes packages from every client on that minor. + +### Edge script (this repo) + +The script routes `PACKAGES`, `PACKAGES.gz` and `PACKAGES.rds` requests, and nothing else. + +``` +normalize path +parseClient(UA) -> { rMinor, arch, os } # rMinor from "R (4.5.3 …)" or "R/4.5.3" +darwin branches # unchanged +if path is /{arch}/{os}/latest/src/contrib/PACKAGES* + already under // ? pass through # loop guard + rMinor known && slot in UNION_SLOTS ? 302 -> …/src/contrib//PACKAGES* + else pass through # flat slot, today's behaviour +if path is /src/contrib/… # bare root + resolve arch + os; unknown -> 302 to CRAN + then apply the same PACKAGES* rule +else pass through +``` + +Redirects carry `Cache-Control: no-store`. +Every cacheable URL is therefore UA-independent, and no cache key has to vary by R version. + +### Repaired bare-root detection + +The bare `https://cran.rpkgs.com` form is currently broken for every Linux client that uses a stock R user agent. +`ALPINE_REGEX`, `UBUNTU_REGEX` and `RHEL_REGEX` only match a Posit-style user agent the user has to set by hand; stock R never carries the distro, so the script falls through to `extractOs()` and redirects to a slot that does not exist: + +``` +UA: R (4.5.3 x86_64-pc-linux-musl …) -> 302 /amd64/linux-musl/latest/… (404) +UA: R (4.5.3 x86_64-pc-linux-gnu …) -> 302 /amd64/linux-gnu/latest/… (404) +``` + +The fallback to a phantom `linux-musl` / `linux-gnu` slot is removed. +An unidentifiable distro redirects to CRAN, which is the existing behaviour for an unparseable user agent. +The R _minor_ is always present in a stock user agent, so per-minor routing itself does not depend on distro detection. + +### Rollout gate + +`UNION_SLOTS` is a `bunnynet_compute_script_variable` listing the slots whose per-minor index is already a union. +It is empty by default, so deploying the script changes nothing until `bincraft` has backfilled a slot, and a rollback is a variable edit rather than a code deploy. +All slots currently carry `4.4`, `4.5` and `4.6`; a client on any other minor falls through to the flat slot. + +### Deployment from this repo + +The script is a file in the repo, applied by the existing OpenTofu configuration: + +``` +edge/rpkgs-router.ts # the script +edge/rpkgs-router.test.ts # UA x path -> expected Location matrix +cdn.tf # bunnynet_compute_script + _variable +``` + +Provider `BunnyWay/bunnynet` v0.17.0 (already pinned) ships `bunnynet_compute_script` with `content` loadable via `file()`, plus `bunnynet_compute_script_variable`. +`middleware_script = bunnynet_compute_script.rpkgs_router.id` replaces the hard-coded `29277`, after a one-time `tofu import` of the existing script. + +## Error handling + +- Unknown R minor, or a slot not listed in `UNION_SLOTS`: pass through to the flat slot. + The client sees exactly today's behaviour. +- Unparseable distro on the bare-root form: redirect to CRAN. +- A request already under `…/src/contrib//`: pass through, so a redirect can never loop. +- A per-minor slot that does not exist for a listed minor: the client gets the origin's 404. + `UNION_SLOTS` is the operator's assertion that the slot is ready, so this is a configuration error, not a runtime condition to paper over. + +## Testing + +Local, before any apply: `deno run -A edge/rpkgs-router.ts` serves the middleware against the real origin, so `edge/rpkgs-router.test.ts` drives the whole matrix against that local server. + +- User agent matrix: R 4.4 / 4.5 / 4.6 on musl and gnu, both arches, Posit-style and stock forms, plus a darwin UA and a non-R UA. +- Path matrix: `PACKAGES`, `PACKAGES.gz`, `PACKAGES.rds`, a tarball, a path already under `4.5/`, and `/src/contrib/…` on the bare root. +- Assertion is the `Location` header (or its absence), not the body. + +After apply, a smoke test against `cran.rpkgs.com`: + +- `available.packages()` inside `reg.devxy.io/r/r-alpine:4.5-3.24` returns the union count, and `"curl" %in% rownames(...)` is `TRUE`. +- A flat-slot package still downloads from `…/src/contrib/`, and a per-minor package downloads from `…/src/contrib//`. + +## Out of scope + +- `Meta/archive.rds` stays flat-only, so `remotes::install_version()` does not see per-minor archives. +- The `alpine324` source-tarball defect: that slot serves CRAN sources stamped as binaries, and needs a rebuild independent of this work. +- Any change to how `uvr` resolves per-minor URLs; it already addresses the slots directly. + +## Split of work + +1. `bincraft`: union index writer plus its guard, and a re-index of one slot to validate. +2. This repo: `edge/rpkgs-router.ts`, its test matrix, and the `cdn.tf` resources with `UNION_SLOTS` empty. +3. Enable `UNION_SLOTS` slot by slot as `bincraft` backfills them. diff --git a/specs/2026-08-12-shard-weekly-rebuild-design.md b/specs/2026-08-12-shard-weekly-rebuild-design.md new file mode 100644 index 0000000..f4ccbaa --- /dev/null +++ b/specs/2026-08-12-shard-weekly-rebuild-design.md @@ -0,0 +1,167 @@ +# Design: Sharding and resuming the weekly rebuild + +Date: 2026-08-12 +Status: Approved (pending spec review) + +## Problem + +`weekly-rebuild-missing` runs one job per `-` and walks that slot's rebuild list serially in a single `R -q -e` invocation (`.crow/weekly-rebuild-missing.yaml:165`). +Until 2026-08-09 that was cheap, because every source fallback was skipped as "already built" and the list was effectively empty. +Since bincraft #105/#106/#107 and build-cran-binaries #159 the gate works, and the lists are now large. + +Share of records whose object is byte-identical to CRAN's source, measured against `cran.r-project.org` MD5s on 2026-08-12: + +| slot | records | source-served | share | +| ------------------ | ------: | ------------: | -----------: | +| `amd64/resolute` | 24 212 | 15 023 | 62.1% | +| `arm64/resolute` | 24 291 | 13 670 | 56.3% | +| `arm64/alpine324` | 24 328 | 9 514 | 39.2% | +| `amd64/alpine324` | 24 343 | 8 917 | 36.7% | +| `arm64/rhel10` | 24 695 | 5 384 | 21.9% | +| `amd64/rhel10` | 24 881 | 4 712 | 19.2% | +| 12 remaining slots | ~24 700 | 850 to 2 130 | 3.5% to 8.7% | + +A single serial job cannot absorb that. +Pipeline 10910 (`weekly_rebuild_missing:alpine-324-amd64`) started on 2026-08-09, ran for roughly two days, reached `[8692/23885] cholera`, and was killed there. + +Two distinct failures follow from that shape. + +**No parallelism.** The work is embarrassingly parallel across packages, but one job does all of it. + +**No resumability, and no clean stopping point.** The loop has no terminating condition other than exhausting the list, so the only way to stop it is a kill. +A restarted run re-reads the same list and walks it from the first entry. +It skips completed packages via `check_s3_root_package()`, but that costs a CRAN version resolution and an S3 `HEAD` per package, thousands of times, before it reaches new work. +Worse, a kill is not a pipeline failure: the `Purge CDN cache` step is guarded by `when: status: [success, failure]` (`.crow/weekly-rebuild-missing.yaml:206-207`), and on 10910 it produced no output at all. +So the ~4 600 binaries that run did publish stayed hidden behind stale edge copies. + +## Goal + +Turn each slot's rebuild into bounded, parallel, restartable units, without introducing state that can disagree with the bucket. + +## Design + +### 1. Shard the matrix three ways + +Each of the 18 `OS`/`ARCH` rows in `.crow/weekly-rebuild-missing.yaml` gains `SPLIT_INTO: 3` and `SPLIT_INDEX: 1|2|3`, giving 54 rows. +This mirrors `.crow/build-all-versions.yaml:57-98`, which already shards its matrix four ways per arch. + +Routing needs no change. +The cron filter `cron: weekly-rebuild-missing-${OS}-${ARCH}` and the manual `evaluate: weekly_rebuild_missing == "${OS}-${ARCH}"` both match all three shards of a slot. +Placement stays on the `rpkgs-${ARCH}` group label, so shards queue against available capacity rather than oversubscribing it. + +### 2. Extract the loop into `local/rebuild-missing.R` + +The build is currently a single ~1 500-character `R -q -e` argument. +Shard arithmetic and resume logic do not belong in a YAML string, and none of it is testable there. +The loop moves to `local/rebuild-missing.R`, invoked as `Rscript local/rebuild-missing.R $SPLIT_INTO $SPLIT_INDEX`, mirroring `local/build-all.R`. +Its body is unchanged in substance: read `/tmp/rebuild_pkgs.txt`, subtract `local/excluded-packages.json`, loop with `tryCatch` around `bincraft::build_binary_package()`. + +The slice is **interleaved**, not contiguous: + +```r +# the list is alphabetical and build cost clusters by name (Rcpp*, Bioc*, +# rstan*), so contiguous thirds would be badly unbalanced +mine <- pkgs[seq(split_index, length(pkgs), by = split_into)] +``` + +`local/build-all.R:64` uses contiguous chunks via `cut()`. +That is fine there because its list is every CRAN package and version, so the chunks average out. +Here the list is a filtered backlog in which expensive families sit adjacent, so interleaving is the better default. +Interleaving also makes each shard's `[i/n]` progress representative of the slot as a whole. + +### 3. Resume by re-deriving state from the bucket + +Before the loop, the shard performs one `s3fs::s3_dir_info()` on `devxy-rpkgs-binaries///latest/src/contrib` and reads the `etag` column. +It fetches CRAN's `PACKAGES` once for the latest version and published `MD5sum` of every package. +A package is still outstanding if and only if the object at `_.tar.gz` has an ETag equal to CRAN's `MD5sum` for that version, which is the definition `check_s3_root_package()` already applies one package at a time. + +```r +# one paginated listing instead of ~2900 sequential HEAD requests per shard +info <- s3fs::s3_dir_info(slot_dir) +etag <- setNames(gsub('^"|"$', "", info$etag), basename(info$uri)) + +key <- sprintf("%s_%s.tar.gz", mine, cran_version[mine]) +# keep a package when no object exists yet, or when the object is still +# byte-identical to CRAN's source; drop it once a real binary is published +mine <- mine[is.na(etag[key]) | etag[key] == cran_md5[key]] +``` + +This is the whole resume mechanism. +There is no progress file, no volume, and no database cursor. +A restarted shard recomputes ground truth and continues where it stopped, and it is correct even when a sibling shard, a `process-updates` cron, or a manual `just rebuild` completed something in the meantime. + +Three properties make this the right source of truth: + +- **It is what the build itself checks.** Any other store can disagree with the bucket; this one cannot. +- **It is agent-independent.** `.crow/weekly-rebuild-missing.yaml` mounts no `volumes:`, unlike `.crow/build-all-versions.yaml:132-133`, so `/mnt/cache` is per-job and cannot carry progress anyway. +- **It costs one listing.** `cranlike`'s `s3` fork already does exactly this call against this bucket at ~24 000 objects, so the approach is proven at the required scale. + +It must read ETags rather than the slot index's `Built` field, which is how `local/packages-to-build.R:104-130` answers the same question. +Under this design the index is not rewritten until the dependent re-index pipeline runs (section 5), so mid-run it cannot reflect the current run's progress. + +Packages that genuinely fail to build re-publish their CRAN source, so they stay outstanding and would be retried on every restart. +That is already handled upstream: `bincraft::filter_packages_with_errors()` (`R/build_binaries.R:1018`, `:1143`) drops anything with `error_occurred = TRUE`, and `store_build_metadata = TRUE` is passed on every call. +No additional poison-pill filter is needed here. + +Only the flat `src/contrib` path is considered. +The rebuild call passes no `is_r_minor_sensitive`, so it defaults to `FALSE` and only ever targets the flat path; the resume filter matches that scope deliberately. + +### 4. Give each shard a wall-clock budget + +`local/rebuild-missing.R` takes a budget, defaulting to 20 hours, and breaks out of the loop once it is exceeded: + +```r +# exit cleanly rather than being killed, so the dependent re-index still runs +if (difftime(Sys.time(), started, units = "hours") > budget_hours) { + cat(sprintf("Budget of %sh reached after %d/%d packages; stopping cleanly\n", budget_hours, i, n)) + break +} +``` + +It exits 0 and reports how much of the slice it covered. +Every run then has a terminating condition, the re-index and purge always fire, and the remainder is picked up by the next run with no bookkeeping, because section 3 recomputes the outstanding set from scratch. + +### 5. Move the re-index and purge into `.crow/weekly-rebuild-reindex.yaml` + +Three shards per slot means three concurrent `upload_package_index()` calls on the same S3 prefix. +`cranlike::update_PACKAGES()` lists the live bucket, so an early lister that uploads last publishes an index missing its siblings' work. +The re-index steps (`.crow/weekly-rebuild-missing.yaml:171-176`) and the purge step (`:187-207`) therefore leave that file entirely. + +The new file carries: + +```yaml +depends_on: + - weekly-rebuild-missing +runs_on: [success, failure] +``` + +`runs_on: [success, failure]` validates as a workflow-level key under `crow lint`, so a failing shard no longer withholds the re-index. +The file uses the same 18-row matrix and the same `when:` gating as `weekly-rebuild-missing`, so it only re-indexes slots that actually ran. +Each row re-indexes the flat slot and every per-minor slot. +`scripts/purge_cdn_zone.sh` runs once on a single row, because all hostnames share pull zone `3857050` and 18 identical zone purges would be waste. + +## Failure behaviour + +| case | today | after | +| -------------------------- | ----------------------------------- | ----------------------------------------------------- | +| one package errors | `tryCatch` logs, loop continues | unchanged | +| a shard fails outright | purge runs, re-index does not | re-index and purge run via `runs_on` | +| a shard exceeds its budget | cannot happen, runs until killed | exits 0, re-index and purge run | +| a shard is killed | nothing runs | still nothing; trigger the re-index pipeline alone | +| a shard restarts | re-walks the list, HEAD per package | one listing, resumes at the first outstanding package | + +The known cost of `depends_on` being file-level rather than row-level: on the weekly cron no slot is re-indexed until the slowest of all 54 jobs finishes. +The 20-hour budget bounds that at roughly one day. + +## Out of scope + +- `build-all-versions` still cannot rebuild source fallbacks, because `local/build-all.R:113-122` drops every version with any `single_builds` row for the platform and arch, which is precisely the source-fallback set. That is a separate change. +- Bunny Perma-Cache eviction. `scripts/purge_cdn_zone.sh` purges the regular edge cache only; see the note in `CLAUDE.md` and issue history. +- The audit that produces the rebuild list is unchanged. + +## Verification + +- `crow lint .crow/` passes for both pipeline files. +- `local/rebuild-missing.R` gets unit coverage in `local/tests/` for the two pure pieces: the interleaved slice (disjoint, covering, deterministic) and the outstanding-set filter (source-served ETag kept, binary ETag dropped, absent object kept). +- A single-slot manual run of `alpine-324-amd64` shard 1 confirms the listing shortcut against the live bucket, and that the reported outstanding count is close to the 8 917 measured above divided by three. +- Restarting that shard mid-run confirms it resumes rather than replaying, by comparing the outstanding count it reports on the second start. diff --git a/vars.tf b/vars.tf index 8168262..b831269 100644 --- a/vars.tf +++ b/vars.tf @@ -7,3 +7,21 @@ variable "HETZNER_S3_SECRET_KEY_K3S" { type = string sensitive = true } + +variable "B2_S3_ACCESS_KEY" { + type = string + sensitive = true +} + +variable "B2_S3_SECRET_KEY" { + type = string + sensitive = true +} + +variable "cdn_block_ips" { + description = "IP addresses to block at the BunnyCDN pullzones (cran.devxy.io, cran.rpkgs.com)." + type = list(string) + default = [ + "185.172.53.0", + ] +}