refactor: migrate package installation from pak to uvr
bincraft dropped pak in favour of uvr, so the pipelines, helper scripts and images in this repo move with it. - add local/uvr-install.sh as the single replacement for pak::pak(); it bootstraps a pinned uvr, mints a throwaway project under TMPDIR and runs `uvr add --no-install` + `uvr sync --library`, because `uvr add` refuses to run outside a project and only `sync` honours --library - install bincraft via its Forgejo spec (forgejo::codefloe.com/rpkgs/ bincraft@<tag>) instead of a git:: URL, keeping the git ls-remote tag resolution - pass UVR_R_BIN/UVR_TARGET_LIB from install-bincraft.R so the per-R-minor passes target their own R and library - replace R_PKG_CACHE_DIR with UVR_CACHE_DIR/UVR_PACKAGES_DIR on the persistent volume, preserving the amd64-off/arm64-on split - drop trim_pkgcache_metadata() and its test; uvr's cache does not grow the way pkgcache's _metadata dir did - let uvr install system requirements from its vendored r-system-requirements rules, replacing pak::sysreqs_db_update() - migrate the shiny app image, the alpine reprex and the CRAN loading test; ship uvr-install.sh in the build-one image - track the uvr pin with renovate
This commit is contained in:
parent
fe2289af56
commit
f3c18b6805
20 changed files with 327 additions and 245 deletions
|
|
@ -45,7 +45,7 @@ steps:
|
||||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
- mkdir -p /mnt/cache/R-pkgs
|
- mkdir -p /mnt/cache/R-pkgs
|
||||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))'
|
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
|
||||||
- /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT
|
- /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT
|
||||||
backend_options:
|
backend_options:
|
||||||
kubernetes:
|
kubernetes:
|
||||||
|
|
|
||||||
|
|
@ -10,22 +10,22 @@ variables:
|
||||||
- arm64
|
- arm64
|
||||||
default: amd64
|
default: amd64
|
||||||
OS:
|
OS:
|
||||||
description: "Base OS image name."
|
description: 'Base OS image name.'
|
||||||
options:
|
options:
|
||||||
- alpine
|
- alpine
|
||||||
- redhat
|
- redhat
|
||||||
- ubuntu
|
- ubuntu
|
||||||
default: alpine
|
default: alpine
|
||||||
OS_VERSION:
|
OS_VERSION:
|
||||||
description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)."
|
description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).'
|
||||||
options:
|
options:
|
||||||
- "3.24"
|
- '3.24'
|
||||||
- "8"
|
- '8'
|
||||||
- "9"
|
- '9'
|
||||||
- "10"
|
- '10'
|
||||||
- "jammy"
|
- 'jammy'
|
||||||
- "noble"
|
- 'noble'
|
||||||
default: "3.24"
|
default: '3.24'
|
||||||
R_VERSION:
|
R_VERSION:
|
||||||
description: 'Primary R version under /opt/R.'
|
description: 'Primary R version under /opt/R.'
|
||||||
options:
|
options:
|
||||||
|
|
@ -66,23 +66,24 @@ steps:
|
||||||
from_secret: B2_S3_SECRET_KEY
|
from_secret: B2_S3_SECRET_KEY
|
||||||
PGPASS:
|
PGPASS:
|
||||||
from_secret: PGPASS
|
from_secret: PGPASS
|
||||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
|
||||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||||
|
# Keep uvr's downloads and extracted-package entries on the persistent
|
||||||
|
# volume instead of the container-local ~/.uvr default.
|
||||||
|
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||||
|
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||||
CCACHE_DIR: /mnt/cache/ccache
|
CCACHE_DIR: /mnt/cache/ccache
|
||||||
volumes:
|
volumes:
|
||||||
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
||||||
commands:
|
commands:
|
||||||
# one-time full wipe to fix corrupted .so files from previous failed builds
|
# one-time full wipe to fix corrupted .so files from previous failed builds
|
||||||
# - rm -rf /mnt/cache/R-pkgs
|
# - rm -rf /mnt/cache/R-pkgs
|
||||||
# Clear churny pkgcache metadata left by a prior crashed run (the "patched"
|
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||||
# repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*).
|
|
||||||
# Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs.
|
|
||||||
- rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true
|
|
||||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
|
||||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
# Pin the same bincraft version the build steps use, so the precomputed
|
# Pin the same bincraft version the build steps use, so the precomputed
|
||||||
# snapshot and the per-agent library stay consistent across the pipeline.
|
# snapshot and the per-agent library stay consistent across the pipeline.
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")'
|
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||||
|
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite
|
||||||
|
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))"
|
- /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))"
|
||||||
backend_options:
|
backend_options:
|
||||||
docker:
|
docker:
|
||||||
|
|
|
||||||
|
|
@ -11,25 +11,25 @@ variables:
|
||||||
- arm64
|
- arm64
|
||||||
default: amd64
|
default: amd64
|
||||||
OS:
|
OS:
|
||||||
description: "Base OS image name."
|
description: 'Base OS image name.'
|
||||||
options:
|
options:
|
||||||
- alpine
|
- alpine
|
||||||
- redhat
|
- redhat
|
||||||
- ubuntu
|
- ubuntu
|
||||||
default: alpine
|
default: alpine
|
||||||
OS_VERSION:
|
OS_VERSION:
|
||||||
description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)."
|
description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).'
|
||||||
options:
|
options:
|
||||||
- "3.22"
|
- '3.22'
|
||||||
- "3.23"
|
- '3.23'
|
||||||
- "3.24"
|
- '3.24'
|
||||||
- "8"
|
- '8'
|
||||||
- "9"
|
- '9'
|
||||||
- "10"
|
- '10'
|
||||||
- "jammy"
|
- 'jammy'
|
||||||
- "noble"
|
- 'noble'
|
||||||
- "resolute"
|
- 'resolute'
|
||||||
default: "3.24"
|
default: '3.24'
|
||||||
R_VERSION:
|
R_VERSION:
|
||||||
description: 'Primary R version under /opt/R.'
|
description: 'Primary R version under /opt/R.'
|
||||||
options:
|
options:
|
||||||
|
|
@ -47,38 +47,49 @@ labels:
|
||||||
platform: linux/${ARCH}
|
platform: linux/${ARCH}
|
||||||
group: rpkgs-${ARCH}
|
group: rpkgs-${ARCH}
|
||||||
|
|
||||||
|
# Empty UVR_CACHE_DIR/UVR_PACKAGES_DIR fall back to uvr's container-local
|
||||||
|
# ~/.uvr defaults; amd64 deliberately does not persist them (as with the
|
||||||
|
# pkgcache dir it replaces), arm64 does.
|
||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
- ARCH: amd64
|
- ARCH: amd64
|
||||||
R_PKG_CACHE_DIR: ''
|
UVR_CACHE_DIR: ''
|
||||||
|
UVR_PACKAGES_DIR: ''
|
||||||
SPLIT_INTO: 4
|
SPLIT_INTO: 4
|
||||||
SPLIT_INDEX: 1
|
SPLIT_INDEX: 1
|
||||||
- ARCH: amd64
|
- ARCH: amd64
|
||||||
R_PKG_CACHE_DIR: ''
|
UVR_CACHE_DIR: ''
|
||||||
|
UVR_PACKAGES_DIR: ''
|
||||||
SPLIT_INTO: 4
|
SPLIT_INTO: 4
|
||||||
SPLIT_INDEX: 2
|
SPLIT_INDEX: 2
|
||||||
- ARCH: amd64
|
- ARCH: amd64
|
||||||
R_PKG_CACHE_DIR: ''
|
UVR_CACHE_DIR: ''
|
||||||
|
UVR_PACKAGES_DIR: ''
|
||||||
SPLIT_INTO: 4
|
SPLIT_INTO: 4
|
||||||
SPLIT_INDEX: 3
|
SPLIT_INDEX: 3
|
||||||
- ARCH: amd64
|
- ARCH: amd64
|
||||||
R_PKG_CACHE_DIR: ''
|
UVR_CACHE_DIR: ''
|
||||||
|
UVR_PACKAGES_DIR: ''
|
||||||
SPLIT_INTO: 4
|
SPLIT_INTO: 4
|
||||||
SPLIT_INDEX: 4
|
SPLIT_INDEX: 4
|
||||||
- ARCH: arm64
|
- ARCH: arm64
|
||||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||||
|
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||||
SPLIT_INTO: 4
|
SPLIT_INTO: 4
|
||||||
SPLIT_INDEX: 1
|
SPLIT_INDEX: 1
|
||||||
- ARCH: arm64
|
- ARCH: arm64
|
||||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||||
|
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||||
SPLIT_INTO: 4
|
SPLIT_INTO: 4
|
||||||
SPLIT_INDEX: 2
|
SPLIT_INDEX: 2
|
||||||
- ARCH: arm64
|
- ARCH: arm64
|
||||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||||
|
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||||
SPLIT_INTO: 4
|
SPLIT_INTO: 4
|
||||||
SPLIT_INDEX: 3
|
SPLIT_INDEX: 3
|
||||||
- ARCH: arm64
|
- ARCH: arm64
|
||||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||||
|
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||||
SPLIT_INTO: 4
|
SPLIT_INTO: 4
|
||||||
SPLIT_INDEX: 4
|
SPLIT_INDEX: 4
|
||||||
|
|
||||||
|
|
@ -107,8 +118,10 @@ steps:
|
||||||
from_secret: GITHUB_PAT
|
from_secret: GITHUB_PAT
|
||||||
# normal env vars
|
# normal env vars
|
||||||
GIT_USER: pat-s
|
GIT_USER: pat-s
|
||||||
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
# set the location of uvr's caches, which persist the R package
|
||||||
R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR}
|
# dependencies needed to install the packages themselves
|
||||||
|
UVR_CACHE_DIR: ${UVR_CACHE_DIR}
|
||||||
|
UVR_PACKAGES_DIR: ${UVR_PACKAGES_DIR}
|
||||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||||
CCACHE_DIR: /mnt/cache/ccache
|
CCACHE_DIR: /mnt/cache/ccache
|
||||||
NCPUS: 2
|
NCPUS: 2
|
||||||
|
|
@ -116,12 +129,7 @@ steps:
|
||||||
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
||||||
commands:
|
commands:
|
||||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
# Clear churny pkgcache metadata left by a prior crashed run (the "patched"
|
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||||
# repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*).
|
|
||||||
# Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. Within-run
|
|
||||||
# growth is bounded separately by trim_pkgcache_metadata() in build-all.R.
|
|
||||||
- rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true
|
|
||||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
|
||||||
# The primary pass must not rely on build-all-versions-install-deps having
|
# The primary pass must not rely on build-all-versions-install-deps having
|
||||||
# run on *this* agent: depends_on only orders the steps, but the cache
|
# run on *this* agent: depends_on only orders the steps, but the cache
|
||||||
# volume is per-agent, so a job landing on an agent where install-deps did
|
# volume is per-agent, so a job landing on an agent where install-deps did
|
||||||
|
|
|
||||||
|
|
@ -49,109 +49,109 @@ matrix:
|
||||||
R_VERSION: 4.5.3
|
R_VERSION: 4.5.3
|
||||||
IMG: alpine:3.22
|
IMG: alpine:3.22
|
||||||
OS_ID: alpine322
|
OS_ID: alpine322
|
||||||
PROCESS_NEW: "FALSE"
|
PROCESS_NEW: 'FALSE'
|
||||||
- OS: alpine-322
|
- OS: alpine-322
|
||||||
ARCH: arm64
|
ARCH: arm64
|
||||||
R_VERSION: 4.5.3
|
R_VERSION: 4.5.3
|
||||||
IMG: alpine:3.23
|
IMG: alpine:3.23
|
||||||
OS_ID: alpine322
|
OS_ID: alpine322
|
||||||
PROCESS_NEW: "FALSE"
|
PROCESS_NEW: 'FALSE'
|
||||||
- OS: alpine-323
|
- OS: alpine-323
|
||||||
ARCH: amd64
|
ARCH: amd64
|
||||||
R_VERSION: 4.5.3
|
R_VERSION: 4.5.3
|
||||||
IMG: alpine:3.23
|
IMG: alpine:3.23
|
||||||
OS_ID: alpine323
|
OS_ID: alpine323
|
||||||
PROCESS_NEW: "FALSE"
|
PROCESS_NEW: 'FALSE'
|
||||||
- OS: alpine-323
|
- OS: alpine-323
|
||||||
ARCH: arm64
|
ARCH: arm64
|
||||||
R_VERSION: 4.5.3
|
R_VERSION: 4.5.3
|
||||||
IMG: alpine:3.23
|
IMG: alpine:3.23
|
||||||
OS_ID: alpine323
|
OS_ID: alpine323
|
||||||
PROCESS_NEW: "FALSE"
|
PROCESS_NEW: 'FALSE'
|
||||||
- OS: alpine-324
|
- OS: alpine-324
|
||||||
ARCH: amd64
|
ARCH: amd64
|
||||||
R_VERSION: 4.5.3
|
R_VERSION: 4.5.3
|
||||||
IMG: alpine:3.24
|
IMG: alpine:3.24
|
||||||
OS_ID: alpine324
|
OS_ID: alpine324
|
||||||
PROCESS_NEW: "FALSE"
|
PROCESS_NEW: 'FALSE'
|
||||||
- OS: alpine-324
|
- OS: alpine-324
|
||||||
ARCH: arm64
|
ARCH: arm64
|
||||||
R_VERSION: 4.5.3
|
R_VERSION: 4.5.3
|
||||||
IMG: alpine:3.24
|
IMG: alpine:3.24
|
||||||
OS_ID: alpine324
|
OS_ID: alpine324
|
||||||
PROCESS_NEW: "FALSE"
|
PROCESS_NEW: 'FALSE'
|
||||||
- OS: redhat-8
|
- OS: redhat-8
|
||||||
ARCH: amd64
|
ARCH: amd64
|
||||||
R_VERSION: 4.4.3
|
R_VERSION: 4.4.3
|
||||||
IMG: redhat:8
|
IMG: redhat:8
|
||||||
OS_ID: rhel8
|
OS_ID: rhel8
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: redhat-8
|
- OS: redhat-8
|
||||||
ARCH: arm64
|
ARCH: arm64
|
||||||
R_VERSION: 4.4.3
|
R_VERSION: 4.4.3
|
||||||
IMG: redhat:8
|
IMG: redhat:8
|
||||||
OS_ID: rhel8
|
OS_ID: rhel8
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: redhat-9
|
- OS: redhat-9
|
||||||
ARCH: amd64
|
ARCH: amd64
|
||||||
R_VERSION: 4.4.3
|
R_VERSION: 4.4.3
|
||||||
IMG: redhat:9
|
IMG: redhat:9
|
||||||
OS_ID: rhel9
|
OS_ID: rhel9
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: redhat-9
|
- OS: redhat-9
|
||||||
ARCH: arm64
|
ARCH: arm64
|
||||||
R_VERSION: 4.4.3
|
R_VERSION: 4.4.3
|
||||||
IMG: redhat:9
|
IMG: redhat:9
|
||||||
OS_ID: rhel9
|
OS_ID: rhel9
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: redhat-10
|
- OS: redhat-10
|
||||||
ARCH: amd64
|
ARCH: amd64
|
||||||
R_VERSION: 4.5.3
|
R_VERSION: 4.5.3
|
||||||
IMG: redhat:10
|
IMG: redhat:10
|
||||||
OS_ID: rhel10
|
OS_ID: rhel10
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: redhat-10
|
- OS: redhat-10
|
||||||
ARCH: arm64
|
ARCH: arm64
|
||||||
R_VERSION: 4.5.3
|
R_VERSION: 4.5.3
|
||||||
IMG: redhat:10
|
IMG: redhat:10
|
||||||
OS_ID: rhel10
|
OS_ID: rhel10
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: ubuntu-2204
|
- OS: ubuntu-2204
|
||||||
ARCH: amd64
|
ARCH: amd64
|
||||||
R_VERSION: 4.4.3
|
R_VERSION: 4.4.3
|
||||||
IMG: ubuntu:jammy
|
IMG: ubuntu:jammy
|
||||||
OS_ID: jammy
|
OS_ID: jammy
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: ubuntu-2204
|
- OS: ubuntu-2204
|
||||||
ARCH: arm64
|
ARCH: arm64
|
||||||
R_VERSION: 4.4.3
|
R_VERSION: 4.4.3
|
||||||
IMG: ubuntu:jammy
|
IMG: ubuntu:jammy
|
||||||
OS_ID: jammy
|
OS_ID: jammy
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: ubuntu-2404
|
- OS: ubuntu-2404
|
||||||
ARCH: amd64
|
ARCH: amd64
|
||||||
R_VERSION: 4.4.3
|
R_VERSION: 4.4.3
|
||||||
IMG: ubuntu:noble
|
IMG: ubuntu:noble
|
||||||
OS_ID: noble
|
OS_ID: noble
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: ubuntu-2404
|
- OS: ubuntu-2404
|
||||||
ARCH: arm64
|
ARCH: arm64
|
||||||
R_VERSION: 4.4.3
|
R_VERSION: 4.4.3
|
||||||
IMG: ubuntu:noble
|
IMG: ubuntu:noble
|
||||||
OS_ID: noble
|
OS_ID: noble
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: ubuntu-2604
|
- OS: ubuntu-2604
|
||||||
ARCH: amd64
|
ARCH: amd64
|
||||||
R_VERSION: 4.5.3
|
R_VERSION: 4.5.3
|
||||||
IMG: ubuntu:resolute
|
IMG: ubuntu:resolute
|
||||||
OS_ID: resolute
|
OS_ID: resolute
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
- OS: ubuntu-2604
|
- OS: ubuntu-2604
|
||||||
ARCH: arm64
|
ARCH: arm64
|
||||||
R_VERSION: 4.5.3
|
R_VERSION: 4.5.3
|
||||||
IMG: ubuntu:resolute
|
IMG: ubuntu:resolute
|
||||||
OS_ID: resolute
|
OS_ID: resolute
|
||||||
PROCESS_NEW: "TRUE"
|
PROCESS_NEW: 'TRUE'
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: 'Processing Updates'
|
- name: 'Processing Updates'
|
||||||
|
|
@ -182,8 +182,10 @@ steps:
|
||||||
NTFY_AUTH: TRUE
|
NTFY_AUTH: TRUE
|
||||||
NTFY_PASSWORD:
|
NTFY_PASSWORD:
|
||||||
from_secret: ntfy_token
|
from_secret: ntfy_token
|
||||||
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
# set the location of uvr's caches, which persist the R package
|
||||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
# dependencies needed to install the packages themselves
|
||||||
|
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||||
|
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||||
R_VERSION: ${R_VERSION}
|
R_VERSION: ${R_VERSION}
|
||||||
CCACHE_DIR: /mnt/cache/ccache
|
CCACHE_DIR: /mnt/cache/ccache
|
||||||
|
|
@ -194,10 +196,10 @@ steps:
|
||||||
INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
|
INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
|
||||||
commands:
|
commands:
|
||||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
|
||||||
|
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
|
||||||
# rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
|
# rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
|
||||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||||
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
|
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
|
||||||
|
|
|
||||||
|
|
@ -142,7 +142,9 @@ steps:
|
||||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
- mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs
|
- mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs
|
||||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))'
|
# bincraft is taken from the default branch here (not the latest release
|
||||||
|
# tag), matching what the pak call did before the uvr migration.
|
||||||
|
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft httr2 jsonlite
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")'
|
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")'
|
||||||
backend_options:
|
backend_options:
|
||||||
docker:
|
docker:
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,7 @@ steps:
|
||||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
- mkdir -p /mnt/cache/R-pkgs
|
- mkdir -p /mnt/cache/R-pkgs
|
||||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))'
|
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
|
||||||
- /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue
|
- /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue
|
||||||
- /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue
|
- /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue
|
||||||
backend_options:
|
backend_options:
|
||||||
|
|
|
||||||
|
|
@ -141,7 +141,8 @@ steps:
|
||||||
FORGEJO_TOKEN:
|
FORGEJO_TOKEN:
|
||||||
from_secret: FORGEJO_TOKEN
|
from_secret: FORGEJO_TOKEN
|
||||||
GIT_USER: pat-s
|
GIT_USER: pat-s
|
||||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||||
|
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||||
R_VERSION: ${R_VERSION}
|
R_VERSION: ${R_VERSION}
|
||||||
CCACHE_DIR: /mnt/cache/ccache
|
CCACHE_DIR: /mnt/cache/ccache
|
||||||
|
|
@ -150,12 +151,12 @@ steps:
|
||||||
NCPUS: 2
|
NCPUS: 2
|
||||||
commands:
|
commands:
|
||||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")'
|
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2
|
||||||
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
|
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
|
||||||
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1
|
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1
|
||||||
backend_options:
|
backend_options:
|
||||||
|
|
|
||||||
19
README.md
19
README.md
|
|
@ -112,9 +112,9 @@ Processing updates with an existing database file takes around 5 minutes.
|
||||||
|
|
||||||
### Inferring System Dependencies
|
### Inferring System Dependencies
|
||||||
|
|
||||||
R package dependencies and their system dependencies are installed through {pak}.
|
R package dependencies and their system dependencies are installed through [uvr](https://github.com/nbafrank/uvr).
|
||||||
{pak} allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`.
|
`uvr` allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`.
|
||||||
Additionally, it automatically infers package dependencies using JSON rules from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements).
|
Additionally, it automatically infers package dependencies using JSON rules vendored from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements).
|
||||||
Not all R packages specify required system dependencies in their DESCRIPTION file, and not all listed dependencies have existing rules in `rstudio/r-system-requirements`.
|
Not all R packages specify required system dependencies in their DESCRIPTION file, and not all listed dependencies have existing rules in `rstudio/r-system-requirements`.
|
||||||
For Alpine, no rules existed until recently, establishing a foundation for semi-automated package installation on Alpine Linux.
|
For Alpine, no rules existed until recently, establishing a foundation for semi-automated package installation on Alpine Linux.
|
||||||
|
|
||||||
|
|
@ -152,12 +152,13 @@ A Shiny dashboard providing a search functionality of the database and grouped s
|
||||||
|
|
||||||
Is supported by writing `Meta/archive.rds` during each package index update, listing all available archived packages.
|
Is supported by writing `Meta/archive.rds` during each package index update, listing all available archived packages.
|
||||||
|
|
||||||
### `pak::pak(package@version)`
|
### `uvr add package@version`
|
||||||
|
|
||||||
`pak` searches for `Archive/<package>` and can install all versions it finds.
|
Clients that resolve archived versions search for `Archive/<package>` and can install all versions they find.
|
||||||
|
This holds for `uvr` as well as for older `pak`-based clients.
|
||||||
|
|
||||||
Ensure to use a clean cache if other repositories have been used previously.
|
Ensure to use a clean cache if other repositories have been used previously.
|
||||||
If in doubt or when testing, call `pak::meta_clean(force = TRUE)`.
|
If in doubt or when testing, call `uvr cache clean`.
|
||||||
|
|
||||||
## Lessons Learned
|
## Lessons Learned
|
||||||
|
|
||||||
|
|
@ -180,7 +181,7 @@ If in doubt or when testing, call `pak::meta_clean(force = TRUE)`.
|
||||||
|
|
||||||
## URL Composition and Platform Identifiers
|
## URL Composition and Platform Identifiers
|
||||||
|
|
||||||
Platform identifiers have been aligned with those used in <https://github.com/rstudio/r-system-requirements> to ensure proper recognition by the automatic syslib dependency installer of `pak`, specifically via the environment variable `PKG_SYSREQS_PLATFORM`:
|
Platform identifiers have been aligned with those used in <https://github.com/rstudio/r-system-requirements>, which `uvr` vendors and uses for its automatic syslib dependency installer (`uvr sync --install-system-deps`):
|
||||||
|
|
||||||
- redhat-9
|
- redhat-9
|
||||||
- redhat-8
|
- redhat-8
|
||||||
|
|
@ -237,8 +238,8 @@ internal error 1 in memDecompress
|
||||||
Solution:
|
Solution:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
rm -rf /mnt/cache/R-pkgs/pak /mnt/cache/pkgcache/ /root/.cache/R/
|
uvr cache clean
|
||||||
R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
rm -rf /mnt/cache/uvr /root/.cache/R/
|
||||||
```
|
```
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
|
|
||||||
|
|
@ -30,8 +30,9 @@
|
||||||
# from_secret: HETZNER_S3_SECRET_KEY_K3S
|
# from_secret: HETZNER_S3_SECRET_KEY_K3S
|
||||||
# # normal env vars
|
# # normal env vars
|
||||||
# GIT_USER: pat-s
|
# GIT_USER: pat-s
|
||||||
# # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
# # set the location of uvr's caches, which persist the R package dependencies needed to install the packages themselves
|
||||||
# # R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
# # UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||||
|
# # UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||||
# R_LIBS_USER: /mnt/cache/R-pkgs
|
# R_LIBS_USER: /mnt/cache/R-pkgs
|
||||||
# CCACHE_DIR: /mnt/cache/ccache
|
# CCACHE_DIR: /mnt/cache/ccache
|
||||||
# volumes:
|
# volumes:
|
||||||
|
|
@ -39,9 +40,8 @@
|
||||||
# commands:
|
# commands:
|
||||||
# - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
# - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||||
# - mkdir -p /mnt/cache/R-pkgs
|
# - mkdir -p /mnt/cache/R-pkgs
|
||||||
# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/pkgcache
|
# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/uvr
|
||||||
# - /opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
# - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft
|
||||||
# - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git", dependencies = TRUE)'
|
|
||||||
# - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
# - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||||
# # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine322', 'amd64'); saveRDS(pkgs, '/mnt/cache/pkgs_amd64.rds')"
|
# # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine322', 'amd64'); saveRDS(pkgs, '/mnt/cache/pkgs_amd64.rds')"
|
||||||
# - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine323', 'arm64'); saveRDS(pkgs, '/mnt/cache/pkgs_arm64.rds')"
|
# - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine323', 'arm64'); saveRDS(pkgs, '/mnt/cache/pkgs_arm64.rds')"
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,15 @@
|
||||||
FROM devxygmbh/r-alpine:4.4-3.20 AS build
|
FROM devxygmbh/r-alpine:4.4-3.20 AS build
|
||||||
|
|
||||||
# ARG GITHUB_PAT
|
# ARG GITHUB_PAT
|
||||||
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
RUN apk add --no-cache curl ca-certificates \
|
||||||
|
&& curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh \
|
||||||
|
| UVR_INSTALL_DIR=/usr/local/bin sh
|
||||||
|
|
||||||
|
# One uvr project drives both dependency installs below. UVR_LIBRARY points the
|
||||||
|
# syncs at the image's R library instead of the project-local .uvr/library/, so
|
||||||
|
# `Rscript app.R` finds the packages without a uvr-aware .Rprofile.
|
||||||
|
ENV UVR_LIBRARY=/usr/lib/R/library
|
||||||
|
RUN mkdir -p /uvr && cd /uvr && uvr init --here
|
||||||
|
|
||||||
COPY --link ./DESCRIPTION .
|
COPY --link ./DESCRIPTION .
|
||||||
COPY --link ./R ./R
|
COPY --link ./R ./R
|
||||||
|
|
@ -11,13 +19,19 @@ COPY --link ./shiny/app.R /app/app.R
|
||||||
|
|
||||||
RUN ls -la
|
RUN ls -la
|
||||||
|
|
||||||
# install R package deps
|
# install R package deps. uvr has no equivalent of bare `pak::pak()`, which
|
||||||
RUN R -q -e 'pak::pak()'
|
# reads the DESCRIPTION in the working directory, so extract the dependency
|
||||||
|
# names and hand them to `uvr add`.
|
||||||
|
RUN R -q --no-echo -e "d <- read.dcf('DESCRIPTION'); f <- intersect(colnames(d), c('Depends', 'Imports', 'LinkingTo')); p <- trimws(sub('[(].*', '', unlist(strsplit(paste(d[, f], collapse = ','), ',')))); writeLines(setdiff(p[nzchar(p)], c('R', rownames(installed.packages()))), '/tmp/deps.txt')" \
|
||||||
|
&& cd /uvr && xargs -r uvr add --no-install < /tmp/deps.txt \
|
||||||
|
&& uvr sync --install-system-deps
|
||||||
|
|
||||||
RUN R CMD INSTALL --no-docs --without-keep.source .
|
RUN R CMD INSTALL --no-docs --without-keep.source .
|
||||||
|
|
||||||
# install shiny app deps
|
# install shiny app deps
|
||||||
RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); pak::pak(pkgs)"
|
RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); writeLines(pkgs, '/tmp/app-deps.txt')" \
|
||||||
|
&& cd /uvr && xargs -r uvr add --no-install < /tmp/app-deps.txt \
|
||||||
|
&& uvr sync --install-system-deps
|
||||||
|
|
||||||
ENV PGPASS=""
|
ENV PGPASS=""
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,10 @@ ARG CACHEBUST
|
||||||
WORKDIR /work
|
WORKDIR /work
|
||||||
COPY build-one.R /work/build-one.R
|
COPY build-one.R /work/build-one.R
|
||||||
# Resolve and install the latest bincraft release dynamically (no hardcoded pin).
|
# Resolve and install the latest bincraft release dynamically (no hardcoded pin).
|
||||||
|
# uvr-install.sh lands under /work/local/ because install-bincraft.R looks for it
|
||||||
|
# there when the working directory is not a repo checkout.
|
||||||
COPY install-bincraft.R /work/install-bincraft.R
|
COPY install-bincraft.R /work/install-bincraft.R
|
||||||
|
COPY uvr-install.sh /work/local/uvr-install.sh
|
||||||
# Ship the patch registry so build-one.R's `patches = "local/patches"` resolves
|
# Ship the patch registry so build-one.R's `patches = "local/patches"` resolves
|
||||||
# (build context is `local/`, CWD is /work).
|
# (build context is `local/`, CWD is /work).
|
||||||
COPY patches /work/local/patches
|
COPY patches /work/local/patches
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,14 @@
|
||||||
docker run --rm -it --platform linux/arm64 alpine sh
|
docker run --rm -it --platform linux/arm64 alpine sh
|
||||||
|
|
||||||
apk add --no-cache R R-dev g++
|
apk add --no-cache R R-dev g++ curl ca-certificates
|
||||||
|
|
||||||
R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | UVR_INSTALL_DIR=/usr/local/bin sh
|
||||||
|
|
||||||
R -q -e 'pak::pak(c("gert", "purrr"))'
|
# `uvr add` always writes to .uvr/library/; only `uvr sync` honours UVR_LIBRARY,
|
||||||
|
# so add without installing and let the sync place the packages.
|
||||||
|
export UVR_LIBRARY=/usr/lib/R/library
|
||||||
|
mkdir -p /uvr && cd /uvr && uvr init --here
|
||||||
|
uvr add --no-install gert purrr && uvr sync --install-system-deps
|
||||||
|
|
||||||
R
|
R
|
||||||
|
|
||||||
|
|
@ -22,14 +26,20 @@ unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE)
|
||||||
tag <- all_tags$name
|
tag <- all_tags$name
|
||||||
package_name <- rep(package_name, length(tag))
|
package_name <- rep(package_name, length(tag))
|
||||||
|
|
||||||
|
# uvr has no `pak::local_install_deps()`; read the DESCRIPTION of the checkout
|
||||||
|
# and `uvr add` the dependency names instead.
|
||||||
purrr::walk2(package_name[1], tag, \(x, y) {
|
purrr::walk2(package_name[1], tag, \(x, y) {
|
||||||
print(y)
|
print(y)
|
||||||
system("git config --global advice.detachedHead false")
|
system("git config --global advice.detachedHead false")
|
||||||
|
src <- sprintf("/tmp/%s_%s", x[1], y)
|
||||||
system2("git", args = c(
|
system2("git", args = c(
|
||||||
"clone", "-q", sprintf("--branch=%s", tail(y, 1)),
|
"clone", "-q", sprintf("--branch=%s", tail(y, 1)),
|
||||||
sprintf("https://github.com/cran/%s", x[1]), sprintf("/tmp/%s_%s", x[1], y)
|
sprintf("https://github.com/cran/%s", x[1]), src
|
||||||
))
|
))
|
||||||
pak::local_install_deps(sprintf("/tmp/%s_%s", x[1], y))
|
d <- read.dcf(file.path(src, "DESCRIPTION"))
|
||||||
|
f <- intersect(colnames(d), c("Depends", "Imports", "LinkingTo"))
|
||||||
|
deps <- trimws(sub("[(].*", "", unlist(strsplit(paste(d[, f], collapse = ","), ","))))
|
||||||
|
deps <- setdiff(deps[nzchar(deps)], "R")
|
||||||
|
system2("uvr", c("add", "--no-install", deps))
|
||||||
|
system2("uvr", c("sync", "--install-system-deps"))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,9 @@ package_cache_files <- c(
|
||||||
"/mnt/cache/packages/s3_cache.rds"
|
"/mnt/cache/packages/s3_cache.rds"
|
||||||
)
|
)
|
||||||
if (!all(file.exists(package_cache_files))) {
|
if (!all(file.exists(package_cache_files))) {
|
||||||
message("Package snapshot missing from cache; recomputing via packages-to-build.R")
|
message(
|
||||||
|
"Package snapshot missing from cache; recomputing via packages-to-build.R"
|
||||||
|
)
|
||||||
dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
|
dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
|
||||||
save_rds_atomic <- function(obj, path) {
|
save_rds_atomic <- function(obj, path) {
|
||||||
tmp <- paste0(path, ".tmp.", Sys.getpid())
|
tmp <- paste0(path, ".tmp.", Sys.getpid())
|
||||||
|
|
@ -36,7 +38,10 @@ if (!all(file.exists(package_cache_files))) {
|
||||||
}
|
}
|
||||||
source(file.path("local", "packages-to-build.R"))
|
source(file.path("local", "packages-to-build.R"))
|
||||||
save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds")
|
save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds")
|
||||||
save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
save_rds_atomic(
|
||||||
|
pkgs[r_minor_sensitive == TRUE],
|
||||||
|
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds"
|
||||||
|
)
|
||||||
message("Package snapshot recomputed.")
|
message("Package snapshot recomputed.")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -112,20 +117,22 @@ built <- DBI::dbGetQuery(
|
||||||
)
|
)
|
||||||
DBI::dbDisconnect(con)
|
DBI::dbDisconnect(con)
|
||||||
before <- nrow(chunk)
|
before <- nrow(chunk)
|
||||||
chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ]
|
chunk <- chunk[
|
||||||
sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk))
|
!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag),
|
||||||
|
]
|
||||||
|
sprintf(
|
||||||
|
"Skipped %d already-attempted package versions; %d remaining for this job",
|
||||||
|
before - nrow(chunk),
|
||||||
|
nrow(chunk)
|
||||||
|
)
|
||||||
|
|
||||||
# Read pre-computed S3 listing from install-deps step
|
# Read pre-computed S3 listing from install-deps step
|
||||||
# This avoids loading s3fs/reticulate/Python in the build container,
|
# This avoids loading s3fs/reticulate/Python in the build container,
|
||||||
# saving significant memory for pak subprocess forks
|
# saving significant memory for the dependency-installer subprocesses
|
||||||
s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
||||||
sprintf("S3 cache: %s files", length(s3_cache))
|
sprintf("S3 cache: %s files", length(s3_cache))
|
||||||
|
|
||||||
n <- nrow(chunk)
|
n <- nrow(chunk)
|
||||||
# Every `trim_every` packages, bound the pkgcache _metadata dir so a full-platform
|
|
||||||
# run does not accumulate thousands of ~70 MB snapshots and fill the host disk.
|
|
||||||
# No-op on amd64 (R_PKG_CACHE_DIR is empty / cache not persisted).
|
|
||||||
trim_every <- 25L
|
|
||||||
mapply(
|
mapply(
|
||||||
function(pkg, ver, sens, i) {
|
function(pkg, ver, sens, i) {
|
||||||
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
||||||
|
|
@ -151,12 +158,6 @@ mapply(
|
||||||
upload = TRUE,
|
upload = TRUE,
|
||||||
store_build_metadata = TRUE
|
store_build_metadata = TRUE
|
||||||
)
|
)
|
||||||
if (i %% trim_every == 0L) {
|
|
||||||
removed <- trim_pkgcache_metadata()
|
|
||||||
if (removed > 0L) {
|
|
||||||
cat(sprintf(" [pkgcache trim] removed %d stale _metadata entries\n", removed))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
chunk$Package,
|
chunk$Package,
|
||||||
chunk$Version,
|
chunk$Version,
|
||||||
|
|
|
||||||
|
|
@ -11,10 +11,11 @@
|
||||||
#
|
#
|
||||||
# How it works: list the remote tags with `git ls-remote` (no token needed for
|
# How it works: list the remote tags with `git ls-remote` (no token needed for
|
||||||
# the public repo), keep the `vX.Y.Z` release tags, pick the highest version,
|
# the public repo), keep the `vX.Y.Z` release tags, pick the highest version,
|
||||||
# and install it with pak. pak is idempotent on the git ref, so re-running keeps
|
# and install it with uvr via `local/uvr-install.sh`. uvr is idempotent on the
|
||||||
# the package when it is already current and only updates when a newer tag ships.
|
# git ref, so re-running keeps the package when it is already current and only
|
||||||
# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is
|
# updates when a newer tag ships. Filtering/sorting is done in R (not via git's
|
||||||
# identical across git versions and `system2()` argument handling.
|
# `--sort`/refspec) so behaviour is identical across git versions and
|
||||||
|
# `system2()` argument handling.
|
||||||
|
|
||||||
repo_url <- Sys.getenv(
|
repo_url <- Sys.getenv(
|
||||||
"BINCRAFT_GIT_URL",
|
"BINCRAFT_GIT_URL",
|
||||||
|
|
@ -44,7 +45,43 @@ latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][
|
||||||
]
|
]
|
||||||
|
|
||||||
message(sprintf("Installing latest bincraft release: %s", latest))
|
message(sprintf("Installing latest bincraft release: %s", latest))
|
||||||
pak::pak(sprintf("git::%s@%s", repo_url, latest))
|
|
||||||
|
# uvr addresses Forgejo repos as `forgejo::host/owner/repo@ref` rather than as a
|
||||||
|
# git URL, so drop the scheme and the trailing `.git` from `repo_url`.
|
||||||
|
spec <- sprintf(
|
||||||
|
"forgejo::%s@%s",
|
||||||
|
sub("\\.git$", "", sub("^[a-z]+://", "", repo_url)),
|
||||||
|
latest
|
||||||
|
)
|
||||||
|
|
||||||
|
# `local/uvr-install.sh` when run from the repo root, `/work/local/` in the
|
||||||
|
# build-one image, which copies the two scripts into a flatter layout.
|
||||||
|
helper <- Sys.getenv("UVR_INSTALL_SH", unset = "")
|
||||||
|
if (!nzchar(helper)) {
|
||||||
|
candidates <- c("local/uvr-install.sh", "/work/local/uvr-install.sh")
|
||||||
|
found <- candidates[file.exists(candidates)]
|
||||||
|
if (length(found) == 0L) {
|
||||||
|
stop("Could not locate uvr-install.sh; set UVR_INSTALL_SH", call. = FALSE)
|
||||||
|
}
|
||||||
|
helper <- found[1L]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Point uvr at the R running this script and at the library it would install
|
||||||
|
# into, so the per-R-minor passes in the build pipelines (which call a different
|
||||||
|
# Rscript with R_LIBS_USER pointed elsewhere) target their own R and library.
|
||||||
|
Sys.setenv(
|
||||||
|
UVR_R_BIN = file.path(R.home("bin"), "R"),
|
||||||
|
UVR_TARGET_LIB = .libPaths()[1L]
|
||||||
|
)
|
||||||
|
|
||||||
|
status <- system2(helper, shQuote(spec))
|
||||||
|
if (!identical(status, 0L)) {
|
||||||
|
stop(
|
||||||
|
sprintf("uvr failed to install %s (exit %s)", spec, status),
|
||||||
|
call. = FALSE
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
message(sprintf(
|
message(sprintf(
|
||||||
"bincraft %s installed (%s)",
|
"bincraft %s installed (%s)",
|
||||||
as.character(utils::packageVersion("bincraft")),
|
as.character(utils::packageVersion("bincraft")),
|
||||||
|
|
|
||||||
|
|
@ -15,15 +15,15 @@ suppressPackageStartupMessages(library(data.table))
|
||||||
# Sys.setenv("OS_VERSION" = "3.22")
|
# Sys.setenv("OS_VERSION" = "3.22")
|
||||||
# Sys.setenv("ARCH" = "arm64")
|
# Sys.setenv("ARCH" = "arm64")
|
||||||
|
|
||||||
arch = Sys.getenv("ARCH")
|
arch <- Sys.getenv("ARCH")
|
||||||
# target: alpine-322, ubuntu-2404, redhat-9, etc.
|
# target: alpine-322, ubuntu-2404, redhat-9, etc.
|
||||||
platform = paste(
|
platform <- paste(
|
||||||
Sys.getenv("OS"),
|
Sys.getenv("OS"),
|
||||||
gsub("[.]", "", Sys.getenv("OS_VERSION")),
|
gsub("[.]", "", Sys.getenv("OS_VERSION")),
|
||||||
sep = "-"
|
sep = "-"
|
||||||
)
|
)
|
||||||
# Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10")
|
# Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10")
|
||||||
codename = bincraft::set_codename(NULL)
|
codename <- bincraft::set_codename(NULL)
|
||||||
|
|
||||||
con <- DBI::dbConnect(
|
con <- DBI::dbConnect(
|
||||||
RPostgres::Postgres(),
|
RPostgres::Postgres(),
|
||||||
|
|
@ -35,8 +35,8 @@ con <- DBI::dbConnect(
|
||||||
sslmode = "require"
|
sslmode = "require"
|
||||||
)
|
)
|
||||||
|
|
||||||
cran_archive = tools::CRAN_archive_db()
|
cran_archive <- tools::CRAN_archive_db()
|
||||||
cran_release = tools::CRAN_package_db()
|
cran_release <- tools::CRAN_package_db()
|
||||||
# Subset cran_archive to only those packages
|
# Subset cran_archive to only those packages
|
||||||
cran_archive_in_release <- cran_archive[
|
cran_archive_in_release <- cran_archive[
|
||||||
names(cran_archive) %in% cran_release$Package
|
names(cran_archive) %in% cran_release$Package
|
||||||
|
|
@ -84,13 +84,14 @@ s3fs::s3_file_system(
|
||||||
region_name = "eu-central-003",
|
region_name = "eu-central-003",
|
||||||
refresh = TRUE
|
refresh = TRUE
|
||||||
)
|
)
|
||||||
s3_pkgs = s3fs::s3_dir_ls(
|
s3_pkgs <- s3fs::s3_dir_ls(
|
||||||
sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename),
|
sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename),
|
||||||
recurse = TRUE
|
recurse = TRUE
|
||||||
)
|
)
|
||||||
|
|
||||||
# Save the raw S3 file listing for the build step to use as s3_package_cache
|
# Save the raw S3 file listing for the build step to use as s3_package_cache
|
||||||
# This avoids loading s3fs/reticulate in the build container, saving memory for pak forks
|
# This avoids loading s3fs/reticulate in the build container, saving memory for
|
||||||
|
# the dependency-installer subprocesses
|
||||||
saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds")
|
saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds")
|
||||||
|
|
||||||
file_names <- basename(s3_pkgs)
|
file_names <- basename(s3_pkgs)
|
||||||
|
|
|
||||||
|
|
@ -31,42 +31,3 @@ parse_build_args <- function(args) {
|
||||||
ncpus = as.integer(pos[3L])
|
ncpus = as.integer(pos[3L])
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the
|
|
||||||
# "patched" repo mints a new content hash on every PACKAGES change, so each build
|
|
||||||
# writes a fresh ~70 MB _metadata/pkgs-<hash>.rds (+ patched-<hash>/) that is
|
|
||||||
# never reused. Keep the `keep` newest entries by mtime; only remove entries
|
|
||||||
# older than `min_age_secs`, so a concurrent split-job's in-flight files are
|
|
||||||
# never deleted (each build uses a unique hash, so aged entries are
|
|
||||||
# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are
|
|
||||||
# not matched and thus preserved. Returns the number of entries removed.
|
|
||||||
trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"),
|
|
||||||
keep = 20L,
|
|
||||||
min_age_secs = 600) {
|
|
||||||
meta <- file.path(cache_dir, "R", "pkgcache", "_metadata")
|
|
||||||
if (!nzchar(cache_dir) || !dir.exists(meta)) {
|
|
||||||
return(0L)
|
|
||||||
}
|
|
||||||
entries <- c(
|
|
||||||
Sys.glob(file.path(meta, "patched-*")),
|
|
||||||
Sys.glob(file.path(meta, "pkgs-*.rds"))
|
|
||||||
)
|
|
||||||
if (length(entries) == 0L) {
|
|
||||||
return(0L)
|
|
||||||
}
|
|
||||||
info <- file.info(entries)
|
|
||||||
order_new_first <- order(info$mtime, decreasing = TRUE)
|
|
||||||
ranked <- entries[order_new_first]
|
|
||||||
ranked_mtime <- info$mtime[order_new_first]
|
|
||||||
if (length(ranked) <= keep) {
|
|
||||||
return(0L)
|
|
||||||
}
|
|
||||||
candidates <- ranked[(keep + 1L):length(ranked)]
|
|
||||||
candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)])
|
|
||||||
removable <- candidates[candidate_age >= min_age_secs]
|
|
||||||
if (length(removable) == 0L) {
|
|
||||||
return(0L)
|
|
||||||
}
|
|
||||||
unlink(removable, recursive = TRUE, force = TRUE)
|
|
||||||
length(removable)
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,22 +1,28 @@
|
||||||
install.packages(
|
# Installs every CRAN package one by one and checks that it loads. Dependencies
|
||||||
"pak",
|
# go through uvr via local/uvr-install.sh, which bootstraps the uvr binary on
|
||||||
repos = sprintf(
|
# first use and installs into .libPaths()[1].
|
||||||
"https://r-lib.github.io/p/pak/stable/%s/%s/%s",
|
uvr_install <- function(pkg) {
|
||||||
.Platform$pkgType,
|
Sys.setenv(
|
||||||
R.Version()$os,
|
UVR_R_BIN = file.path(R.home("bin"), "R"),
|
||||||
R.Version()$arch
|
UVR_TARGET_LIB = .libPaths()[1L]
|
||||||
)
|
)
|
||||||
)
|
status <- system2("local/uvr-install.sh", shQuote(pkg))
|
||||||
|
if (!identical(status, 0L)) {
|
||||||
|
stop(
|
||||||
|
sprintf("uvr failed to install %s (exit %s)", pkg, status),
|
||||||
|
call. = FALSE
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Sys.setenv(PKG_SYSREQS = TRUE)
|
|
||||||
all_pkgs <- rownames(available.packages())
|
all_pkgs <- rownames(available.packages())
|
||||||
|
|
||||||
to_skip = c("ABRSQOL", "ACA", "ACE.CoCo")
|
to_skip <- c("ABRSQOL", "ACA", "ACE.CoCo")
|
||||||
all_pkgs = setdiff(all_pkgs, to_skip)
|
all_pkgs <- setdiff(all_pkgs, to_skip)
|
||||||
|
|
||||||
for (i in all_pkgs) {
|
for (i in all_pkgs) {
|
||||||
message(sprintf("\nInstalling %s", i))
|
message(sprintf("\nInstalling %s", i))
|
||||||
pak::pkg_install(i)
|
uvr_install(i)
|
||||||
library(i, character.only = TRUE)
|
library(i, character.only = TRUE)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -323,7 +329,7 @@ if (length(to_process) == 0) {
|
||||||
} else {
|
} else {
|
||||||
for (i in to_process) {
|
for (i in to_process) {
|
||||||
message(sprintf("\nInstalling %s", i))
|
message(sprintf("\nInstalling %s", i))
|
||||||
pak::pkg_install(i)
|
uvr_install(i)
|
||||||
library(i, character.only = TRUE)
|
library(i, character.only = TRUE)
|
||||||
}
|
}
|
||||||
# Update to_skip to include all up to the last processed
|
# Update to_skip to include all up to the last processed
|
||||||
|
|
|
||||||
|
|
@ -1,68 +0,0 @@
|
||||||
source(file.path("..", "r-minor-helpers.R"))
|
|
||||||
|
|
||||||
# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is
|
|
||||||
# set to `age_secs` in the past so we can exercise the age gate deterministically.
|
|
||||||
make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) {
|
|
||||||
root <- tempfile("pkgcache-")
|
|
||||||
meta <- file.path(root, "R", "pkgcache", "_metadata")
|
|
||||||
dir.create(meta, recursive = TRUE)
|
|
||||||
old <- Sys.time() - age_secs
|
|
||||||
mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p }
|
|
||||||
mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p }
|
|
||||||
for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i)))
|
|
||||||
for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i)))
|
|
||||||
if (keep_repos) {
|
|
||||||
mk_dir(file.path(meta, "CRAN-075c426938"))
|
|
||||||
mk_dir(file.path(meta, "BioCsoft-1ac964ed6c"))
|
|
||||||
mk_file(file.path(meta, "bioc-sysreqs.dcf.gz"))
|
|
||||||
mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive
|
|
||||||
}
|
|
||||||
root
|
|
||||||
}
|
|
||||||
|
|
||||||
n_churn <- function(root) {
|
|
||||||
meta <- file.path(root, "R", "pkgcache", "_metadata")
|
|
||||||
length(Sys.glob(file.path(meta, "patched-*"))) +
|
|
||||||
length(Sys.glob(file.path(meta, "pkgs-*.rds")))
|
|
||||||
}
|
|
||||||
|
|
||||||
test_that("empty cache_dir is a no-op", {
|
|
||||||
expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L)
|
|
||||||
})
|
|
||||||
|
|
||||||
test_that("missing _metadata dir is a no-op", {
|
|
||||||
expect_identical(
|
|
||||||
trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0),
|
|
||||||
0L
|
|
||||||
)
|
|
||||||
})
|
|
||||||
|
|
||||||
test_that("fewer than keep entries removes nothing", {
|
|
||||||
root <- make_meta(patched = 2L, pkgs = 2L)
|
|
||||||
expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L)
|
|
||||||
expect_identical(n_churn(root), 4L)
|
|
||||||
})
|
|
||||||
|
|
||||||
test_that("trims down to keep newest, leaving churn == keep", {
|
|
||||||
root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old
|
|
||||||
removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0)
|
|
||||||
expect_identical(removed, 40L)
|
|
||||||
expect_identical(n_churn(root), 20L)
|
|
||||||
})
|
|
||||||
|
|
||||||
test_that("entries younger than min_age_secs are protected", {
|
|
||||||
root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60)
|
|
||||||
# keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed
|
|
||||||
expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L)
|
|
||||||
expect_identical(n_churn(root), 30L)
|
|
||||||
})
|
|
||||||
|
|
||||||
test_that("stable repo dirs and pkg downloads are never touched", {
|
|
||||||
root <- make_meta(patched = 30L, pkgs = 30L)
|
|
||||||
trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0)
|
|
||||||
meta <- file.path(root, "R", "pkgcache", "_metadata")
|
|
||||||
expect_true(dir.exists(file.path(meta, "CRAN-075c426938")))
|
|
||||||
expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c")))
|
|
||||||
expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz")))
|
|
||||||
expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg")))
|
|
||||||
})
|
|
||||||
90
local/uvr-install.sh
Executable file
90
local/uvr-install.sh
Executable file
|
|
@ -0,0 +1,90 @@
|
||||||
|
#!/bin/sh
|
||||||
|
# Install R packages into the CI library with uvr (https://github.com/nbafrank/uvr).
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# local/uvr-install.sh httr2 jsonlite
|
||||||
|
# local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft@v4.4.3
|
||||||
|
#
|
||||||
|
# Replaces `pak::pak(...)`. uvr is project-scoped: `uvr add` refuses to run
|
||||||
|
# outside a project and always writes to `.uvr/library/`, and only
|
||||||
|
# `uvr sync --library` can target an existing library. The project is therefore
|
||||||
|
# minted in a scratch directory under TMPDIR and thrown away afterwards; that
|
||||||
|
# also keeps `uvr init`'s `.Rprofile` out of the repo checkout, where it would
|
||||||
|
# hijack `.libPaths()` for every other R call in the pipeline.
|
||||||
|
#
|
||||||
|
# Pruning is a no-op here: uvr disables it whenever `--library` is passed,
|
||||||
|
# precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds
|
||||||
|
# bincraft and its dependencies alongside whatever this script installs).
|
||||||
|
#
|
||||||
|
# System dependencies come from uvr's vendored r-system-requirements rules, so
|
||||||
|
# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed.
|
||||||
|
#
|
||||||
|
# Environment:
|
||||||
|
# UVR_R_BIN R interpreter to install for; set by install-bincraft.R so
|
||||||
|
# the per-R-minor passes target their own R, not the primary
|
||||||
|
# R_VERSION fallback interpreter selector (/opt/R/<version>/bin/R)
|
||||||
|
# UVR_TARGET_LIB target library; defaults to R_LIBS_USER, then to the
|
||||||
|
# active R's .libPaths()[1] (which is where pak wrote)
|
||||||
|
# UVR_INSTALL_DIR where the uvr binary lands (default /usr/local/bin)
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
# renovate: datasource=github-releases depName=nbafrank/uvr
|
||||||
|
UVR_PIN="v0.4.4"
|
||||||
|
|
||||||
|
if [ "$#" -eq 0 ]; then
|
||||||
|
echo "usage: $0 <pkg-spec>..." >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The build images keep R under /opt/R/<version> and off PATH. uvr resolves the
|
||||||
|
# interpreter via PATH and never downloads one unless `uvr r install` is run, so
|
||||||
|
# put the requested R first.
|
||||||
|
r_bin="${UVR_R_BIN:-}"
|
||||||
|
if [ -z "$r_bin" ] && [ -n "${R_VERSION:-}" ] && [ -x "/opt/R/${R_VERSION}/bin/R" ]; then
|
||||||
|
r_bin="/opt/R/${R_VERSION}/bin/R"
|
||||||
|
fi
|
||||||
|
if [ -n "$r_bin" ]; then
|
||||||
|
PATH="$(dirname "$r_bin"):$PATH"
|
||||||
|
export PATH
|
||||||
|
else
|
||||||
|
r_bin="$(command -v R)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
target_lib="${UVR_TARGET_LIB:-${R_LIBS_USER:-}}"
|
||||||
|
if [ -z "$target_lib" ]; then
|
||||||
|
target_lib="$("$r_bin" --no-echo --no-save -e 'cat(.libPaths()[1])')"
|
||||||
|
fi
|
||||||
|
if [ -z "$target_lib" ]; then
|
||||||
|
echo "error: could not determine a target library; set UVR_TARGET_LIB" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
mkdir -p "$target_lib"
|
||||||
|
|
||||||
|
# Pin the manifest to the active R so the lockfile's R stays in step with the
|
||||||
|
# library's R sentinel. Without that, uvr can decide the library is ABI-stale
|
||||||
|
# and wipe it -- and this target is shared with bincraft. uvr only discovers R
|
||||||
|
# via PATH/R_HOME (it does not scan /opt/R), so the R put on PATH above is the
|
||||||
|
# only candidate this constraint can resolve to.
|
||||||
|
# shellcheck disable=SC2016 # $major/$minor are R expressions, not shell vars
|
||||||
|
r_full="$("$r_bin" --no-echo --no-save -e 'cat(paste(R.version$major, R.version$minor, sep = "."))')"
|
||||||
|
|
||||||
|
install_dir="${UVR_INSTALL_DIR:-/usr/local/bin}"
|
||||||
|
uvr_bin="${install_dir}/uvr"
|
||||||
|
if [ ! -x "$uvr_bin" ]; then
|
||||||
|
echo "Bootstrapping uvr ${UVR_PIN} into ${install_dir}"
|
||||||
|
UVR_INSTALL_DIR="$install_dir" UVR_VERSION="$UVR_PIN" \
|
||||||
|
sh -c 'curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | sh'
|
||||||
|
fi
|
||||||
|
|
||||||
|
project_dir="${TMPDIR:-/tmp}/uvr-ci-$$"
|
||||||
|
rm -rf "$project_dir"
|
||||||
|
mkdir -p "$project_dir"
|
||||||
|
trap 'rm -rf "$project_dir"' EXIT
|
||||||
|
cd "$project_dir"
|
||||||
|
|
||||||
|
"$uvr_bin" init --here --r-version "$r_full"
|
||||||
|
# --no-install: resolve and lock only. The install happens in the sync below,
|
||||||
|
# which is the only command that honours --library.
|
||||||
|
"$uvr_bin" add --no-install "$@"
|
||||||
|
"$uvr_bin" sync --library "$target_lib" --install-system-deps
|
||||||
|
|
@ -1,7 +1,12 @@
|
||||||
{
|
{
|
||||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||||
"extends": ["local>devxy/renovate-config"],
|
"extends": ["local>devxy/renovate-config"],
|
||||||
"ignorePaths": ["docker/**", ".crow/process-updates.yaml", ".crow/build-all-versions.yaml", ".crow/weekly-rebuild-missing.yaml"],
|
"ignorePaths": [
|
||||||
|
"docker/**",
|
||||||
|
".crow/process-updates.yaml",
|
||||||
|
".crow/build-all-versions.yaml",
|
||||||
|
".crow/weekly-rebuild-missing.yaml"
|
||||||
|
],
|
||||||
"customManagers": [
|
"customManagers": [
|
||||||
{
|
{
|
||||||
"customType": "regex",
|
"customType": "regex",
|
||||||
|
|
@ -14,6 +19,13 @@
|
||||||
"packageNameTemplate": "https://codefloe.com/rpkgs/bincraft.git",
|
"packageNameTemplate": "https://codefloe.com/rpkgs/bincraft.git",
|
||||||
"datasourceTemplate": "git-tags",
|
"datasourceTemplate": "git-tags",
|
||||||
"extractVersionTemplate": "^v?(?<version>.+)$"
|
"extractVersionTemplate": "^v?(?<version>.+)$"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"customType": "regex",
|
||||||
|
"fileMatch": ["^local/uvr-install\\.sh$"],
|
||||||
|
"matchStrings": [
|
||||||
|
"# renovate: datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)\\s+UVR_PIN=\"(?<currentValue>[^\"]+)\""
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue