refactor: migrate package installation from pak to uvr #147
20 changed files with 326 additions and 245 deletions
|
|
@ -45,7 +45,7 @@ steps:
|
|||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /mnt/cache/R-pkgs
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))'
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT
|
||||
backend_options:
|
||||
kubernetes:
|
||||
|
|
|
|||
|
|
@ -10,22 +10,22 @@ variables:
|
|||
- arm64
|
||||
default: amd64
|
||||
OS:
|
||||
description: "Base OS image name."
|
||||
description: 'Base OS image name.'
|
||||
options:
|
||||
- alpine
|
||||
- redhat
|
||||
- ubuntu
|
||||
default: alpine
|
||||
OS_VERSION:
|
||||
description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)."
|
||||
description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).'
|
||||
options:
|
||||
- "3.24"
|
||||
- "8"
|
||||
- "9"
|
||||
- "10"
|
||||
- "jammy"
|
||||
- "noble"
|
||||
default: "3.24"
|
||||
- '3.24'
|
||||
- '8'
|
||||
- '9'
|
||||
- '10'
|
||||
- 'jammy'
|
||||
- 'noble'
|
||||
default: '3.24'
|
||||
R_VERSION:
|
||||
description: 'Primary R version under /opt/R.'
|
||||
options:
|
||||
|
|
@ -66,23 +66,24 @@ steps:
|
|||
from_secret: B2_S3_SECRET_KEY
|
||||
PGPASS:
|
||||
from_secret: PGPASS
|
||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
# Keep uvr's downloads and extracted-package entries on the persistent
|
||||
# volume instead of the container-local ~/.uvr default.
|
||||
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||
CCACHE_DIR: /mnt/cache/ccache
|
||||
volumes:
|
||||
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
||||
commands:
|
||||
# one-time full wipe to fix corrupted .so files from previous failed builds
|
||||
# - rm -rf /mnt/cache/R-pkgs
|
||||
# Clear churny pkgcache metadata left by a prior crashed run (the "patched"
|
||||
# repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*).
|
||||
# Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs.
|
||||
- rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true
|
||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
# Pin the same bincraft version the build steps use, so the precomputed
|
||||
# snapshot and the per-agent library stay consistent across the pipeline.
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")'
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||
- /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))"
|
||||
backend_options:
|
||||
docker:
|
||||
|
|
|
|||
|
|
@ -11,25 +11,25 @@ variables:
|
|||
- arm64
|
||||
default: amd64
|
||||
OS:
|
||||
description: "Base OS image name."
|
||||
description: 'Base OS image name.'
|
||||
options:
|
||||
- alpine
|
||||
- redhat
|
||||
- ubuntu
|
||||
default: alpine
|
||||
OS_VERSION:
|
||||
description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)."
|
||||
description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).'
|
||||
options:
|
||||
- "3.22"
|
||||
- "3.23"
|
||||
- "3.24"
|
||||
- "8"
|
||||
- "9"
|
||||
- "10"
|
||||
- "jammy"
|
||||
- "noble"
|
||||
- "resolute"
|
||||
default: "3.24"
|
||||
- '3.22'
|
||||
- '3.23'
|
||||
- '3.24'
|
||||
- '8'
|
||||
- '9'
|
||||
- '10'
|
||||
- 'jammy'
|
||||
- 'noble'
|
||||
- 'resolute'
|
||||
default: '3.24'
|
||||
R_VERSION:
|
||||
description: 'Primary R version under /opt/R.'
|
||||
options:
|
||||
|
|
@ -47,38 +47,49 @@ labels:
|
|||
platform: linux/${ARCH}
|
||||
group: rpkgs-${ARCH}
|
||||
|
||||
# Empty UVR_CACHE_DIR/UVR_PACKAGES_DIR fall back to uvr's container-local
|
||||
# ~/.uvr defaults; amd64 deliberately does not persist them (as with the
|
||||
# pkgcache dir it replaces), arm64 does.
|
||||
matrix:
|
||||
include:
|
||||
- ARCH: amd64
|
||||
R_PKG_CACHE_DIR: ''
|
||||
UVR_CACHE_DIR: ''
|
||||
UVR_PACKAGES_DIR: ''
|
||||
SPLIT_INTO: 4
|
||||
SPLIT_INDEX: 1
|
||||
- ARCH: amd64
|
||||
R_PKG_CACHE_DIR: ''
|
||||
UVR_CACHE_DIR: ''
|
||||
UVR_PACKAGES_DIR: ''
|
||||
SPLIT_INTO: 4
|
||||
SPLIT_INDEX: 2
|
||||
- ARCH: amd64
|
||||
R_PKG_CACHE_DIR: ''
|
||||
UVR_CACHE_DIR: ''
|
||||
UVR_PACKAGES_DIR: ''
|
||||
SPLIT_INTO: 4
|
||||
SPLIT_INDEX: 3
|
||||
- ARCH: amd64
|
||||
R_PKG_CACHE_DIR: ''
|
||||
UVR_CACHE_DIR: ''
|
||||
UVR_PACKAGES_DIR: ''
|
||||
SPLIT_INTO: 4
|
||||
SPLIT_INDEX: 4
|
||||
- ARCH: arm64
|
||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||
SPLIT_INTO: 4
|
||||
SPLIT_INDEX: 1
|
||||
- ARCH: arm64
|
||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||
SPLIT_INTO: 4
|
||||
SPLIT_INDEX: 2
|
||||
- ARCH: arm64
|
||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||
SPLIT_INTO: 4
|
||||
SPLIT_INDEX: 3
|
||||
- ARCH: arm64
|
||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||
SPLIT_INTO: 4
|
||||
SPLIT_INDEX: 4
|
||||
|
||||
|
|
@ -107,8 +118,10 @@ steps:
|
|||
from_secret: GITHUB_PAT
|
||||
# normal env vars
|
||||
GIT_USER: pat-s
|
||||
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
||||
R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR}
|
||||
# set the location of uvr's caches, which persist the R package
|
||||
# dependencies needed to install the packages themselves
|
||||
UVR_CACHE_DIR: ${UVR_CACHE_DIR}
|
||||
UVR_PACKAGES_DIR: ${UVR_PACKAGES_DIR}
|
||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
CCACHE_DIR: /mnt/cache/ccache
|
||||
NCPUS: 2
|
||||
|
|
@ -116,12 +129,7 @@ steps:
|
|||
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
# Clear churny pkgcache metadata left by a prior crashed run (the "patched"
|
||||
# repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*).
|
||||
# Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. Within-run
|
||||
# growth is bounded separately by trim_pkgcache_metadata() in build-all.R.
|
||||
- rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true
|
||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
# The primary pass must not rely on build-all-versions-install-deps having
|
||||
# run on *this* agent: depends_on only orders the steps, but the cache
|
||||
# volume is per-agent, so a job landing on an agent where install-deps did
|
||||
|
|
|
|||
|
|
@ -49,109 +49,109 @@ matrix:
|
|||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.22
|
||||
OS_ID: alpine322
|
||||
PROCESS_NEW: "FALSE"
|
||||
PROCESS_NEW: 'FALSE'
|
||||
- OS: alpine-322
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
OS_ID: alpine322
|
||||
PROCESS_NEW: "FALSE"
|
||||
PROCESS_NEW: 'FALSE'
|
||||
- OS: alpine-323
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
OS_ID: alpine323
|
||||
PROCESS_NEW: "FALSE"
|
||||
PROCESS_NEW: 'FALSE'
|
||||
- OS: alpine-323
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.23
|
||||
OS_ID: alpine323
|
||||
PROCESS_NEW: "FALSE"
|
||||
PROCESS_NEW: 'FALSE'
|
||||
- OS: alpine-324
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
OS_ID: alpine324
|
||||
PROCESS_NEW: "FALSE"
|
||||
PROCESS_NEW: 'FALSE'
|
||||
- OS: alpine-324
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: alpine:3.24
|
||||
OS_ID: alpine324
|
||||
PROCESS_NEW: "FALSE"
|
||||
PROCESS_NEW: 'FALSE'
|
||||
- OS: redhat-8
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
OS_ID: rhel8
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: redhat-8
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:8
|
||||
OS_ID: rhel8
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: redhat-9
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
OS_ID: rhel9
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: redhat-9
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: redhat:9
|
||||
OS_ID: rhel9
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: redhat-10
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
OS_ID: rhel10
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: redhat-10
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: redhat:10
|
||||
OS_ID: rhel10
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: ubuntu-2204
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
OS_ID: jammy
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: ubuntu-2204
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:jammy
|
||||
OS_ID: jammy
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: ubuntu-2404
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
OS_ID: noble
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: ubuntu-2404
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.4.3
|
||||
IMG: ubuntu:noble
|
||||
OS_ID: noble
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: ubuntu-2604
|
||||
ARCH: amd64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: ubuntu:resolute
|
||||
OS_ID: resolute
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
- OS: ubuntu-2604
|
||||
ARCH: arm64
|
||||
R_VERSION: 4.5.3
|
||||
IMG: ubuntu:resolute
|
||||
OS_ID: resolute
|
||||
PROCESS_NEW: "TRUE"
|
||||
PROCESS_NEW: 'TRUE'
|
||||
|
||||
steps:
|
||||
- name: 'Processing Updates'
|
||||
|
|
@ -182,8 +182,10 @@ steps:
|
|||
NTFY_AUTH: TRUE
|
||||
NTFY_PASSWORD:
|
||||
from_secret: ntfy_token
|
||||
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
# set the location of uvr's caches, which persist the R package
|
||||
# dependencies needed to install the packages themselves
|
||||
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
R_VERSION: ${R_VERSION}
|
||||
CCACHE_DIR: /mnt/cache/ccache
|
||||
|
|
@ -194,10 +196,10 @@ steps:
|
|||
INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
|
||||
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
# rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run
|
||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
|
||||
|
|
|
|||
|
|
@ -142,7 +142,8 @@ steps:
|
|||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))'
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")'
|
||||
backend_options:
|
||||
docker:
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ steps:
|
|||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /mnt/cache/R-pkgs
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))'
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue
|
||||
backend_options:
|
||||
|
|
|
|||
|
|
@ -141,7 +141,8 @@ steps:
|
|||
FORGEJO_TOKEN:
|
||||
from_secret: FORGEJO_TOKEN
|
||||
GIT_USER: pat-s
|
||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||
UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
R_VERSION: ${R_VERSION}
|
||||
CCACHE_DIR: /mnt/cache/ccache
|
||||
|
|
@ -150,12 +151,12 @@ steps:
|
|||
NCPUS: 2
|
||||
commands:
|
||||
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
- mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-*
|
||||
- /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")'
|
||||
- UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2
|
||||
- /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")'
|
||||
- $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1
|
||||
backend_options:
|
||||
|
|
|
|||
19
README.md
19
README.md
|
|
@ -112,9 +112,9 @@ Processing updates with an existing database file takes around 5 minutes.
|
|||
|
||||
### Inferring System Dependencies
|
||||
|
||||
R package dependencies and their system dependencies are installed through {pak}.
|
||||
{pak} allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`.
|
||||
Additionally, it automatically infers package dependencies using JSON rules from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements).
|
||||
R package dependencies and their system dependencies are installed through [uvr](https://github.com/nbafrank/uvr).
|
||||
`uvr` allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`.
|
||||
Additionally, it automatically infers package dependencies using JSON rules vendored from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements).
|
||||
Not all R packages specify required system dependencies in their DESCRIPTION file, and not all listed dependencies have existing rules in `rstudio/r-system-requirements`.
|
||||
For Alpine, no rules existed until recently, establishing a foundation for semi-automated package installation on Alpine Linux.
|
||||
|
||||
|
|
@ -152,12 +152,13 @@ A Shiny dashboard providing a search functionality of the database and grouped s
|
|||
|
||||
Is supported by writing `Meta/archive.rds` during each package index update, listing all available archived packages.
|
||||
|
||||
### `pak::pak(package@version)`
|
||||
### `uvr add package@version`
|
||||
|
||||
`pak` searches for `Archive/<package>` and can install all versions it finds.
|
||||
Clients that resolve archived versions search for `Archive/<package>` and can install all versions they find.
|
||||
This holds for `uvr` as well as for older `pak`-based clients.
|
||||
|
||||
Ensure to use a clean cache if other repositories have been used previously.
|
||||
If in doubt or when testing, call `pak::meta_clean(force = TRUE)`.
|
||||
If in doubt or when testing, call `uvr cache clean`.
|
||||
|
||||
## Lessons Learned
|
||||
|
||||
|
|
@ -180,7 +181,7 @@ If in doubt or when testing, call `pak::meta_clean(force = TRUE)`.
|
|||
|
||||
## URL Composition and Platform Identifiers
|
||||
|
||||
Platform identifiers have been aligned with those used in <https://github.com/rstudio/r-system-requirements> to ensure proper recognition by the automatic syslib dependency installer of `pak`, specifically via the environment variable `PKG_SYSREQS_PLATFORM`:
|
||||
Platform identifiers have been aligned with those used in <https://github.com/rstudio/r-system-requirements>, which `uvr` vendors and uses for its automatic syslib dependency installer (`uvr sync --install-system-deps`):
|
||||
|
||||
- redhat-9
|
||||
- redhat-8
|
||||
|
|
@ -237,8 +238,8 @@ internal error 1 in memDecompress
|
|||
Solution:
|
||||
|
||||
```sh
|
||||
rm -rf /mnt/cache/R-pkgs/pak /mnt/cache/pkgcache/ /root/.cache/R/
|
||||
R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
||||
uvr cache clean
|
||||
rm -rf /mnt/cache/uvr /root/.cache/R/
|
||||
```
|
||||
|
||||
</details>
|
||||
|
|
|
|||
|
|
@ -30,8 +30,9 @@
|
|||
# from_secret: HETZNER_S3_SECRET_KEY_K3S
|
||||
# # normal env vars
|
||||
# GIT_USER: pat-s
|
||||
# # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
||||
# # R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
# # set the location of uvr's caches, which persist the R package dependencies needed to install the packages themselves
|
||||
# # UVR_CACHE_DIR: /mnt/cache/uvr/cache
|
||||
# # UVR_PACKAGES_DIR: /mnt/cache/uvr/packages
|
||||
# R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
# CCACHE_DIR: /mnt/cache/ccache
|
||||
# volumes:
|
||||
|
|
@ -39,9 +40,8 @@
|
|||
# commands:
|
||||
# - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
||||
# - mkdir -p /mnt/cache/R-pkgs
|
||||
# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/pkgcache
|
||||
# - /opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
||||
# - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git", dependencies = TRUE)'
|
||||
# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/uvr
|
||||
# - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft
|
||||
# - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
||||
# # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine322', 'amd64'); saveRDS(pkgs, '/mnt/cache/pkgs_amd64.rds')"
|
||||
# - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine323', 'arm64'); saveRDS(pkgs, '/mnt/cache/pkgs_arm64.rds')"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,15 @@
|
|||
FROM devxygmbh/r-alpine:4.4-3.20 AS build
|
||||
|
||||
# ARG GITHUB_PAT
|
||||
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
||||
RUN apk add --no-cache curl ca-certificates \
|
||||
&& curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh \
|
||||
| UVR_INSTALL_DIR=/usr/local/bin sh
|
||||
|
||||
# One uvr project drives both dependency installs below. UVR_LIBRARY points the
|
||||
# syncs at the image's R library instead of the project-local .uvr/library/, so
|
||||
# `Rscript app.R` finds the packages without a uvr-aware .Rprofile.
|
||||
ENV UVR_LIBRARY=/usr/lib/R/library
|
||||
RUN mkdir -p /uvr && cd /uvr && uvr init --here
|
||||
|
||||
COPY --link ./DESCRIPTION .
|
||||
COPY --link ./R ./R
|
||||
|
|
@ -11,13 +19,19 @@ COPY --link ./shiny/app.R /app/app.R
|
|||
|
||||
RUN ls -la
|
||||
|
||||
# install R package deps
|
||||
RUN R -q -e 'pak::pak()'
|
||||
# install R package deps. uvr has no equivalent of bare `pak::pak()`, which
|
||||
# reads the DESCRIPTION in the working directory, so extract the dependency
|
||||
# names and hand them to `uvr add`.
|
||||
RUN R -q --no-echo -e "d <- read.dcf('DESCRIPTION'); f <- intersect(colnames(d), c('Depends', 'Imports', 'LinkingTo')); p <- trimws(sub('[(].*', '', unlist(strsplit(paste(d[, f], collapse = ','), ',')))); writeLines(setdiff(p[nzchar(p)], c('R', rownames(installed.packages()))), '/tmp/deps.txt')" \
|
||||
&& cd /uvr && xargs -r uvr add --no-install < /tmp/deps.txt \
|
||||
&& uvr sync --install-system-deps
|
||||
|
||||
RUN R CMD INSTALL --no-docs --without-keep.source .
|
||||
|
||||
# install shiny app deps
|
||||
RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); pak::pak(pkgs)"
|
||||
RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); writeLines(pkgs, '/tmp/app-deps.txt')" \
|
||||
&& cd /uvr && xargs -r uvr add --no-install < /tmp/app-deps.txt \
|
||||
&& uvr sync --install-system-deps
|
||||
|
||||
ENV PGPASS=""
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,10 @@ ARG CACHEBUST
|
|||
WORKDIR /work
|
||||
COPY build-one.R /work/build-one.R
|
||||
# Resolve and install the latest bincraft release dynamically (no hardcoded pin).
|
||||
# uvr-install.sh lands under /work/local/ because install-bincraft.R looks for it
|
||||
# there when the working directory is not a repo checkout.
|
||||
COPY install-bincraft.R /work/install-bincraft.R
|
||||
COPY uvr-install.sh /work/local/uvr-install.sh
|
||||
# Ship the patch registry so build-one.R's `patches = "local/patches"` resolves
|
||||
# (build context is `local/`, CWD is /work).
|
||||
COPY patches /work/local/patches
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
docker run --rm -it --platform linux/arm64 alpine sh
|
||||
|
||||
apk add --no-cache R R-dev g++
|
||||
apk add --no-cache R R-dev g++ curl ca-certificates
|
||||
|
||||
R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
|
||||
curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | UVR_INSTALL_DIR=/usr/local/bin sh
|
||||
|
||||
R -q -e 'pak::pak(c("gert", "purrr"))'
|
||||
# `uvr add` always writes to .uvr/library/; only `uvr sync` honours UVR_LIBRARY,
|
||||
# so add without installing and let the sync place the packages.
|
||||
export UVR_LIBRARY=/usr/lib/R/library
|
||||
mkdir -p /uvr && cd /uvr && uvr init --here
|
||||
uvr add --no-install gert purrr && uvr sync --install-system-deps
|
||||
|
||||
R
|
||||
|
||||
|
|
@ -22,14 +26,20 @@ unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE)
|
|||
tag <- all_tags$name
|
||||
package_name <- rep(package_name, length(tag))
|
||||
|
||||
# uvr has no `pak::local_install_deps()`; read the DESCRIPTION of the checkout
|
||||
# and `uvr add` the dependency names instead.
|
||||
purrr::walk2(package_name[1], tag, \(x, y) {
|
||||
print(y)
|
||||
system("git config --global advice.detachedHead false")
|
||||
src <- sprintf("/tmp/%s_%s", x[1], y)
|
||||
system2("git", args = c(
|
||||
"clone", "-q", sprintf("--branch=%s", tail(y, 1)),
|
||||
sprintf("https://github.com/cran/%s", x[1]), sprintf("/tmp/%s_%s", x[1], y)
|
||||
sprintf("https://github.com/cran/%s", x[1]), src
|
||||
))
|
||||
pak::local_install_deps(sprintf("/tmp/%s_%s", x[1], y))
|
||||
d <- read.dcf(file.path(src, "DESCRIPTION"))
|
||||
f <- intersect(colnames(d), c("Depends", "Imports", "LinkingTo"))
|
||||
deps <- trimws(sub("[(].*", "", unlist(strsplit(paste(d[, f], collapse = ","), ","))))
|
||||
deps <- setdiff(deps[nzchar(deps)], "R")
|
||||
system2("uvr", c("add", "--no-install", deps))
|
||||
system2("uvr", c("sync", "--install-system-deps"))
|
||||
})
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,9 @@ package_cache_files <- c(
|
|||
"/mnt/cache/packages/s3_cache.rds"
|
||||
)
|
||||
if (!all(file.exists(package_cache_files))) {
|
||||
message("Package snapshot missing from cache; recomputing via packages-to-build.R")
|
||||
message(
|
||||
"Package snapshot missing from cache; recomputing via packages-to-build.R"
|
||||
)
|
||||
dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE)
|
||||
save_rds_atomic <- function(obj, path) {
|
||||
tmp <- paste0(path, ".tmp.", Sys.getpid())
|
||||
|
|
@ -36,7 +38,10 @@ if (!all(file.exists(package_cache_files))) {
|
|||
}
|
||||
source(file.path("local", "packages-to-build.R"))
|
||||
save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds")
|
||||
save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds")
|
||||
save_rds_atomic(
|
||||
pkgs[r_minor_sensitive == TRUE],
|
||||
"/mnt/cache/packages/r_minor_sensitive_pkgs.rds"
|
||||
)
|
||||
message("Package snapshot recomputed.")
|
||||
}
|
||||
|
||||
|
|
@ -112,20 +117,22 @@ built <- DBI::dbGetQuery(
|
|||
)
|
||||
DBI::dbDisconnect(con)
|
||||
before <- nrow(chunk)
|
||||
chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ]
|
||||
sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk))
|
||||
chunk <- chunk[
|
||||
!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag),
|
||||
]
|
||||
sprintf(
|
||||
"Skipped %d already-attempted package versions; %d remaining for this job",
|
||||
before - nrow(chunk),
|
||||
nrow(chunk)
|
||||
)
|
||||
|
||||
# Read pre-computed S3 listing from install-deps step
|
||||
# This avoids loading s3fs/reticulate/Python in the build container,
|
||||
# saving significant memory for pak subprocess forks
|
||||
# saving significant memory for the dependency-installer subprocesses
|
||||
s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
||||
sprintf("S3 cache: %s files", length(s3_cache))
|
||||
|
||||
n <- nrow(chunk)
|
||||
# Every `trim_every` packages, bound the pkgcache _metadata dir so a full-platform
|
||||
# run does not accumulate thousands of ~70 MB snapshots and fill the host disk.
|
||||
# No-op on amd64 (R_PKG_CACHE_DIR is empty / cache not persisted).
|
||||
trim_every <- 25L
|
||||
mapply(
|
||||
function(pkg, ver, sens, i) {
|
||||
cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens))
|
||||
|
|
@ -151,12 +158,6 @@ mapply(
|
|||
upload = TRUE,
|
||||
store_build_metadata = TRUE
|
||||
)
|
||||
if (i %% trim_every == 0L) {
|
||||
removed <- trim_pkgcache_metadata()
|
||||
if (removed > 0L) {
|
||||
cat(sprintf(" [pkgcache trim] removed %d stale _metadata entries\n", removed))
|
||||
}
|
||||
}
|
||||
},
|
||||
chunk$Package,
|
||||
chunk$Version,
|
||||
|
|
|
|||
|
|
@ -11,10 +11,11 @@
|
|||
#
|
||||
# How it works: list the remote tags with `git ls-remote` (no token needed for
|
||||
# the public repo), keep the `vX.Y.Z` release tags, pick the highest version,
|
||||
# and install it with pak. pak is idempotent on the git ref, so re-running keeps
|
||||
# the package when it is already current and only updates when a newer tag ships.
|
||||
# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is
|
||||
# identical across git versions and `system2()` argument handling.
|
||||
# and install it with uvr via `local/uvr-install.sh`. uvr is idempotent on the
|
||||
# git ref, so re-running keeps the package when it is already current and only
|
||||
# updates when a newer tag ships. Filtering/sorting is done in R (not via git's
|
||||
# `--sort`/refspec) so behaviour is identical across git versions and
|
||||
# `system2()` argument handling.
|
||||
|
||||
repo_url <- Sys.getenv(
|
||||
"BINCRAFT_GIT_URL",
|
||||
|
|
@ -44,7 +45,43 @@ latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][
|
|||
]
|
||||
|
||||
message(sprintf("Installing latest bincraft release: %s", latest))
|
||||
pak::pak(sprintf("git::%s@%s", repo_url, latest))
|
||||
|
||||
# uvr addresses Forgejo repos as `forgejo::host/owner/repo@ref` rather than as a
|
||||
# git URL, so drop the scheme and the trailing `.git` from `repo_url`.
|
||||
spec <- sprintf(
|
||||
"forgejo::%s@%s",
|
||||
sub("\\.git$", "", sub("^[a-z]+://", "", repo_url)),
|
||||
latest
|
||||
)
|
||||
|
||||
# `local/uvr-install.sh` when run from the repo root, `/work/local/` in the
|
||||
# build-one image, which copies the two scripts into a flatter layout.
|
||||
helper <- Sys.getenv("UVR_INSTALL_SH", unset = "")
|
||||
if (!nzchar(helper)) {
|
||||
candidates <- c("local/uvr-install.sh", "/work/local/uvr-install.sh")
|
||||
found <- candidates[file.exists(candidates)]
|
||||
if (length(found) == 0L) {
|
||||
stop("Could not locate uvr-install.sh; set UVR_INSTALL_SH", call. = FALSE)
|
||||
}
|
||||
helper <- found[1L]
|
||||
}
|
||||
|
||||
# Point uvr at the R running this script and at the library it would install
|
||||
# into, so the per-R-minor passes in the build pipelines (which call a different
|
||||
# Rscript with R_LIBS_USER pointed elsewhere) target their own R and library.
|
||||
Sys.setenv(
|
||||
UVR_R_BIN = file.path(R.home("bin"), "R"),
|
||||
UVR_TARGET_LIB = .libPaths()[1L]
|
||||
)
|
||||
|
||||
status <- system2(helper, shQuote(spec))
|
||||
if (!identical(status, 0L)) {
|
||||
stop(
|
||||
sprintf("uvr failed to install %s (exit %s)", spec, status),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
|
||||
message(sprintf(
|
||||
"bincraft %s installed (%s)",
|
||||
as.character(utils::packageVersion("bincraft")),
|
||||
|
|
|
|||
|
|
@ -15,15 +15,15 @@ suppressPackageStartupMessages(library(data.table))
|
|||
# Sys.setenv("OS_VERSION" = "3.22")
|
||||
# Sys.setenv("ARCH" = "arm64")
|
||||
|
||||
arch = Sys.getenv("ARCH")
|
||||
arch <- Sys.getenv("ARCH")
|
||||
# target: alpine-322, ubuntu-2404, redhat-9, etc.
|
||||
platform = paste(
|
||||
platform <- paste(
|
||||
Sys.getenv("OS"),
|
||||
gsub("[.]", "", Sys.getenv("OS_VERSION")),
|
||||
sep = "-"
|
||||
)
|
||||
# Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10")
|
||||
codename = bincraft::set_codename(NULL)
|
||||
codename <- bincraft::set_codename(NULL)
|
||||
|
||||
con <- DBI::dbConnect(
|
||||
RPostgres::Postgres(),
|
||||
|
|
@ -35,8 +35,8 @@ con <- DBI::dbConnect(
|
|||
sslmode = "require"
|
||||
)
|
||||
|
||||
cran_archive = tools::CRAN_archive_db()
|
||||
cran_release = tools::CRAN_package_db()
|
||||
cran_archive <- tools::CRAN_archive_db()
|
||||
cran_release <- tools::CRAN_package_db()
|
||||
# Subset cran_archive to only those packages
|
||||
cran_archive_in_release <- cran_archive[
|
||||
names(cran_archive) %in% cran_release$Package
|
||||
|
|
@ -84,13 +84,14 @@ s3fs::s3_file_system(
|
|||
region_name = "eu-central-003",
|
||||
refresh = TRUE
|
||||
)
|
||||
s3_pkgs = s3fs::s3_dir_ls(
|
||||
s3_pkgs <- s3fs::s3_dir_ls(
|
||||
sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename),
|
||||
recurse = TRUE
|
||||
)
|
||||
|
||||
# Save the raw S3 file listing for the build step to use as s3_package_cache
|
||||
# This avoids loading s3fs/reticulate in the build container, saving memory for pak forks
|
||||
# This avoids loading s3fs/reticulate in the build container, saving memory for
|
||||
# the dependency-installer subprocesses
|
||||
saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds")
|
||||
|
||||
file_names <- basename(s3_pkgs)
|
||||
|
|
|
|||
|
|
@ -31,42 +31,3 @@ parse_build_args <- function(args) {
|
|||
ncpus = as.integer(pos[3L])
|
||||
)
|
||||
}
|
||||
|
||||
# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the
|
||||
# "patched" repo mints a new content hash on every PACKAGES change, so each build
|
||||
# writes a fresh ~70 MB _metadata/pkgs-<hash>.rds (+ patched-<hash>/) that is
|
||||
# never reused. Keep the `keep` newest entries by mtime; only remove entries
|
||||
# older than `min_age_secs`, so a concurrent split-job's in-flight files are
|
||||
# never deleted (each build uses a unique hash, so aged entries are
|
||||
# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are
|
||||
# not matched and thus preserved. Returns the number of entries removed.
|
||||
trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"),
|
||||
keep = 20L,
|
||||
min_age_secs = 600) {
|
||||
meta <- file.path(cache_dir, "R", "pkgcache", "_metadata")
|
||||
if (!nzchar(cache_dir) || !dir.exists(meta)) {
|
||||
return(0L)
|
||||
}
|
||||
entries <- c(
|
||||
Sys.glob(file.path(meta, "patched-*")),
|
||||
Sys.glob(file.path(meta, "pkgs-*.rds"))
|
||||
)
|
||||
if (length(entries) == 0L) {
|
||||
return(0L)
|
||||
}
|
||||
info <- file.info(entries)
|
||||
order_new_first <- order(info$mtime, decreasing = TRUE)
|
||||
ranked <- entries[order_new_first]
|
||||
ranked_mtime <- info$mtime[order_new_first]
|
||||
if (length(ranked) <= keep) {
|
||||
return(0L)
|
||||
}
|
||||
candidates <- ranked[(keep + 1L):length(ranked)]
|
||||
candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)])
|
||||
removable <- candidates[candidate_age >= min_age_secs]
|
||||
if (length(removable) == 0L) {
|
||||
return(0L)
|
||||
}
|
||||
unlink(removable, recursive = TRUE, force = TRUE)
|
||||
length(removable)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,22 +1,28 @@
|
|||
install.packages(
|
||||
"pak",
|
||||
repos = sprintf(
|
||||
"https://r-lib.github.io/p/pak/stable/%s/%s/%s",
|
||||
.Platform$pkgType,
|
||||
R.Version()$os,
|
||||
R.Version()$arch
|
||||
# Installs every CRAN package one by one and checks that it loads. Dependencies
|
||||
# go through uvr via local/uvr-install.sh, which bootstraps the uvr binary on
|
||||
# first use and installs into .libPaths()[1].
|
||||
uvr_install <- function(pkg) {
|
||||
Sys.setenv(
|
||||
UVR_R_BIN = file.path(R.home("bin"), "R"),
|
||||
UVR_TARGET_LIB = .libPaths()[1L]
|
||||
)
|
||||
)
|
||||
status <- system2("local/uvr-install.sh", shQuote(pkg))
|
||||
if (!identical(status, 0L)) {
|
||||
stop(
|
||||
sprintf("uvr failed to install %s (exit %s)", pkg, status),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
Sys.setenv(PKG_SYSREQS = TRUE)
|
||||
all_pkgs <- rownames(available.packages())
|
||||
|
||||
to_skip = c("ABRSQOL", "ACA", "ACE.CoCo")
|
||||
all_pkgs = setdiff(all_pkgs, to_skip)
|
||||
to_skip <- c("ABRSQOL", "ACA", "ACE.CoCo")
|
||||
all_pkgs <- setdiff(all_pkgs, to_skip)
|
||||
|
||||
for (i in all_pkgs) {
|
||||
message(sprintf("\nInstalling %s", i))
|
||||
pak::pkg_install(i)
|
||||
uvr_install(i)
|
||||
library(i, character.only = TRUE)
|
||||
}
|
||||
|
||||
|
|
@ -323,7 +329,7 @@ if (length(to_process) == 0) {
|
|||
} else {
|
||||
for (i in to_process) {
|
||||
message(sprintf("\nInstalling %s", i))
|
||||
pak::pkg_install(i)
|
||||
uvr_install(i)
|
||||
library(i, character.only = TRUE)
|
||||
}
|
||||
# Update to_skip to include all up to the last processed
|
||||
|
|
|
|||
|
|
@ -1,68 +0,0 @@
|
|||
source(file.path("..", "r-minor-helpers.R"))
|
||||
|
||||
# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is
|
||||
# set to `age_secs` in the past so we can exercise the age gate deterministically.
|
||||
make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) {
|
||||
root <- tempfile("pkgcache-")
|
||||
meta <- file.path(root, "R", "pkgcache", "_metadata")
|
||||
dir.create(meta, recursive = TRUE)
|
||||
old <- Sys.time() - age_secs
|
||||
mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p }
|
||||
mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p }
|
||||
for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i)))
|
||||
for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i)))
|
||||
if (keep_repos) {
|
||||
mk_dir(file.path(meta, "CRAN-075c426938"))
|
||||
mk_dir(file.path(meta, "BioCsoft-1ac964ed6c"))
|
||||
mk_file(file.path(meta, "bioc-sysreqs.dcf.gz"))
|
||||
mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive
|
||||
}
|
||||
root
|
||||
}
|
||||
|
||||
n_churn <- function(root) {
|
||||
meta <- file.path(root, "R", "pkgcache", "_metadata")
|
||||
length(Sys.glob(file.path(meta, "patched-*"))) +
|
||||
length(Sys.glob(file.path(meta, "pkgs-*.rds")))
|
||||
}
|
||||
|
||||
test_that("empty cache_dir is a no-op", {
|
||||
expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L)
|
||||
})
|
||||
|
||||
test_that("missing _metadata dir is a no-op", {
|
||||
expect_identical(
|
||||
trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0),
|
||||
0L
|
||||
)
|
||||
})
|
||||
|
||||
test_that("fewer than keep entries removes nothing", {
|
||||
root <- make_meta(patched = 2L, pkgs = 2L)
|
||||
expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L)
|
||||
expect_identical(n_churn(root), 4L)
|
||||
})
|
||||
|
||||
test_that("trims down to keep newest, leaving churn == keep", {
|
||||
root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old
|
||||
removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0)
|
||||
expect_identical(removed, 40L)
|
||||
expect_identical(n_churn(root), 20L)
|
||||
})
|
||||
|
||||
test_that("entries younger than min_age_secs are protected", {
|
||||
root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60)
|
||||
# keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed
|
||||
expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L)
|
||||
expect_identical(n_churn(root), 30L)
|
||||
})
|
||||
|
||||
test_that("stable repo dirs and pkg downloads are never touched", {
|
||||
root <- make_meta(patched = 30L, pkgs = 30L)
|
||||
trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0)
|
||||
meta <- file.path(root, "R", "pkgcache", "_metadata")
|
||||
expect_true(dir.exists(file.path(meta, "CRAN-075c426938")))
|
||||
expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c")))
|
||||
expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz")))
|
||||
expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg")))
|
||||
})
|
||||
90
local/uvr-install.sh
Executable file
90
local/uvr-install.sh
Executable file
|
|
@ -0,0 +1,90 @@
|
|||
#!/bin/sh
|
||||
# Install R packages into the CI library with uvr (https://github.com/nbafrank/uvr).
|
||||
#
|
||||
# Usage:
|
||||
# local/uvr-install.sh httr2 jsonlite
|
||||
# local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft@v4.4.3
|
||||
#
|
||||
# Replaces `pak::pak(...)`. uvr is project-scoped: `uvr add` refuses to run
|
||||
# outside a project and always writes to `.uvr/library/`, and only
|
||||
# `uvr sync --library` can target an existing library. The project is therefore
|
||||
# minted in a scratch directory under TMPDIR and thrown away afterwards; that
|
||||
# also keeps `uvr init`'s `.Rprofile` out of the repo checkout, where it would
|
||||
# hijack `.libPaths()` for every other R call in the pipeline.
|
||||
#
|
||||
# Pruning is a no-op here: uvr disables it whenever `--library` is passed,
|
||||
# precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds
|
||||
# bincraft and its dependencies alongside whatever this script installs).
|
||||
#
|
||||
# System dependencies come from uvr's vendored r-system-requirements rules, so
|
||||
# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed.
|
||||
#
|
||||
# Environment:
|
||||
# UVR_R_BIN R interpreter to install for; set by install-bincraft.R so
|
||||
# the per-R-minor passes target their own R, not the primary
|
||||
# R_VERSION fallback interpreter selector (/opt/R/<version>/bin/R)
|
||||
# UVR_TARGET_LIB target library; defaults to R_LIBS_USER, then to the
|
||||
# active R's .libPaths()[1] (which is where pak wrote)
|
||||
# UVR_INSTALL_DIR where the uvr binary lands (default /usr/local/bin)
|
||||
|
||||
set -eu
|
||||
|
||||
# renovate: datasource=github-releases depName=nbafrank/uvr
|
||||
UVR_PIN="v0.4.4"
|
||||
|
||||
if [ "$#" -eq 0 ]; then
|
||||
echo "usage: $0 <pkg-spec>..." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# The build images keep R under /opt/R/<version> and off PATH. uvr resolves the
|
||||
# interpreter via PATH and never downloads one unless `uvr r install` is run, so
|
||||
# put the requested R first.
|
||||
r_bin="${UVR_R_BIN:-}"
|
||||
if [ -z "$r_bin" ] && [ -n "${R_VERSION:-}" ] && [ -x "/opt/R/${R_VERSION}/bin/R" ]; then
|
||||
r_bin="/opt/R/${R_VERSION}/bin/R"
|
||||
fi
|
||||
if [ -n "$r_bin" ]; then
|
||||
PATH="$(dirname "$r_bin"):$PATH"
|
||||
export PATH
|
||||
else
|
||||
r_bin="$(command -v R)"
|
||||
fi
|
||||
|
||||
target_lib="${UVR_TARGET_LIB:-${R_LIBS_USER:-}}"
|
||||
if [ -z "$target_lib" ]; then
|
||||
target_lib="$("$r_bin" --no-echo --no-save -e 'cat(.libPaths()[1])')"
|
||||
fi
|
||||
if [ -z "$target_lib" ]; then
|
||||
echo "error: could not determine a target library; set UVR_TARGET_LIB" >&2
|
||||
exit 2
|
||||
fi
|
||||
mkdir -p "$target_lib"
|
||||
|
||||
# Pin the manifest to the active R so the lockfile's R stays in step with the
|
||||
# library's R sentinel. Without that, uvr can decide the library is ABI-stale
|
||||
# and wipe it -- and this target is shared with bincraft. uvr only discovers R
|
||||
# via PATH/R_HOME (it does not scan /opt/R), so the R put on PATH above is the
|
||||
# only candidate this constraint can resolve to.
|
||||
# shellcheck disable=SC2016 # $major/$minor are R expressions, not shell vars
|
||||
r_full="$("$r_bin" --no-echo --no-save -e 'cat(paste(R.version$major, R.version$minor, sep = "."))')"
|
||||
|
||||
install_dir="${UVR_INSTALL_DIR:-/usr/local/bin}"
|
||||
uvr_bin="${install_dir}/uvr"
|
||||
if [ ! -x "$uvr_bin" ]; then
|
||||
echo "Bootstrapping uvr ${UVR_PIN} into ${install_dir}"
|
||||
UVR_INSTALL_DIR="$install_dir" UVR_VERSION="$UVR_PIN" \
|
||||
sh -c 'curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | sh'
|
||||
fi
|
||||
|
||||
project_dir="${TMPDIR:-/tmp}/uvr-ci-$$"
|
||||
rm -rf "$project_dir"
|
||||
mkdir -p "$project_dir"
|
||||
trap 'rm -rf "$project_dir"' EXIT
|
||||
cd "$project_dir"
|
||||
|
||||
"$uvr_bin" init --here --r-version "$r_full"
|
||||
# --no-install: resolve and lock only. The install happens in the sync below,
|
||||
# which is the only command that honours --library.
|
||||
"$uvr_bin" add --no-install "$@"
|
||||
"$uvr_bin" sync --library "$target_lib" --install-system-deps
|
||||
|
|
@ -1,7 +1,12 @@
|
|||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": ["local>devxy/renovate-config"],
|
||||
"ignorePaths": ["docker/**", ".crow/process-updates.yaml", ".crow/build-all-versions.yaml", ".crow/weekly-rebuild-missing.yaml"],
|
||||
"ignorePaths": [
|
||||
"docker/**",
|
||||
".crow/process-updates.yaml",
|
||||
".crow/build-all-versions.yaml",
|
||||
".crow/weekly-rebuild-missing.yaml"
|
||||
],
|
||||
"customManagers": [
|
||||
{
|
||||
"customType": "regex",
|
||||
|
|
@ -14,6 +19,13 @@
|
|||
"packageNameTemplate": "https://codefloe.com/rpkgs/bincraft.git",
|
||||
"datasourceTemplate": "git-tags",
|
||||
"extractVersionTemplate": "^v?(?<version>.+)$"
|
||||
},
|
||||
{
|
||||
"customType": "regex",
|
||||
"fileMatch": ["^local/uvr-install\\.sh$"],
|
||||
"matchStrings": [
|
||||
"# renovate: datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)\\s+UVR_PIN=\"(?<currentValue>[^\"]+)\""
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue