diff --git a/.crow/auto-apply-patches.yaml b/.crow/auto-apply-patches.yaml index b9c8697..0076353 100644 --- a/.crow/auto-apply-patches.yaml +++ b/.crow/auto-apply-patches.yaml @@ -45,7 +45,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-pr --limit $PATCH_LIMIT backend_options: kubernetes: diff --git a/.crow/build-all-versions-install-deps.yaml b/.crow/build-all-versions-install-deps.yaml index e0cca86..448cff3 100644 --- a/.crow/build-all-versions-install-deps.yaml +++ b/.crow/build-all-versions-install-deps.yaml @@ -10,22 +10,22 @@ variables: - arm64 default: amd64 OS: - description: "Base OS image name." + description: 'Base OS image name.' options: - alpine - redhat - ubuntu default: alpine OS_VERSION: - description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' options: - - "3.24" - - "8" - - "9" - - "10" - - "jammy" - - "noble" - default: "3.24" + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -66,23 +66,24 @@ steps: from_secret: B2_S3_SECRET_KEY PGPASS: from_secret: PGPASS - R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs + # Keep uvr's downloads and extracted-package entries on the persistent + # volume instead of the container-local ~/.uvr default. + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages CCACHE_DIR: /mnt/cache/ccache volumes: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: # one-time full wipe to fix corrupted .so files from previous failed builds # - rm -rf /mnt/cache/R-pkgs - # Clear churny pkgcache metadata left by a prior crashed run (the "patched" - # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). - # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. - - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # Pin the same bincraft version the build steps use, so the precomputed # snapshot and the per-agent library stay consistent across the pipeline. - - /opt/R/$R_VERSION/bin/R -q -e 'pak::sysreqs_db_update(); source("local/install-bincraft.R"); pak::pak(c("RPostgres", "s3fs", "data.table", "future", "jsonlite")); packageVersion("bincraft")' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh RPostgres s3fs data.table future jsonlite + - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - /opt/R/$R_VERSION/bin/R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); saveRDS(pkgs[r_minor_sensitive == TRUE], '/mnt/cache/packages/r_minor_sensitive_pkgs.rds'); sprintf('Precomputed %s package versions (%s r-minor-sensitive)', nrow(pkgs), nrow(pkgs[r_minor_sensitive == TRUE]))" backend_options: docker: diff --git a/.crow/build-all-versions.yaml b/.crow/build-all-versions.yaml index 3932811..9ab888a 100644 --- a/.crow/build-all-versions.yaml +++ b/.crow/build-all-versions.yaml @@ -11,25 +11,25 @@ variables: - arm64 default: amd64 OS: - description: "Base OS image name." + description: 'Base OS image name.' options: - alpine - redhat - ubuntu default: alpine OS_VERSION: - description: "OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble)." + description: 'OS image tag. Must match OS (alpine: 3.24; redhat: 8/9/10; ubuntu: jammy/noble).' options: - - "3.22" - - "3.23" - - "3.24" - - "8" - - "9" - - "10" - - "jammy" - - "noble" - - "resolute" - default: "3.24" + - '3.22' + - '3.23' + - '3.24' + - '8' + - '9' + - '10' + - 'jammy' + - 'noble' + - 'resolute' + default: '3.24' R_VERSION: description: 'Primary R version under /opt/R.' options: @@ -47,38 +47,49 @@ labels: platform: linux/${ARCH} group: rpkgs-${ARCH} +# Empty UVR_CACHE_DIR/UVR_PACKAGES_DIR fall back to uvr's container-local +# ~/.uvr defaults; amd64 deliberately does not persist them (as with the +# pkgcache dir it replaces), arm64 does. matrix: include: - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: amd64 - R_PKG_CACHE_DIR: '' + UVR_CACHE_DIR: '' + UVR_PACKAGES_DIR: '' SPLIT_INTO: 4 SPLIT_INDEX: 4 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 1 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 2 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 3 - ARCH: arm64 - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages SPLIT_INTO: 4 SPLIT_INDEX: 4 @@ -107,8 +118,10 @@ steps: from_secret: GITHUB_PAT # normal env vars GIT_USER: pat-s - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: ${R_PKG_CACHE_DIR} + # set the location of uvr's caches, which persist the R package + # dependencies needed to install the packages themselves + UVR_CACHE_DIR: ${UVR_CACHE_DIR} + UVR_PACKAGES_DIR: ${UVR_PACKAGES_DIR} R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache NCPUS: 2 @@ -116,12 +129,7 @@ steps: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - # Clear churny pkgcache metadata left by a prior crashed run (the "patched" - # repo mints a new hash per PACKAGES change -> unbounded pkgs-*.rds/patched-*). - # Keep pkg/ downloads and the stable CRAN/BioC/INLA repo dirs. Within-run - # growth is bounded separately by trim_pkgcache_metadata() in build-all.R. - - rm -rf /mnt/cache/pkgcache/R/pkgcache/_metadata/patched-* /mnt/cache/pkgcache/R/pkgcache/_metadata/pkgs-*.rds || true - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # The primary pass must not rely on build-all-versions-install-deps having # run on *this* agent: depends_on only orders the steps, but the cache # volume is per-agent, so a job landing on an agent where install-deps did diff --git a/.crow/process-updates.yaml b/.crow/process-updates.yaml index 59b1c26..1e4833c 100644 --- a/.crow/process-updates.yaml +++ b/.crow/process-updates.yaml @@ -49,109 +49,109 @@ matrix: R_VERSION: 4.5.3 IMG: alpine:3.22 OS_ID: alpine322 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-322 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.23 OS_ID: alpine322 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-323 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.23 OS_ID: alpine323 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-323 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.23 OS_ID: alpine323 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-324 ARCH: amd64 R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine324 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: alpine-324 ARCH: arm64 R_VERSION: 4.5.3 IMG: alpine:3.24 OS_ID: alpine324 - PROCESS_NEW: "FALSE" + PROCESS_NEW: 'FALSE' - OS: redhat-8 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-8 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:8 OS_ID: rhel8 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-9 ARCH: amd64 R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-9 ARCH: arm64 R_VERSION: 4.4.3 IMG: redhat:9 OS_ID: rhel9 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-10 ARCH: amd64 R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: redhat-10 ARCH: arm64 R_VERSION: 4.5.3 IMG: redhat:10 OS_ID: rhel10 - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2204 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2204 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:jammy OS_ID: jammy - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2404 ARCH: amd64 R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2404 ARCH: arm64 R_VERSION: 4.4.3 IMG: ubuntu:noble OS_ID: noble - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2604 ARCH: amd64 R_VERSION: 4.5.3 IMG: ubuntu:resolute OS_ID: resolute - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' - OS: ubuntu-2604 ARCH: arm64 R_VERSION: 4.5.3 IMG: ubuntu:resolute OS_ID: resolute - PROCESS_NEW: "TRUE" + PROCESS_NEW: 'TRUE' steps: - name: 'Processing Updates' @@ -182,8 +182,10 @@ steps: NTFY_AUTH: TRUE NTFY_PASSWORD: from_secret: ntfy_token - # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + # set the location of uvr's caches, which persist the R package + # dependencies needed to install the packages themselves + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache @@ -194,10 +196,10 @@ steps: INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/R-pkgs/pkgcache /mnt/cache/pkgcache/R/pkgcache + - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # rhel-10 ships xwfb-run (Xwayland) instead of xvfb-run; prefer it and start weston, else fall back to xvfb-run - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise diff --git a/.crow/weekly-audit-missing.yaml b/.crow/weekly-audit-missing.yaml index d66a37b..4efe409 100644 --- a/.crow/weekly-audit-missing.yaml +++ b/.crow/weekly-audit-missing.yaml @@ -142,7 +142,8 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' + - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/R -q -e 'source("local/weekly-missing-binaries-audit.R")' backend_options: docker: diff --git a/.crow/weekly-patch-proposals.yaml b/.crow/weekly-patch-proposals.yaml index 0284e93..027d41c 100644 --- a/.crow/weekly-patch-proposals.yaml +++ b/.crow/weekly-patch-proposals.yaml @@ -35,7 +35,7 @@ steps: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - mkdir -p /mnt/cache/R-pkgs - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak(c("httr2", "jsonlite"))' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 jsonlite - /opt/R/$R_VERSION/bin/Rscript local/propose-patches.R --open-issue - /opt/R/$R_VERSION/bin/Rscript local/proposal-tracking.R --open-issue backend_options: diff --git a/.crow/weekly-rebuild-missing.yaml b/.crow/weekly-rebuild-missing.yaml index 53f178f..03a13f9 100644 --- a/.crow/weekly-rebuild-missing.yaml +++ b/.crow/weekly-rebuild-missing.yaml @@ -141,7 +141,8 @@ steps: FORGEJO_TOKEN: from_secret: FORGEJO_TOKEN GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache + UVR_CACHE_DIR: /mnt/cache/uvr/cache + UVR_PACKAGES_DIR: /mnt/cache/uvr/packages R_LIBS_USER: /mnt/cache/R-pkgs R_VERSION: ${R_VERSION} CCACHE_DIR: /mnt/cache/ccache @@ -150,12 +151,12 @@ steps: NCPUS: 2 commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + - mkdir -p /mnt/cache/uvr/cache /mnt/cache/uvr/packages /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - rm -rf /mnt/cache/R-pkgs/00LOCK-* - /opt/R/$R_VERSION/bin/Rscript local/install-bincraft.R - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("httr2")' + - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh httr2 - /opt/R/$R_VERSION/bin/R -q -e 'source("local/fetch-rebuild-packages-from-issue.R")' - $XVFB $XVFB_ARGS -- /opt/R/$R_VERSION/bin/R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs <- readLines('/tmp/rebuild_pkgs.txt'); if (length(pkgs) == 0) { cat('Nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, patches = 'local/patches', s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 backend_options: diff --git a/README.md b/README.md index 538df34..989069e 100644 --- a/README.md +++ b/README.md @@ -112,9 +112,9 @@ Processing updates with an existing database file takes around 5 minutes. ### Inferring System Dependencies -R package dependencies and their system dependencies are installed through {pak}. -{pak} allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. -Additionally, it automatically infers package dependencies using JSON rules from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). +R package dependencies and their system dependencies are installed through [uvr](https://github.com/nbafrank/uvr). +`uvr` allows for parallel downloads and installation, significantly speeding up package installation compared to `install.packages()`. +Additionally, it automatically infers package dependencies using JSON rules vendored from [rstudio/r-system-requirements](https://github.com/rstudio/r-system-requirements). Not all R packages specify required system dependencies in their DESCRIPTION file, and not all listed dependencies have existing rules in `rstudio/r-system-requirements`. For Alpine, no rules existed until recently, establishing a foundation for semi-automated package installation on Alpine Linux. @@ -152,12 +152,13 @@ A Shiny dashboard providing a search functionality of the database and grouped s Is supported by writing `Meta/archive.rds` during each package index update, listing all available archived packages. -### `pak::pak(package@version)` +### `uvr add package@version` -`pak` searches for `Archive/` and can install all versions it finds. +Clients that resolve archived versions search for `Archive/` and can install all versions they find. +This holds for `uvr` as well as for older `pak`-based clients. Ensure to use a clean cache if other repositories have been used previously. -If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. +If in doubt or when testing, call `uvr cache clean`. ## Lessons Learned @@ -180,7 +181,7 @@ If in doubt or when testing, call `pak::meta_clean(force = TRUE)`. ## URL Composition and Platform Identifiers -Platform identifiers have been aligned with those used in to ensure proper recognition by the automatic syslib dependency installer of `pak`, specifically via the environment variable `PKG_SYSREQS_PLATFORM`: +Platform identifiers have been aligned with those used in , which `uvr` vendors and uses for its automatic syslib dependency installer (`uvr sync --install-system-deps`): - redhat-9 - redhat-8 @@ -237,8 +238,8 @@ internal error 1 in memDecompress Solution: ```sh -rm -rf /mnt/cache/R-pkgs/pak /mnt/cache/pkgcache/ /root/.cache/R/ -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +uvr cache clean +rm -rf /mnt/cache/uvr /root/.cache/R/ ``` diff --git a/build-all-versions-install-deps.yaml b/build-all-versions-install-deps.yaml index 926d550..dd02e8a 100644 --- a/build-all-versions-install-deps.yaml +++ b/build-all-versions-install-deps.yaml @@ -30,8 +30,9 @@ # from_secret: HETZNER_S3_SECRET_KEY_K3S # # normal env vars # GIT_USER: pat-s -# # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves -# # R_PKG_CACHE_DIR: /mnt/cache/pkgcache +# # set the location of uvr's caches, which persist the R package dependencies needed to install the packages themselves +# # UVR_CACHE_DIR: /mnt/cache/uvr/cache +# # UVR_PACKAGES_DIR: /mnt/cache/uvr/packages # R_LIBS_USER: /mnt/cache/R-pkgs # CCACHE_DIR: /mnt/cache/ccache # volumes: @@ -39,9 +40,8 @@ # commands: # - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . # - mkdir -p /mnt/cache/R-pkgs -# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/pkgcache -# - /opt/R/$R_VERSION/bin/R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' -# - /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git", dependencies = TRUE)' +# - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft /mnt/cache/uvr +# - UVR_R_BIN=/opt/R/$R_VERSION/bin/R local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft # - /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")' # # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine322', 'amd64'); saveRDS(pkgs, '/mnt/cache/pkgs_amd64.rds')" # - /opt/R/$R_VERSION/bin/R -q -e "future::plan('multisession', workers = 6L); pkgs = bincraft::query_packages_without_historic_versions('alpine323', 'arm64'); saveRDS(pkgs, '/mnt/cache/pkgs_arm64.rds')" diff --git a/docker/Containerfile-shiny-app b/docker/Containerfile-shiny-app index 7c7994f..4b0bd6c 100644 --- a/docker/Containerfile-shiny-app +++ b/docker/Containerfile-shiny-app @@ -1,7 +1,15 @@ FROM devxygmbh/r-alpine:4.4-3.20 AS build # ARG GITHUB_PAT -RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +RUN apk add --no-cache curl ca-certificates \ + && curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh \ + | UVR_INSTALL_DIR=/usr/local/bin sh + +# One uvr project drives both dependency installs below. UVR_LIBRARY points the +# syncs at the image's R library instead of the project-local .uvr/library/, so +# `Rscript app.R` finds the packages without a uvr-aware .Rprofile. +ENV UVR_LIBRARY=/usr/lib/R/library +RUN mkdir -p /uvr && cd /uvr && uvr init --here COPY --link ./DESCRIPTION . COPY --link ./R ./R @@ -11,13 +19,19 @@ COPY --link ./shiny/app.R /app/app.R RUN ls -la -# install R package deps -RUN R -q -e 'pak::pak()' +# install R package deps. uvr has no equivalent of bare `pak::pak()`, which +# reads the DESCRIPTION in the working directory, so extract the dependency +# names and hand them to `uvr add`. +RUN R -q --no-echo -e "d <- read.dcf('DESCRIPTION'); f <- intersect(colnames(d), c('Depends', 'Imports', 'LinkingTo')); p <- trimws(sub('[(].*', '', unlist(strsplit(paste(d[, f], collapse = ','), ',')))); writeLines(setdiff(p[nzchar(p)], c('R', rownames(installed.packages()))), '/tmp/deps.txt')" \ + && cd /uvr && xargs -r uvr add --no-install < /tmp/deps.txt \ + && uvr sync --install-system-deps RUN R CMD INSTALL --no-docs --without-keep.source . # install shiny app deps -RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); pak::pak(pkgs)" +RUN R -q -e "install.packages('renv'); pkgs <- renv::dependencies('/app/app.R')[['Package']]; pkgs = setdiff(pkgs, 'bincraft'); writeLines(pkgs, '/tmp/app-deps.txt')" \ + && cd /uvr && xargs -r uvr add --no-install < /tmp/app-deps.txt \ + && uvr sync --install-system-deps ENV PGPASS="" diff --git a/docker/build-one.Dockerfile b/docker/build-one.Dockerfile index 5bc107a..8e61ebd 100644 --- a/docker/build-one.Dockerfile +++ b/docker/build-one.Dockerfile @@ -17,7 +17,10 @@ ARG CACHEBUST WORKDIR /work COPY build-one.R /work/build-one.R # Resolve and install the latest bincraft release dynamically (no hardcoded pin). +# uvr-install.sh lands under /work/local/ because install-bincraft.R looks for it +# there when the working directory is not a repo checkout. COPY install-bincraft.R /work/install-bincraft.R +COPY uvr-install.sh /work/local/uvr-install.sh # Ship the patch registry so build-one.R's `patches = "local/patches"` resolves # (build context is `local/`, CWD is /work). COPY patches /work/local/patches diff --git a/docker/reprex/alpine.sh b/docker/reprex/alpine.sh index 94806e5..851a9cc 100644 --- a/docker/reprex/alpine.sh +++ b/docker/reprex/alpine.sh @@ -1,10 +1,14 @@ docker run --rm -it --platform linux/arm64 alpine sh -apk add --no-cache R R-dev g++ +apk add --no-cache R R-dev g++ curl ca-certificates -R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' +curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | UVR_INSTALL_DIR=/usr/local/bin sh -R -q -e 'pak::pak(c("gert", "purrr"))' +# `uvr add` always writes to .uvr/library/; only `uvr sync` honours UVR_LIBRARY, +# so add without installing and let the sync place the packages. +export UVR_LIBRARY=/usr/lib/R/library +mkdir -p /uvr && cd /uvr && uvr init --here +uvr add --no-install gert purrr && uvr sync --install-system-deps R @@ -22,14 +26,20 @@ unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE) tag <- all_tags$name package_name <- rep(package_name, length(tag)) +# uvr has no `pak::local_install_deps()`; read the DESCRIPTION of the checkout +# and `uvr add` the dependency names instead. purrr::walk2(package_name[1], tag, \(x, y) { print(y) system("git config --global advice.detachedHead false") + src <- sprintf("/tmp/%s_%s", x[1], y) system2("git", args = c( "clone", "-q", sprintf("--branch=%s", tail(y, 1)), - sprintf("https://github.com/cran/%s", x[1]), sprintf("/tmp/%s_%s", x[1], y) + sprintf("https://github.com/cran/%s", x[1]), src )) - pak::local_install_deps(sprintf("/tmp/%s_%s", x[1], y)) + d <- read.dcf(file.path(src, "DESCRIPTION")) + f <- intersect(colnames(d), c("Depends", "Imports", "LinkingTo")) + deps <- trimws(sub("[(].*", "", unlist(strsplit(paste(d[, f], collapse = ","), ",")))) + deps <- setdiff(deps[nzchar(deps)], "R") + system2("uvr", c("add", "--no-install", deps)) + system2("uvr", c("sync", "--install-system-deps")) }) - - diff --git a/local/build-all.R b/local/build-all.R index 1df1f83..37fc593 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -27,7 +27,9 @@ package_cache_files <- c( "/mnt/cache/packages/s3_cache.rds" ) if (!all(file.exists(package_cache_files))) { - message("Package snapshot missing from cache; recomputing via packages-to-build.R") + message( + "Package snapshot missing from cache; recomputing via packages-to-build.R" + ) dir.create("/mnt/cache/packages", showWarnings = FALSE, recursive = TRUE) save_rds_atomic <- function(obj, path) { tmp <- paste0(path, ".tmp.", Sys.getpid()) @@ -36,7 +38,10 @@ if (!all(file.exists(package_cache_files))) { } source(file.path("local", "packages-to-build.R")) save_rds_atomic(pkgs, "/mnt/cache/packages/pkgs_to_build.rds") - save_rds_atomic(pkgs[r_minor_sensitive == TRUE], "/mnt/cache/packages/r_minor_sensitive_pkgs.rds") + save_rds_atomic( + pkgs[r_minor_sensitive == TRUE], + "/mnt/cache/packages/r_minor_sensitive_pkgs.rds" + ) message("Package snapshot recomputed.") } @@ -112,20 +117,22 @@ built <- DBI::dbGetQuery( ) DBI::dbDisconnect(con) before <- nrow(chunk) -chunk <- chunk[!paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), ] -sprintf("Skipped %d already-attempted package versions; %d remaining for this job", before - nrow(chunk), nrow(chunk)) +chunk <- chunk[ + !paste(chunk$Package, chunk$Version) %in% paste(built$name, built$tag), +] +sprintf( + "Skipped %d already-attempted package versions; %d remaining for this job", + before - nrow(chunk), + nrow(chunk) +) # Read pre-computed S3 listing from install-deps step # This avoids loading s3fs/reticulate/Python in the build container, -# saving significant memory for pak subprocess forks +# saving significant memory for the dependency-installer subprocesses s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") sprintf("S3 cache: %s files", length(s3_cache)) n <- nrow(chunk) -# Every `trim_every` packages, bound the pkgcache _metadata dir so a full-platform -# run does not accumulate thousands of ~70 MB snapshots and fill the host disk. -# No-op on amd64 (R_PKG_CACHE_DIR is empty / cache not persisted). -trim_every <- 25L mapply( function(pkg, ver, sens, i) { cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) @@ -151,12 +158,6 @@ mapply( upload = TRUE, store_build_metadata = TRUE ) - if (i %% trim_every == 0L) { - removed <- trim_pkgcache_metadata() - if (removed > 0L) { - cat(sprintf(" [pkgcache trim] removed %d stale _metadata entries\n", removed)) - } - } }, chunk$Package, chunk$Version, diff --git a/local/install-bincraft.R b/local/install-bincraft.R index 4665b3b..e39cb4f 100644 --- a/local/install-bincraft.R +++ b/local/install-bincraft.R @@ -11,10 +11,11 @@ # # How it works: list the remote tags with `git ls-remote` (no token needed for # the public repo), keep the `vX.Y.Z` release tags, pick the highest version, -# and install it with pak. pak is idempotent on the git ref, so re-running keeps -# the package when it is already current and only updates when a newer tag ships. -# Filtering/sorting is done in R (not via git's `--sort`/refspec) so behaviour is -# identical across git versions and `system2()` argument handling. +# and install it with uvr via `local/uvr-install.sh`. uvr is idempotent on the +# git ref, so re-running keeps the package when it is already current and only +# updates when a newer tag ships. Filtering/sorting is done in R (not via git's +# `--sort`/refspec) so behaviour is identical across git versions and +# `system2()` argument handling. repo_url <- Sys.getenv( "BINCRAFT_GIT_URL", @@ -44,7 +45,43 @@ latest <- tags[order(package_version(sub("^v", "", tags)), decreasing = TRUE)][ ] message(sprintf("Installing latest bincraft release: %s", latest)) -pak::pak(sprintf("git::%s@%s", repo_url, latest)) + +# uvr addresses Forgejo repos as `forgejo::host/owner/repo@ref` rather than as a +# git URL, so drop the scheme and the trailing `.git` from `repo_url`. +spec <- sprintf( + "forgejo::%s@%s", + sub("\\.git$", "", sub("^[a-z]+://", "", repo_url)), + latest +) + +# `local/uvr-install.sh` when run from the repo root, `/work/local/` in the +# build-one image, which copies the two scripts into a flatter layout. +helper <- Sys.getenv("UVR_INSTALL_SH", unset = "") +if (!nzchar(helper)) { + candidates <- c("local/uvr-install.sh", "/work/local/uvr-install.sh") + found <- candidates[file.exists(candidates)] + if (length(found) == 0L) { + stop("Could not locate uvr-install.sh; set UVR_INSTALL_SH", call. = FALSE) + } + helper <- found[1L] +} + +# Point uvr at the R running this script and at the library it would install +# into, so the per-R-minor passes in the build pipelines (which call a different +# Rscript with R_LIBS_USER pointed elsewhere) target their own R and library. +Sys.setenv( + UVR_R_BIN = file.path(R.home("bin"), "R"), + UVR_TARGET_LIB = .libPaths()[1L] +) + +status <- system2(helper, shQuote(spec)) +if (!identical(status, 0L)) { + stop( + sprintf("uvr failed to install %s (exit %s)", spec, status), + call. = FALSE + ) +} + message(sprintf( "bincraft %s installed (%s)", as.character(utils::packageVersion("bincraft")), diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 7a3652d..887952d 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -15,15 +15,15 @@ suppressPackageStartupMessages(library(data.table)) # Sys.setenv("OS_VERSION" = "3.22") # Sys.setenv("ARCH" = "arm64") -arch = Sys.getenv("ARCH") +arch <- Sys.getenv("ARCH") # target: alpine-322, ubuntu-2404, redhat-9, etc. -platform = paste( +platform <- paste( Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-" ) # Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10") -codename = bincraft::set_codename(NULL) +codename <- bincraft::set_codename(NULL) con <- DBI::dbConnect( RPostgres::Postgres(), @@ -35,8 +35,8 @@ con <- DBI::dbConnect( sslmode = "require" ) -cran_archive = tools::CRAN_archive_db() -cran_release = tools::CRAN_package_db() +cran_archive <- tools::CRAN_archive_db() +cran_release <- tools::CRAN_package_db() # Subset cran_archive to only those packages cran_archive_in_release <- cran_archive[ names(cran_archive) %in% cran_release$Package @@ -84,13 +84,14 @@ s3fs::s3_file_system( region_name = "eu-central-003", refresh = TRUE ) -s3_pkgs = s3fs::s3_dir_ls( +s3_pkgs <- s3fs::s3_dir_ls( sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename), recurse = TRUE ) # Save the raw S3 file listing for the build step to use as s3_package_cache -# This avoids loading s3fs/reticulate in the build container, saving memory for pak forks +# This avoids loading s3fs/reticulate in the build container, saving memory for +# the dependency-installer subprocesses saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds") file_names <- basename(s3_pkgs) diff --git a/local/r-minor-helpers.R b/local/r-minor-helpers.R index 6f01c29..65aafdf 100644 --- a/local/r-minor-helpers.R +++ b/local/r-minor-helpers.R @@ -31,42 +31,3 @@ parse_build_args <- function(args) { ncpus = as.integer(pos[3L]) ) } - -# Bound the {pkgcache} metadata dir, which otherwise grows without limit: the -# "patched" repo mints a new content hash on every PACKAGES change, so each build -# writes a fresh ~70 MB _metadata/pkgs-.rds (+ patched-/) that is -# never reused. Keep the `keep` newest entries by mtime; only remove entries -# older than `min_age_secs`, so a concurrent split-job's in-flight files are -# never deleted (each build uses a unique hash, so aged entries are -# unreferenced). Stable repo dirs (CRAN-*, BioC*, INLA-*) and pkg/ downloads are -# not matched and thus preserved. Returns the number of entries removed. -trim_pkgcache_metadata <- function(cache_dir = Sys.getenv("R_PKG_CACHE_DIR"), - keep = 20L, - min_age_secs = 600) { - meta <- file.path(cache_dir, "R", "pkgcache", "_metadata") - if (!nzchar(cache_dir) || !dir.exists(meta)) { - return(0L) - } - entries <- c( - Sys.glob(file.path(meta, "patched-*")), - Sys.glob(file.path(meta, "pkgs-*.rds")) - ) - if (length(entries) == 0L) { - return(0L) - } - info <- file.info(entries) - order_new_first <- order(info$mtime, decreasing = TRUE) - ranked <- entries[order_new_first] - ranked_mtime <- info$mtime[order_new_first] - if (length(ranked) <= keep) { - return(0L) - } - candidates <- ranked[(keep + 1L):length(ranked)] - candidate_age <- as.numeric(Sys.time()) - as.numeric(ranked_mtime[(keep + 1L):length(ranked)]) - removable <- candidates[candidate_age >= min_age_secs] - if (length(removable) == 0L) { - return(0L) - } - unlink(removable, recursive = TRUE, force = TRUE) - length(removable) -} diff --git a/local/test-package-loading.R b/local/test-package-loading.R index 26a2e82..a3b7a28 100644 --- a/local/test-package-loading.R +++ b/local/test-package-loading.R @@ -1,22 +1,28 @@ -install.packages( - "pak", - repos = sprintf( - "https://r-lib.github.io/p/pak/stable/%s/%s/%s", - .Platform$pkgType, - R.Version()$os, - R.Version()$arch +# Installs every CRAN package one by one and checks that it loads. Dependencies +# go through uvr via local/uvr-install.sh, which bootstraps the uvr binary on +# first use and installs into .libPaths()[1]. +uvr_install <- function(pkg) { + Sys.setenv( + UVR_R_BIN = file.path(R.home("bin"), "R"), + UVR_TARGET_LIB = .libPaths()[1L] ) -) + status <- system2("local/uvr-install.sh", shQuote(pkg)) + if (!identical(status, 0L)) { + stop( + sprintf("uvr failed to install %s (exit %s)", pkg, status), + call. = FALSE + ) + } +} -Sys.setenv(PKG_SYSREQS = TRUE) all_pkgs <- rownames(available.packages()) -to_skip = c("ABRSQOL", "ACA", "ACE.CoCo") -all_pkgs = setdiff(all_pkgs, to_skip) +to_skip <- c("ABRSQOL", "ACA", "ACE.CoCo") +all_pkgs <- setdiff(all_pkgs, to_skip) for (i in all_pkgs) { message(sprintf("\nInstalling %s", i)) - pak::pkg_install(i) + uvr_install(i) library(i, character.only = TRUE) } @@ -323,7 +329,7 @@ if (length(to_process) == 0) { } else { for (i in to_process) { message(sprintf("\nInstalling %s", i)) - pak::pkg_install(i) + uvr_install(i) library(i, character.only = TRUE) } # Update to_skip to include all up to the last processed diff --git a/local/tests/test-trim-pkgcache.R b/local/tests/test-trim-pkgcache.R deleted file mode 100644 index 1a5b418..0000000 --- a/local/tests/test-trim-pkgcache.R +++ /dev/null @@ -1,68 +0,0 @@ -source(file.path("..", "r-minor-helpers.R")) - -# Build a fake _metadata dir under a temp R_PKG_CACHE_DIR. Each entry's mtime is -# set to `age_secs` in the past so we can exercise the age gate deterministically. -make_meta <- function(patched = 0L, pkgs = 0L, keep_repos = TRUE, age_secs = 3600) { - root <- tempfile("pkgcache-") - meta <- file.path(root, "R", "pkgcache", "_metadata") - dir.create(meta, recursive = TRUE) - old <- Sys.time() - age_secs - mk_dir <- function(p) { dir.create(p); Sys.setFileTime(p, old); p } - mk_file <- function(p) { writeLines("x", p); Sys.setFileTime(p, old); p } - for (i in seq_len(patched)) mk_dir(file.path(meta, sprintf("patched-%03d", i))) - for (i in seq_len(pkgs)) mk_file(file.path(meta, sprintf("pkgs-%03d.rds", i))) - if (keep_repos) { - mk_dir(file.path(meta, "CRAN-075c426938")) - mk_dir(file.path(meta, "BioCsoft-1ac964ed6c")) - mk_file(file.path(meta, "bioc-sysreqs.dcf.gz")) - mk_dir(file.path(root, "R", "pkgcache", "pkg")) # downloads, must survive - } - root -} - -n_churn <- function(root) { - meta <- file.path(root, "R", "pkgcache", "_metadata") - length(Sys.glob(file.path(meta, "patched-*"))) + - length(Sys.glob(file.path(meta, "pkgs-*.rds"))) -} - -test_that("empty cache_dir is a no-op", { - expect_identical(trim_pkgcache_metadata("", keep = 5L, min_age_secs = 0), 0L) -}) - -test_that("missing _metadata dir is a no-op", { - expect_identical( - trim_pkgcache_metadata(tempfile("absent-"), keep = 5L, min_age_secs = 0), - 0L - ) -}) - -test_that("fewer than keep entries removes nothing", { - root <- make_meta(patched = 2L, pkgs = 2L) - expect_identical(trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0), 0L) - expect_identical(n_churn(root), 4L) -}) - -test_that("trims down to keep newest, leaving churn == keep", { - root <- make_meta(patched = 30L, pkgs = 30L) # 60 churn entries, all old - removed <- trim_pkgcache_metadata(root, keep = 20L, min_age_secs = 0) - expect_identical(removed, 40L) - expect_identical(n_churn(root), 20L) -}) - -test_that("entries younger than min_age_secs are protected", { - root <- make_meta(patched = 30L, pkgs = 0L, keep_repos = FALSE, age_secs = 60) - # keep=5 would drop 25, but all are 60s old < 600s gate -> nothing removed - expect_identical(trim_pkgcache_metadata(root, keep = 5L, min_age_secs = 600), 0L) - expect_identical(n_churn(root), 30L) -}) - -test_that("stable repo dirs and pkg downloads are never touched", { - root <- make_meta(patched = 30L, pkgs = 30L) - trim_pkgcache_metadata(root, keep = 0L, min_age_secs = 0) - meta <- file.path(root, "R", "pkgcache", "_metadata") - expect_true(dir.exists(file.path(meta, "CRAN-075c426938"))) - expect_true(dir.exists(file.path(meta, "BioCsoft-1ac964ed6c"))) - expect_true(file.exists(file.path(meta, "bioc-sysreqs.dcf.gz"))) - expect_true(dir.exists(file.path(root, "R", "pkgcache", "pkg"))) -}) diff --git a/local/uvr-install.sh b/local/uvr-install.sh new file mode 100755 index 0000000..e1c4d0f --- /dev/null +++ b/local/uvr-install.sh @@ -0,0 +1,90 @@ +#!/bin/sh +# Install R packages into the CI library with uvr (https://github.com/nbafrank/uvr). +# +# Usage: +# local/uvr-install.sh httr2 jsonlite +# local/uvr-install.sh forgejo::codefloe.com/rpkgs/bincraft@v4.4.3 +# +# Replaces `pak::pak(...)`. uvr is project-scoped: `uvr add` refuses to run +# outside a project and always writes to `.uvr/library/`, and only +# `uvr sync --library` can target an existing library. The project is therefore +# minted in a scratch directory under TMPDIR and thrown away afterwards; that +# also keeps `uvr init`'s `.Rprofile` out of the repo checkout, where it would +# hijack `.libPaths()` for every other R call in the pipeline. +# +# Pruning is a no-op here: uvr disables it whenever `--library` is passed, +# precisely because such a target may be shared (`/mnt/cache/R-pkgs` holds +# bincraft and its dependencies alongside whatever this script installs). +# +# System dependencies come from uvr's vendored r-system-requirements rules, so +# `pak::sysreqs_db_update()` and `PKG_SYSREQS_PLATFORM` are no longer needed. +# +# Environment: +# UVR_R_BIN R interpreter to install for; set by install-bincraft.R so +# the per-R-minor passes target their own R, not the primary +# R_VERSION fallback interpreter selector (/opt/R//bin/R) +# UVR_TARGET_LIB target library; defaults to R_LIBS_USER, then to the +# active R's .libPaths()[1] (which is where pak wrote) +# UVR_INSTALL_DIR where the uvr binary lands (default /usr/local/bin) + +set -eu + +# renovate: datasource=github-releases depName=nbafrank/uvr +UVR_PIN="v0.4.4" + +if [ "$#" -eq 0 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi + +# The build images keep R under /opt/R/ and off PATH. uvr resolves the +# interpreter via PATH and never downloads one unless `uvr r install` is run, so +# put the requested R first. +r_bin="${UVR_R_BIN:-}" +if [ -z "$r_bin" ] && [ -n "${R_VERSION:-}" ] && [ -x "/opt/R/${R_VERSION}/bin/R" ]; then + r_bin="/opt/R/${R_VERSION}/bin/R" +fi +if [ -n "$r_bin" ]; then + PATH="$(dirname "$r_bin"):$PATH" + export PATH +else + r_bin="$(command -v R)" +fi + +target_lib="${UVR_TARGET_LIB:-${R_LIBS_USER:-}}" +if [ -z "$target_lib" ]; then + target_lib="$("$r_bin" --no-echo --no-save -e 'cat(.libPaths()[1])')" +fi +if [ -z "$target_lib" ]; then + echo "error: could not determine a target library; set UVR_TARGET_LIB" >&2 + exit 2 +fi +mkdir -p "$target_lib" + +# Pin the manifest to the active R so the lockfile's R stays in step with the +# library's R sentinel. Without that, uvr can decide the library is ABI-stale +# and wipe it -- and this target is shared with bincraft. uvr only discovers R +# via PATH/R_HOME (it does not scan /opt/R), so the R put on PATH above is the +# only candidate this constraint can resolve to. +# shellcheck disable=SC2016 # $major/$minor are R expressions, not shell vars +r_full="$("$r_bin" --no-echo --no-save -e 'cat(paste(R.version$major, R.version$minor, sep = "."))')" + +install_dir="${UVR_INSTALL_DIR:-/usr/local/bin}" +uvr_bin="${install_dir}/uvr" +if [ ! -x "$uvr_bin" ]; then + echo "Bootstrapping uvr ${UVR_PIN} into ${install_dir}" + UVR_INSTALL_DIR="$install_dir" UVR_VERSION="$UVR_PIN" \ + sh -c 'curl -fsSL https://raw.githubusercontent.com/nbafrank/uvr/main/install.sh | sh' +fi + +project_dir="${TMPDIR:-/tmp}/uvr-ci-$$" +rm -rf "$project_dir" +mkdir -p "$project_dir" +trap 'rm -rf "$project_dir"' EXIT +cd "$project_dir" + +"$uvr_bin" init --here --r-version "$r_full" +# --no-install: resolve and lock only. The install happens in the sync below, +# which is the only command that honours --library. +"$uvr_bin" add --no-install "$@" +"$uvr_bin" sync --library "$target_lib" --install-system-deps diff --git a/renovate.json b/renovate.json index 8c8e208..7b2c3b3 100644 --- a/renovate.json +++ b/renovate.json @@ -1,7 +1,12 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": ["local>devxy/renovate-config"], - "ignorePaths": ["docker/**", ".crow/process-updates.yaml", ".crow/build-all-versions.yaml", ".crow/weekly-rebuild-missing.yaml"], + "ignorePaths": [ + "docker/**", + ".crow/process-updates.yaml", + ".crow/build-all-versions.yaml", + ".crow/weekly-rebuild-missing.yaml" + ], "customManagers": [ { "customType": "regex", @@ -14,6 +19,13 @@ "packageNameTemplate": "https://codefloe.com/rpkgs/bincraft.git", "datasourceTemplate": "git-tags", "extractVersionTemplate": "^v?(?.+)$" + }, + { + "customType": "regex", + "fileMatch": ["^local/uvr-install\\.sh$"], + "matchStrings": [ + "# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+UVR_PIN=\"(?[^\"]+)\"" + ] } ] }