fix namespace exports
This commit is contained in:
parent
123c206460
commit
1dff54c001
GPG key ID:
3C6318841EF78925
4 changed files with 8 additions and 8 deletions
98
.woodpecker/process_cran_updates.yaml
Normal file
98
.woodpecker/process_cran_updates.yaml
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
when:
|
||||
- event: [cron]
|
||||
cron: update-packages
|
||||
|
||||
skip_clone: true
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- PLATFORM: redhat-9
|
||||
# - PLATFORM: redhat-8
|
||||
# - PLATFORM: ubuntu-2404
|
||||
# - PLATFORM: ubuntu-2204
|
||||
steps:
|
||||
- name: 'Processing Updates: (${PLATFORM})'
|
||||
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
|
||||
image: docker.io/pats22/arm-binaries-r-${PLATFORM}
|
||||
pull: true
|
||||
# to ensure the PACKAGES upload step is being executed
|
||||
failure: ignore
|
||||
environment:
|
||||
GIT_USER: pat-s
|
||||
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
||||
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
CCACHE_DIR: /mnt/cache/ccache
|
||||
volumes:
|
||||
- arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
|
||||
commands:
|
||||
# download latest package version (avoid having to rebuild image every time)
|
||||
# rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
|
||||
- git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git .
|
||||
- rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries /mnt/cache/R-pkgs/00LOCK-rBinaries
|
||||
# for some reason this one is often corrupt on redhat-9
|
||||
- rm -rf /mnt/cache/R-pkgs/RPostgres /mnt/cache/pkgcache/R/pkgcache
|
||||
- R -q -e 'pak::pak("dyfanjones/s3fs")'
|
||||
- R -q -e 'devtools::install(); packageVersion("rBinaries")'
|
||||
- mkdir -p /root/.R && /bin/bash -c 'echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" > /root/.R/Makevars'
|
||||
# manually install some packages into the cache so that some builds don't fail
|
||||
# Hmisc -> ABCanalysis
|
||||
# - R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))'
|
||||
# more complicated system dependencies which pak cannot resolve
|
||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
# set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point
|
||||
- R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error", repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]][${BLOCK}]; pkgs = setdiff(pkgs, c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA" , "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM")); library(rBinaries); future::plan("sequential" rscript_startup = quote(options(crayon.enabled = TRUE))); process_cran_updates(platform = "${PLATFORM}"))'
|
||||
backend_options:
|
||||
kubernetes:
|
||||
### PROD
|
||||
resources:
|
||||
requests:
|
||||
memory: 13Gi
|
||||
cpu: 1000m
|
||||
limits:
|
||||
memory: 13Gi
|
||||
cpu: 1000m
|
||||
# DEBUG
|
||||
# resources:
|
||||
# requests:
|
||||
# memory: 4Gi
|
||||
# cpu: 2000m
|
||||
# limits:
|
||||
# memory: 4Gi
|
||||
# cpu: 2000m
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: 'arm64'
|
||||
tolerations:
|
||||
- key: "CI"
|
||||
operator: "Equal"
|
||||
value: "true"
|
||||
effect: "NoSchedule"
|
||||
# - name: Upload PACKAGES files
|
||||
# secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
|
||||
# image: docker.io/pats22/arm-binaries-r-ubi9
|
||||
# pull: true
|
||||
# environment:
|
||||
# GIT_USER: pat-s
|
||||
# # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
||||
# R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
||||
# R_LIBS_USER: /mnt/cache/R-pkgs
|
||||
# volumes:
|
||||
# - arm-binaries-r-dep-cache-rhel9:/mnt/cache
|
||||
# commands:
|
||||
# # download latest package version (avoid having to rebuild image every time)
|
||||
# # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
|
||||
# - R -q -e 'pak::local_install(); packageVersion("rBinaries")'
|
||||
# - R -q -e 'pak::pak(c("dyfanjones/s3fs", "pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
|
||||
# - cd /mnt/cache/packages
|
||||
# - R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()'
|
||||
# backend_options:
|
||||
# kubernetes:
|
||||
# resources:
|
||||
# requests:
|
||||
# memory: 1000Mi
|
||||
# cpu: 0m
|
||||
# limits:
|
||||
# memory: 1300Mi
|
||||
# cpu: 1000m
|
||||
# nodeSelector:
|
||||
# kubernetes.io/arch: 'arm64'
|
||||
|
|
@ -6,10 +6,10 @@ export(build_single_tag)
|
|||
export(dbcon)
|
||||
export(install_package_system_dependencies)
|
||||
export(list_metadata_tables)
|
||||
export(process_cran_updates)
|
||||
export(query_metadata_table)
|
||||
export(set_bin_path)
|
||||
export(set_codename)
|
||||
export(update_packages)
|
||||
export(upload_package_index)
|
||||
export(upload_single_binary_to_s3)
|
||||
import(progressr)
|
||||
|
|
|
|||
59
R/process_cran_updates.R
Normal file
59
R/process_cran_updates.R
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
#' @title Process updated and new CRAN packages
|
||||
#' @description
|
||||
#' Packages which got removed from CRAN can be deleted by setting `prune = TRUE`.
|
||||
#' Argument `interval` allows to specify a range which should be processed.
|
||||
#'
|
||||
#' @importFrom dplyr bind_rows
|
||||
#' @importFrom purrr walk2
|
||||
#' @export
|
||||
process_cran_updates <- function(
|
||||
package_name,
|
||||
tag,
|
||||
platform = platform,
|
||||
local_clone_dir,
|
||||
interval = lubridate::today(),
|
||||
prune = TRUE,
|
||||
build_for_minor = FALSE,
|
||||
codename = NULL,
|
||||
r_minor_version = NULL,
|
||||
local_build_root = ".",
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries") {
|
||||
# Get list of updated and new packages for a specific day
|
||||
updated_pkgs <- get_updated_cran_packages()
|
||||
new_pkgs <- get_new_cran_packages()
|
||||
|
||||
all_pkgs <- dplyr::bind_rows(updated_pkgs, new_pkgs)
|
||||
|
||||
purrr::walk2(all_pkgs$name, all_pkgs$version, ~ build_binary_package(.x, .y))
|
||||
|
||||
if (prune) {
|
||||
removed_pkgs <- get_removed_cran_packages(interval)
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
endpoint = endpoint,
|
||||
region_name = region,
|
||||
)
|
||||
|
||||
if (!build_for_minor) {
|
||||
local_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", bucket, codename)
|
||||
} else {
|
||||
dir_out_bin <- sprintf("%s/arm64/%s/%s/latest/src/contrib", local_build_root, codename, r_minor_version)
|
||||
remote_bin_dir <- sprintf("%s/arm64/%s/%s/latest/src/contrib", bucket, codename, r_minor_version)
|
||||
}
|
||||
|
||||
files <- s3fs::s3_dir_ls(remote_bin_dir)
|
||||
|
||||
purrr::walk(removed_pkgs$name, ~ {
|
||||
cli::cli_alert("{.fun update_packages}: Removing package {.pkg {.x}} from S3.")
|
||||
files_filtered <- grep("bold_", files, value = TRUE)
|
||||
s3fs::s3_file_delete_async(files_filtered)
|
||||
|
||||
cli::cli_alert_success("{.fun update_packages}: Successfully removed {.pkg {basename(files_filtered)}} from S3.")
|
||||
})
|
||||
}
|
||||
}
|
||||
26
man/process_cran_updates.Rd
Normal file
26
man/process_cran_updates.Rd
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
% Generated by roxygen2: do not edit by hand
|
||||
% Please edit documentation in R/process_cran_updates.R
|
||||
\name{process_cran_updates}
|
||||
\alias{process_cran_updates}
|
||||
\title{Process updated and new CRAN packages}
|
||||
\usage{
|
||||
process_cran_updates(
|
||||
package_name,
|
||||
tag,
|
||||
platform = platform,
|
||||
local_clone_dir,
|
||||
interval = lubridate::today(),
|
||||
prune = TRUE,
|
||||
build_for_minor = FALSE,
|
||||
codename = NULL,
|
||||
r_minor_version = NULL,
|
||||
local_build_root = ".",
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries"
|
||||
)
|
||||
}
|
||||
\description{
|
||||
Packages which got removed from CRAN can be deleted by setting \code{prune = TRUE}.
|
||||
Argument \code{interval} allows to specify a range which should be processed.
|
||||
}
|
||||
Loading…
Reference in a new issue