fix namespace exports

.woodpecker/process_cran_updates.yaml

@@ -0,0 +1,98 @@
+when:
+  - event: [cron]
+    cron: update-packages
+
+skip_clone: true
+
+matrix:
+  include:
+    - PLATFORM: redhat-9
+    # - PLATFORM: redhat-8
+    # - PLATFORM: ubuntu-2404
+    # - PLATFORM: ubuntu-2204
+steps:
+  - name: 'Processing Updates: (${PLATFORM})'
+    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
+    image: docker.io/pats22/arm-binaries-r-${PLATFORM}
+    pull: true
+    # to ensure the PACKAGES upload step is being executed
+    failure: ignore
+    environment:
+      GIT_USER: pat-s
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+      CCACHE_DIR: /mnt/cache/ccache
+    volumes:
+      - arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
+    commands:
+      # download latest package version (avoid having to rebuild image every time)
+      # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
+      - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git .
+      - rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries /mnt/cache/R-pkgs/00LOCK-rBinaries
+      # for some reason this one is often corrupt on redhat-9
+      - rm -rf /mnt/cache/R-pkgs/RPostgres /mnt/cache/pkgcache/R/pkgcache
+      - R -q -e 'pak::pak("dyfanjones/s3fs")'
+      - R -q -e 'devtools::install(); packageVersion("rBinaries")'
+      - mkdir -p /root/.R && /bin/bash -c 'echo -e "CXX_STD = CXX14\nCC=ccache gcc\nCPP=ccache gcc\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=ccache /usr/bin/gfortran\nFC=ccache /usr/bin/gfortran" > /root/.R/Makevars'
+      # manually install some packages into the cache so that some builds don't fail
+      # Hmisc -> ABCanalysis
+      # - R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE, PKG_SYSREQS_PLATFORM = "redhat-9"); pak::pak(c("Hmisc", "pls", "abd", "doRNG", "devtools"))'
+      # more complicated system dependencies which pak cannot resolve
+      - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
+      # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point
+      - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error", repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]][${BLOCK}]; pkgs = setdiff(pkgs, c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR",  "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA" , "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM")); library(rBinaries); future::plan("sequential" rscript_startup = quote(options(crayon.enabled = TRUE))); process_cran_updates(platform = "${PLATFORM}"))'
+    backend_options:
+      kubernetes:
+        ### PROD
+        resources:
+          requests:
+            memory: 13Gi
+            cpu: 1000m
+          limits:
+            memory: 13Gi
+            cpu: 1000m
+        # DEBUG
+        # resources:
+        #   requests:
+        #     memory: 4Gi
+        #     cpu: 2000m
+        #   limits:
+        #     memory: 4Gi
+        #     cpu: 2000m
+        nodeSelector:
+          kubernetes.io/arch: 'arm64'
+        tolerations:
+          - key: "CI"
+            operator: "Equal"
+            value: "true"
+            effect: "NoSchedule"
+  # - name: Upload PACKAGES files
+  #   secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
+  #   image: docker.io/pats22/arm-binaries-r-ubi9
+  #   pull: true
+  #   environment:
+  #     GIT_USER: pat-s
+  #     # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+  #     R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+  #     R_LIBS_USER: /mnt/cache/R-pkgs
+  #   volumes:
+  #     - arm-binaries-r-dep-cache-rhel9:/mnt/cache
+  #   commands:
+  #     # download latest package version (avoid having to rebuild image every time)
+  #     # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
+  #     - R -q -e 'pak::local_install(); packageVersion("rBinaries")'
+  #     - R -q -e 'pak::pak(c("dyfanjones/s3fs", "pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
+  #     - cd /mnt/cache/packages
+  #     - R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()'
+  #   backend_options:
+  #     kubernetes:
+  #       resources:
+  #         requests:
+  #           memory: 1000Mi
+  #           cpu: 0m
+  #         limits:
+  #           memory: 1300Mi
+  #           cpu: 1000m
+  #       nodeSelector:
+  #         kubernetes.io/arch: 'arm64'

NAMESPACE

@@ -6,10 +6,10 @@ export(build_single_tag)
 export(dbcon)
 export(install_package_system_dependencies)
 export(list_metadata_tables)
+export(process_cran_updates)
 export(query_metadata_table)
 export(set_bin_path)
 export(set_codename)
-export(update_packages)
 export(upload_package_index)
 export(upload_single_binary_to_s3)
 import(progressr)

R/process_cran_updates.R

@@ -0,0 +1,59 @@
+#' @title Process updated and new CRAN packages
+#' @description
+#' Packages which got removed from CRAN can be deleted by setting `prune = TRUE`.
+#' Argument `interval` allows to specify a range which should be processed.
+#'
+#' @importFrom dplyr bind_rows
+#' @importFrom purrr walk2
+#' @export
+process_cran_updates <- function(
+    package_name,
+    tag,
+    platform = platform,
+    local_clone_dir,
+    interval = lubridate::today(),
+    prune = TRUE,
+    build_for_minor = FALSE,
+    codename = NULL,
+    r_minor_version = NULL,
+    local_build_root = ".",
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    region = "eu-central-003",
+    bucket = "devxy-arm64-r-binaries") {
+  # Get list of updated and new packages for a specific day
+  updated_pkgs <- get_updated_cran_packages()
+  new_pkgs <- get_new_cran_packages()
+
+  all_pkgs <- dplyr::bind_rows(updated_pkgs, new_pkgs)
+
+  purrr::walk2(all_pkgs$name, all_pkgs$version, ~ build_binary_package(.x, .y))
+
+  if (prune) {
+    removed_pkgs <- get_removed_cran_packages(interval)
+
+    s3fs::s3_file_system(
+      aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
+      aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
+      endpoint = endpoint,
+      region_name = region,
+    )
+
+    if (!build_for_minor) {
+      local_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", local_build_root, codename)
+      remote_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", bucket, codename)
+    } else {
+      dir_out_bin <- sprintf("%s/arm64/%s/%s/latest/src/contrib", local_build_root, codename, r_minor_version)
+      remote_bin_dir <- sprintf("%s/arm64/%s/%s/latest/src/contrib", bucket, codename, r_minor_version)
+    }
+
+    files <- s3fs::s3_dir_ls(remote_bin_dir)
+
+    purrr::walk(removed_pkgs$name, ~ {
+      cli::cli_alert("{.fun update_packages}: Removing package {.pkg {.x}} from S3.")
+      files_filtered <- grep("bold_", files, value = TRUE)
+      s3fs::s3_file_delete_async(files_filtered)
+
+      cli::cli_alert_success("{.fun update_packages}: Successfully removed {.pkg {basename(files_filtered)}} from S3.")
+    })
+  }
+}

man/process_cran_updates.Rd

@@ -0,0 +1,26 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/process_cran_updates.R
+\name{process_cran_updates}
+\alias{process_cran_updates}
+\title{Process updated and new CRAN packages}
+\usage{
+process_cran_updates(
+  package_name,
+  tag,
+  platform = platform,
+  local_clone_dir,
+  interval = lubridate::today(),
+  prune = TRUE,
+  build_for_minor = FALSE,
+  codename = NULL,
+  r_minor_version = NULL,
+  local_build_root = ".",
+  endpoint = "https://s3.eu-central-003.backblazeb2.com",
+  region = "eu-central-003",
+  bucket = "devxy-arm64-r-binaries"
+)
+}
+\description{
+Packages which got removed from CRAN can be deleted by setting \code{prune = TRUE}.
+Argument \code{interval} allows to specify a range which should be processed.
+}