build in new root, upload PACKAGES file at the end

.woodpecker/build.yaml

@@ -15,6 +15,45 @@ when:
 #       PLATFORM_ID: suse155
 steps:
   - name: Build
+
+    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
+    image: docker.io/pats22/arm-binaries-r-ubi9
+    pull: true
+    # to ensure the PACKAGES upload step is being executed
+    failure: ignore
+    environment:
+      GIT_USER: pat-s
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+    volumes:
+      - arm-binaries-r-dep-cache-rhel9:/mnt/cache
+    commands:
+      # download latest package version (avoid having to rebuild image every time)
+      # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
+      - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries
+      - rm -rf /mnt/cache/R-pkgs/00LOCK-arm64-r-binaries
+      - R -q -e 'install.packages(".", repos = NULL, quiet = FALSE); packageVersion("rBinaries")'
+      # manually install some packages into the cache so that some builds don't fail
+      # Hmisc -> ABCanalysis
+      # - R -q -e 'pak::pak("Hmisc")'
+      - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/binaries
+      - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][7:10]; library(rBinaries); future::plan("multisession", workers = 4, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))'
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 2000Mi
+            cpu: 2000m
+          limits:
+            memory: 6000Mi
+            cpu: 4050m
+        nodeSelector:
+          kubernetes.io/arch: 'arm64'
+  # - name: Sleep
+  #   image: alpine:3.20
+  #   commands: sleep 10
+  - name: Upload PACKAGES files
     secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
     image: docker.io/pats22/arm-binaries-r-ubi9
     pull: true
@@ -30,19 +69,16 @@ steps:
       # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
       - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git && cd arm64-r-binaries
       - R -q -e 'install.packages(".", repos = NULL, quiet = TRUE); packageVersion("rBinaries")'
-      # manually install some packages into the cache so that some builds don't fail
+      - R -q -e 'options(crayon.enabled = TRUE); library(rBinaries); upload_package_index()'
-      # Hmisc -> ABCanalysis
+      - R -q -e 'files = list.files("/mnt/cache/binaries/", include.dirs = FALSE, full.names = TRUE, recursive = TRUE); foo = file.remove(files)'
-      # - R -q -e 'pak::pak("Hmisc")'
-      - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs
-      - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][3:5]; library(rBinaries); future::plan("multisession", workers = 4, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))'
     backend_options:
       kubernetes:
         resources:
           requests:
-            memory: 2000Mi
+            memory: 300Mi
-            cpu: 2000m
+            cpu: 0m
           limits:
-            memory: 6000Mi
+            memory: 500Mi
-            cpu: 4050m
+            cpu: 1000m
         nodeSelector:
           kubernetes.io/arch: 'arm64'

NAMESPACE

@@ -9,6 +9,7 @@ export(list_metadata_tables)
 export(query_metadata_table)
 export(set_bin_path)
 export(set_codename)
+export(upload_package_index)
 export(upload_single_binary_to_s3)
 import(progressr)
 importFrom(DBI,dbConnect)
@@ -25,6 +26,7 @@ importFrom(dplyr,group_by)
 importFrom(dplyr,n)
 importFrom(dplyr,summarise)
 importFrom(dplyr,tbl)
+importFrom(fs,file_size)
 importFrom(future,plan)
 importFrom(future.apply,future_mapply)
 importFrom(gert,git_config_set)

R/build_binaries.R

@@ -7,7 +7,7 @@
 #' @export
 build_binary_package <- function(package_name, tag = NULL, codename = NULL,
                                  r_minor_version = NULL, build_for_minor = TRUE,
-                                 local_build_root = "/root",
+                                 local_build_root = "/mnt/cache/binaries/",
                                  local_clone_dir = "/tmp",
                                  platform = "redhat-9",
                                  install_system_dependencies = TRUE,
@@ -144,6 +144,7 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
 
 #' @importFrom cli cli_alert
 #' @importFrom pkgbuild build
+#' @importFrom fs file_size
 #' @export
 build_single_tag <- function(
     package_name, tag = NULL,

R/packages_index.R

@@ -6,6 +6,7 @@ add_to_package_index <- function(
     endpoint = "https://s3.eu-central-003.backblazeb2.com",
     region = "eu-central-003",
     bucket = "devxy-arm64-r-binaries",
+    local_build_root = "/mnt/cache/binaries",
     codename = NULL, r_minor_version = NULL,
     build_for_minor = FALSE,
     debug = FALSE) {
@@ -15,10 +16,10 @@ add_to_package_index <- function(
   }
 
   if (!build_for_minor) {
-    local_bin_dir <- sprintf("/root/__linux__/%s/latest/src/contrib/", codename)
+    local_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", local_build_root, codename)
     remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", bucket, codename)
   } else {
-    dir_out_bin <- sprintf("/root/__linux__/%s/%s/latest/src/contrib/", codename, r_minor_version)
+    dir_out_bin <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", local_build_root, codename, r_minor_version)
     remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version)
   }
 
@@ -30,15 +31,55 @@ add_to_package_index <- function(
   )
 
   # get latest PACKAGES file from S3
-  if (s3fs::s3_file_exists(sprintf("%s/PACKAGES", remote_bin_dir))) {
+  if (!s3fs::s3_file_exists(sprintf("%s/PACKAGES", remote_bin_dir))) {
-    s3fs::s3_file_download(sprintf("%s/PACKAGES", remote_bin_dir), sprintf("%s/PACKAGES", local_bin_dir), overwrite = TRUE)
+    s3fs::s3_file_download(sprintf("%s/PACKAGES", remote_bin_dir), sprintf("%s/PACKAGES", local_bin_dir))
   }
   file_names <- list.files(local_bin_dir, pattern = sprintf("%s*", package_name))
 
   # list all tarballs for the given package
   cranlike::add_PACKAGES(file_names, local_bin_dir)
 
+  # dummy <- s3fs::s3_file_upload(pkgs_upload_local, pkgs_upload_remote, overwrite = TRUE)
+  # s3fs::s3_file_upload(
+  #   sprintf("%s/%s", local_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
+  #   sprintf("%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
+  #   overwrite = TRUE
+  # )
+  return(invisible(TRUE))
+}
+
+#' @export
+upload_package_index <- function(
+    package_name = NULL,
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    region = "eu-central-003",
+    bucket = "devxy-arm64-r-binaries",
+    local_build_root = "/mnt/cache/binaries/",
+    codename = NULL, r_minor_version = NULL,
+    build_for_minor = FALSE,
+    debug = FALSE) {
   cli::cli_alert("{.fun create_package_index}: Uploading PACKAGES* files to S3.")
+
+  codename <- set_codename(codename)
+  if (is.null(r_minor_version)) {
+    r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
+  }
+
+  if (!build_for_minor) {
+    local_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", local_build_root, codename)
+    remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", bucket, codename)
+  } else {
+    dir_out_bin <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", local_build_root, codename, r_minor_version)
+    remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version)
+  }
+
+  s3fs::s3_file_system(
+    aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
+    aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
+    endpoint = endpoint,
+    region_name = region,
+  )
+
   dummy <- purrr::walk2(
     sprintf("%s/%s", local_bin_dir, c("PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
     sprintf("%s/%s", remote_bin_dir, c("PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
@@ -56,11 +97,5 @@ add_to_package_index <- function(
   s3 <- paws.storage::s3(endpoint = endpoint, region = region)
   s3$put_object(Body = pkgs_upload_local, Bucket = bucket, Key = "__linux__/rhel9/latest/src/contrib/PACKAGES")
 
-  # dummy <- s3fs::s3_file_upload(pkgs_upload_local, pkgs_upload_remote, overwrite = TRUE)
-  # s3fs::s3_file_upload(
-  #   sprintf("%s/%s", local_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
-  #   sprintf("%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
-  #   overwrite = TRUE
-  # )
   return(invisible(TRUE))
 }

R/upload.R

@@ -4,7 +4,8 @@ upload_single_binary_to_s3 <- function(
     endpoint = "https://s3.eu-central-003.backblazeb2.com",
     region = "eu-central-003",
     bucket = "devxy-arm64-r-binaries",
-    local_build_root = "/root", codename = NULL,
+    local_build_root = "/mnt/cache/binaries/",
+    codename = NULL,
     package_name, tag,
     r_minor_version = NULL, build_for_minor = FALSE,
     force = FALSE,