update package index after each build

.woodpecker/build.yaml

@@ -32,9 +32,9 @@ steps:
       - R -q -e 'install.packages(".", repos = NULL, quiet = FALSE); packageVersion("rBinaries")'
       # manually install some packages into the cache so that some builds don't fail
       # Hmisc -> ABCanalysis
+      # - R -q -e 'pak::pak("Hmisc")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs
-      - R -q -e 'pak::pak("Hmisc")'
+      - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][3:3]; library(rBinaries); future::plan("multisession", workers = 4, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))'
-      - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][1:500]; library(rBinaries); future::plan("multisession", workers = 4, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))'
     backend_options:
       kubernetes:
         resources:

R/build_binaries.R

@@ -135,6 +135,10 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL,
     }, package_name, tag, future.seed = TRUE)
   })
 
+  cli::cli_h2("Updating package index for package ({.pkg {package_name[1]}})")
+
+  add_to_package_index(package_name[1])
+
   return(invisible(TRUE))
 }
 
@@ -186,10 +190,10 @@ build_single_tag <- function(
     cli::cli_alert("{.fun build_single_tag}: (3/3) Removing {.path {local_clone_dir_single}}.")
     unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
 
-    total_build_time <- round(difftime(Sys.time(), t1, units = "secs"), 2)
+    total_build_time <- round(as.numeric(difftime(Sys.time(), t1, units = "secs")), 2)
 
     # bytes to MB in binary format
-    file_size <- as.numeric(fs::file_size(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) / (1024^2)
+    file_size <- round(as.numeric(fs::file_size(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) / (1024^2), 2)
 
     # skip DB connection if package already exists
     if (file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
@@ -197,12 +201,12 @@ build_single_tag <- function(
         dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io",
         port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS")
       )
-      dbExecute(con, sprintf(
+      dbExecute(con,
         "UPDATE single_builds SET build_duration = $1, size = $2 WHERE package_name = $3 and platform = $4 and tag = $5",
         params = list(
           total_build_time, file_size, package_name, platform, tag
         )
-      ))
+      )
       dbDisconnect(con)
     }
   }

R/packages_index.R

@@ -0,0 +1,54 @@
+#' @importFrom cranlike add_PACKAGES
+#' @importFrom s3fs s3_dir_ls s3_file_system
+#' @export
+add_to_package_index <- function(
+    package_name = NULL,
+    endpoint = "https://s3.eu-central-003.backblazeb2.com",
+    region = "eu-central-003",
+    bucket = "devxy-arm64-r-binaries",
+    codename = NULL, r_minor_version = NULL,
+    build_for_minor = FALSE) {
+  cli::cli_h2("Building package index")
+
+  codename <- set_codename(codename)
+  if (is.null(r_minor_version)) {
+    r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
+  }
+
+  if (!build_for_minor) {
+    local_bin_dir <- sprintf("/root/__linux__/%s/latest/src/contrib/", codename)
+    remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", bucket, codename)
+  } else {
+    dir_out_bin <- sprintf("/root/__linux__/%s/%s/latest/src/contrib/", codename, r_minor_version)
+    remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version)
+  }
+
+  s3fs::s3_file_system(
+    aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
+    aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
+    endpoint = endpoint,
+    region_name = region,
+  )
+
+  # get latest PACKAGES file from S3
+  if (s3fs::s3_file_exists(sprintf("%s/PACKAGES", remote_bin_dir))) {
+    s3fs::s3_file_download(sprintf("%s/PACKAGES", remote_bin_dir), sprintf("%s/PACKAGES", local_bin_dir))
+  }
+  file_names <- list.files(local_bin_dir, pattern = sprintf("%s*", package_name))
+
+  # list all tarballs for the given package
+  cranlike::add_PACKAGES(file_names, local_bin_dir)
+
+  cli::cli_alert("{.fun create_package_index}: Uploading PACKAGES* files to S3.")
+  purrr::walk2(
+    sprintf("%s/%s", local_bin_dir, c("PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
+    sprintf("%s/%s", remote_bin_dir, c("PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
+    \(x, y) s3fs::s3_file_upload(x, y, overwrite = TRUE)
+  )
+  s3fs::s3_file_upload(sprintf("%s/PACKAGES", local_bin_dir), sprintf("%s/PACKAGES", remote_bin_dir), overwrite = TRUE)
+  # s3fs::s3_file_upload(
+  #   sprintf("%s/%s", local_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
+  #   sprintf("%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
+  #   overwrite = TRUE
+  # )
+}