diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index abd9b3d..ef972ee 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -32,9 +32,9 @@ steps: - R -q -e 'install.packages(".", repos = NULL, quiet = FALSE); packageVersion("rBinaries")' # manually install some packages into the cache so that some builds don't fail # Hmisc -> ABCanalysis + # - R -q -e 'pak::pak("Hmisc")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs - - R -q -e 'pak::pak("Hmisc")' - - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][1:500]; library(rBinaries); future::plan("multisession", workers = 4, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))' + - R -q -e 'options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = "error"); pkgs = tools::CRAN_package_db()[[1]][3:3]; library(rBinaries); future::plan("multisession", workers = 4, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE))' backend_options: kubernetes: resources: diff --git a/R/build_binaries.R b/R/build_binaries.R index 5f26040..0e29a3f 100644 --- a/R/build_binaries.R +++ b/R/build_binaries.R @@ -135,6 +135,10 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, }, package_name, tag, future.seed = TRUE) }) + cli::cli_h2("Updating package index for package ({.pkg {package_name[1]}})") + + add_to_package_index(package_name[1]) + return(invisible(TRUE)) } @@ -186,10 +190,10 @@ build_single_tag <- function( cli::cli_alert("{.fun build_single_tag}: (3/3) Removing {.path {local_clone_dir_single}}.") unlink(local_clone_dir_single, force = TRUE, recursive = TRUE) - total_build_time <- round(difftime(Sys.time(), t1, units = "secs"), 2) + total_build_time <- round(as.numeric(difftime(Sys.time(), t1, units = "secs")), 2) # bytes to MB in binary format - file_size <- as.numeric(fs::file_size(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) / (1024^2) + file_size <- round(as.numeric(fs::file_size(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) / (1024^2), 2) # skip DB connection if package already exists if (file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) { @@ -197,12 +201,12 @@ build_single_tag <- function( dbname = "build_metadata", host = "postgres-arm-binaries-r.devxy.io", port = 15432, user = "arm_binaries", password = Sys.getenv("PGPASS") ) - dbExecute(con, sprintf( + dbExecute(con, "UPDATE single_builds SET build_duration = $1, size = $2 WHERE package_name = $3 and platform = $4 and tag = $5", params = list( total_build_time, file_size, package_name, platform, tag ) - )) + ) dbDisconnect(con) } } diff --git a/R/packages_index.R b/R/packages_index.R new file mode 100644 index 0000000..07c900a --- /dev/null +++ b/R/packages_index.R @@ -0,0 +1,54 @@ +#' @importFrom cranlike add_PACKAGES +#' @importFrom s3fs s3_dir_ls s3_file_system +#' @export +add_to_package_index <- function( + package_name = NULL, + endpoint = "https://s3.eu-central-003.backblazeb2.com", + region = "eu-central-003", + bucket = "devxy-arm64-r-binaries", + codename = NULL, r_minor_version = NULL, + build_for_minor = FALSE) { + cli::cli_h2("Building package index") + + codename <- set_codename(codename) + if (is.null(r_minor_version)) { + r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string) + } + + if (!build_for_minor) { + local_bin_dir <- sprintf("/root/__linux__/%s/latest/src/contrib/", codename) + remote_bin_dir <- sprintf("%s/__linux__/%s/latest/src/contrib/", bucket, codename) + } else { + dir_out_bin <- sprintf("/root/__linux__/%s/%s/latest/src/contrib/", codename, r_minor_version) + remote_bin_dir <- sprintf("%s/__linux__/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version) + } + + s3fs::s3_file_system( + aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"), + aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"), + endpoint = endpoint, + region_name = region, + ) + + # get latest PACKAGES file from S3 + if (s3fs::s3_file_exists(sprintf("%s/PACKAGES", remote_bin_dir))) { + s3fs::s3_file_download(sprintf("%s/PACKAGES", remote_bin_dir), sprintf("%s/PACKAGES", local_bin_dir)) + } + file_names <- list.files(local_bin_dir, pattern = sprintf("%s*", package_name)) + + # list all tarballs for the given package + cranlike::add_PACKAGES(file_names, local_bin_dir) + + cli::cli_alert("{.fun create_package_index}: Uploading PACKAGES* files to S3.") + purrr::walk2( + sprintf("%s/%s", local_bin_dir, c("PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), + sprintf("%s/%s", remote_bin_dir, c("PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), + \(x, y) s3fs::s3_file_upload(x, y, overwrite = TRUE) + ) + s3fs::s3_file_upload(sprintf("%s/PACKAGES", local_bin_dir), sprintf("%s/PACKAGES", remote_bin_dir), overwrite = TRUE) + # s3fs::s3_file_upload( + # sprintf("%s/%s", local_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), + # sprintf("%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")), + # overwrite = TRUE + # ) +}