move package assets to own repo
This commit is contained in:
parent
acac6a5d96
commit
9a10fc6856
GPG key ID:
3C6318841EF78925
26 changed files with 1 additions and 1452 deletions
|
|
@ -1,18 +0,0 @@
|
|||
^.*\.Rproj$
|
||||
^\.Rproj\.user$
|
||||
.envrc
|
||||
^LICENSE\.md$
|
||||
.editorconfig
|
||||
.markdownlint.yaml
|
||||
.prettierrc.json
|
||||
.woodpecker
|
||||
devxy-arm64-r-binaries/
|
||||
^PACKAGES.*
|
||||
s3:*
|
||||
docker
|
||||
.*.sh
|
||||
Justfile
|
||||
renovate.json
|
||||
exec.R
|
||||
get-all-pkg-versions.R
|
||||
05-get-updated-cran-pkgs.R
|
||||
|
|
@ -38,6 +38,7 @@ steps:
|
|||
# for some reason paks pkgcache is often corrupt
|
||||
- rm -rf /mnt/cache/R-pkgs/pkgcache/ /mnt/cache/pkgcache/ /root/.cache/R/
|
||||
- R -q -e 'pak::meta_update()'
|
||||
- R -q -e 'pak::pak("git::https://gitlab.com/devxy/r-package-binaries/bincraftr.git")'
|
||||
- R -q -e 'packageVersion("bincraftR")'
|
||||
# more complicated system dependencies which pak cannot resolve
|
||||
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
||||
|
|
|
|||
37
DESCRIPTION
37
DESCRIPTION
|
|
@ -1,37 +0,0 @@
|
|||
Package: bincraftR
|
||||
Title: Toolbox for building R package binaries on Linux
|
||||
Version: 1.0.0
|
||||
Authors@R:
|
||||
person("Patrick", "Schratz", , "mail@devxy.io", role = c("aut", "cre"))
|
||||
Description: Help to build, archive, remove and update R binary packages
|
||||
for Linux. Supports various Linux distributions.
|
||||
License: GPL (>= 3)
|
||||
Imports:
|
||||
cli,
|
||||
cranlike,
|
||||
DBI,
|
||||
dplyr,
|
||||
future,
|
||||
future.apply,
|
||||
future.callr,
|
||||
gert,
|
||||
lubridate,
|
||||
magrittr,
|
||||
pkgbuild,
|
||||
progress,
|
||||
progressr,
|
||||
purrr,
|
||||
RPostgres,
|
||||
s3fs (>= 0.1.7),
|
||||
stringr,
|
||||
xml2
|
||||
Suggests:
|
||||
dbplyr
|
||||
Remotes:
|
||||
pat-s/cranlike@s3,
|
||||
pat-s/desc@description-from-remote
|
||||
Encoding: UTF-8
|
||||
Roxygen: list(markdown = TRUE)
|
||||
RoxygenNote: 7.3.2
|
||||
SystemRequirements: libgit2 (>= 1.0): libgit2-devel (rpm) or libgit2-dev
|
||||
(deb)
|
||||
65
NAMESPACE
65
NAMESPACE
|
|
@ -1,65 +0,0 @@
|
|||
# Generated by roxygen2: do not edit by hand
|
||||
|
||||
export(add_to_package_index)
|
||||
export(archive_package)
|
||||
export(build_binary_package)
|
||||
export(build_single_tag)
|
||||
export(dbcon)
|
||||
export(get_cranberries_feed)
|
||||
export(get_new_cran_packages)
|
||||
export(get_removed_cran_packages)
|
||||
export(get_updated_cran_packages)
|
||||
export(install_package_system_dependencies)
|
||||
export(list_metadata_tables)
|
||||
export(process_cran_updates)
|
||||
export(process_cranberries_rss)
|
||||
export(query_metadata_table)
|
||||
export(set_bin_path)
|
||||
export(set_codename)
|
||||
export(upload_package_index)
|
||||
export(upload_single_binary_to_s3)
|
||||
import(progressr)
|
||||
importFrom(DBI,dbConnect)
|
||||
importFrom(DBI,dbDisconnect)
|
||||
importFrom(DBI,dbExecute)
|
||||
importFrom(DBI,dbGetQuery)
|
||||
importFrom(DBI,dbListTables)
|
||||
importFrom(DBI,dbWriteTable)
|
||||
importFrom(RPostgres,Postgres)
|
||||
importFrom(cli,cli_alert)
|
||||
importFrom(cranlike,add_PACKAGES)
|
||||
importFrom(cranlike,update_PACKAGES)
|
||||
importFrom(dplyr,bind_rows)
|
||||
importFrom(dplyr,filter)
|
||||
importFrom(dplyr,group_by)
|
||||
importFrom(dplyr,n)
|
||||
importFrom(dplyr,summarise)
|
||||
importFrom(dplyr,tbl)
|
||||
importFrom(fs,file_move)
|
||||
importFrom(fs,file_size)
|
||||
importFrom(future,plan)
|
||||
importFrom(future.apply,future_mapply)
|
||||
importFrom(gert,git_clone)
|
||||
importFrom(gert,git_config_global_set)
|
||||
importFrom(lubridate,"%within%")
|
||||
importFrom(lubridate,as_date)
|
||||
importFrom(lubridate,dmy_hms)
|
||||
importFrom(lubridate,interval)
|
||||
importFrom(lubridate,today)
|
||||
importFrom(magrittr,"%>%")
|
||||
importFrom(pak,local_install_dev_deps)
|
||||
importFrom(pkgbuild,build)
|
||||
importFrom(progressr,progressor)
|
||||
importFrom(progressr,with_progress)
|
||||
importFrom(purrr,list_rbind)
|
||||
importFrom(purrr,map)
|
||||
importFrom(purrr,walk2)
|
||||
importFrom(s3fs,s3_dir_ls)
|
||||
importFrom(s3fs,s3_file_exists)
|
||||
importFrom(s3fs,s3_file_system)
|
||||
importFrom(s3fs,s3_file_upload)
|
||||
importFrom(stringr,str_split)
|
||||
importFrom(xml2,read_xml)
|
||||
importFrom(xml2,xml_find_all)
|
||||
importFrom(xml2,xml_find_first)
|
||||
importFrom(xml2,xml_text)
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
#' Archive packages in CRAN-like repositories
|
||||
#' @importFrom stringr str_split
|
||||
#' @export
|
||||
#' @examples
|
||||
#' archive_package("AATtools", codename = "rhel9")
|
||||
#' archive_package("adw", codename = "rhel8", arch = "amd64")
|
||||
#'
|
||||
archive_package <- function(
|
||||
package_name,
|
||||
build_for_minor = FALSE,
|
||||
codename = NULL,
|
||||
r_minor_version = NULL,
|
||||
local_build_root = ".",
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries",
|
||||
pause = NULL,
|
||||
arch = NULL,
|
||||
debug = FALSE) {
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
endpoint = endpoint,
|
||||
region_name = region,
|
||||
)
|
||||
|
||||
if (debug) {
|
||||
message(sprintf("DEBUG archive_package: package_name: %s", package_name))
|
||||
}
|
||||
|
||||
codename <- set_codename(codename)
|
||||
|
||||
if (is.null(arch)) {
|
||||
local_arch <- Sys.info()[["machine"]]
|
||||
if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
|
||||
arch <- "arm64"
|
||||
} else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
|
||||
arch <- "amd64"
|
||||
}
|
||||
}
|
||||
|
||||
if (!build_for_minor) {
|
||||
local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename)
|
||||
} else {
|
||||
dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version)
|
||||
}
|
||||
|
||||
# suppress progressr output here
|
||||
progressr::handlers("void")
|
||||
# don't parallelise
|
||||
future::plan("sequential")
|
||||
|
||||
files <- s3fs::s3_dir_ls(remote_bin_dir)
|
||||
|
||||
foo <- lapply(package_name, function(.x) {
|
||||
cli::cli_h2("Archiving ({.pkg {.x}})")
|
||||
if (!s3fs::s3_dir_exists(sprintf("%s/Archive/%s", remote_bin_dir, .x))) {
|
||||
s3fs::s3_dir_create(sprintf("%s/Archive/%s", remote_bin_dir, .x))
|
||||
}
|
||||
all_versions <- grep(sprintf("/%s_", .x), files, value = TRUE)
|
||||
# only archive if more than one package exists in the root
|
||||
if (length(all_versions) > 1) {
|
||||
# get most recent version from CRAN
|
||||
last_version <- available.packages("https://cloud.r-project.org/src/contrib")[.x, "Version"]
|
||||
# check if last version is available in repo
|
||||
if (any(grepl(sprintf("^%s_%s.tar.gz", package_name, last_version), all_versions))) {
|
||||
index <- which(grepl(sprintf("_%s.tar.gz", last_version), all_versions, fixed = TRUE))
|
||||
old_versions <- all_versions[-index]
|
||||
} else {
|
||||
versions <- rev(pak::pkg_history(.x)$Version)
|
||||
for (i in versions) {
|
||||
if (any(grepl(paste0("^", i, "$"), stringr::str_split(stringr::str_split(all_versions, "_", simplify = T)[, 2], ".tar.gz", simplify = TRUE)[, 1]))) {
|
||||
index <- which(grepl(sprintf("_%s.tar.gz", i), all_versions))
|
||||
old_versions <- all_versions[-index]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
cli::cli_alert("{.fun archive_package}: Archiving {.field {basename(old_versions)}}, keeping {.field {basename(all_versions[index])}}.")
|
||||
s3fs::s3_file_move(old_versions, sprintf("%s/Archive/%s/%s", remote_bin_dir, .x, basename(old_versions)), overwrite = TRUE)
|
||||
cli::cli_alert_success("{.fun archive_package}: Successfully archived package {.pkg {.x}}.")
|
||||
if (!is.null(pause)) {
|
||||
Sys.sleep(pause)
|
||||
}
|
||||
} else {
|
||||
cli::cli_alert("{.fun archive_package}: Skipping {.pkg {.x}} as only one package versions exists.")
|
||||
}
|
||||
})
|
||||
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
|
@ -1,313 +0,0 @@
|
|||
#' Build R binary packages
|
||||
#' @import progressr
|
||||
#' @importFrom future plan
|
||||
#' @importFrom future.apply future_mapply
|
||||
#' @importFrom progressr with_progress progressor
|
||||
#' @importFrom gert git_config_global_set git_clone
|
||||
#' @importFrom pak local_install_dev_deps
|
||||
#' @importFrom pkgbuild build
|
||||
#' @export
|
||||
build_binary_package <- function(package_name, tag = NULL, codename = NULL,
|
||||
r_minor_version = NULL, build_for_minor = FALSE,
|
||||
local_build_root = "/root",
|
||||
local_clone_dir = "/tmp",
|
||||
platform = NULL,
|
||||
arch = NULL,
|
||||
install_system_dependencies = TRUE,
|
||||
deps_verbose = FALSE,
|
||||
debug = FALSE,
|
||||
force = FALSE,
|
||||
archive = TRUE,
|
||||
future_strategy = "multisession",
|
||||
future_workers = 2) {
|
||||
cli::cli_h2("Preparations ({.pkg {package_name}})")
|
||||
codename <- set_codename(codename)
|
||||
|
||||
# map the 'pak' platform names to the ones used in s3
|
||||
if (is.null(platform)) {
|
||||
platform <- switch(codename,
|
||||
"jammy" = "ubuntu-2204",
|
||||
"noble" = "ubuntu-2404",
|
||||
"rhel9" = "redhat-9",
|
||||
"rhel8" = "redhat-8",
|
||||
"alpine320" = "alpine-320"
|
||||
)
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
cli::cli_alert_warning("DEBUG: codename {codename}.")
|
||||
}
|
||||
|
||||
if (is.null(r_minor_version)) {
|
||||
r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
|
||||
}
|
||||
|
||||
# create directory structure
|
||||
dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
|
||||
|
||||
local_bin_path <- set_bin_path(r_minor_version, build_for_minor, local_build_root = local_build_root, codename)
|
||||
|
||||
# set arch
|
||||
local_arch <- Sys.info()[["machine"]]
|
||||
if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
|
||||
arch <- "arm64"
|
||||
} else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
|
||||
arch <- "amd64"
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
cli::cli_alert_warning("DEBUG: dir_out_bin {dir_out_bin}.")
|
||||
}
|
||||
|
||||
dir_out_src <- sprintf("%s/src/contrib/Archive", local_build_root)
|
||||
if (debug) {
|
||||
cli::cli_alert("{.fun build_binary_package}: Creating bin dir {.path {dir_out_bin}}.")
|
||||
cli::cli_alert("{.fun build_binary_package}: Creating src dir {.path {dir_out_src}}.")
|
||||
}
|
||||
dir.create(sprintf("%s/Archive", dir_out_bin), sprintf("%s/Archive", dir_out_src),
|
||||
recursive = TRUE
|
||||
)
|
||||
|
||||
cli::cli_h2("Installing system dependencies ({.pkg {package_name}})")
|
||||
|
||||
gert::git_config_global_set("advice.detachedHead", "false")
|
||||
|
||||
if (is.null(tag) || tag == "latest") {
|
||||
gert::git_clone(sprintf("https://github.com/cran/%s", package_name),
|
||||
path = sprintf("%s/%s", tempdir(), "tmp1"),
|
||||
verbose = FALSE
|
||||
)
|
||||
# gert cannot sort by date (which is a problem for properly sorting tags like 1.0-10 and others)
|
||||
if (!is.null(tag) && tag == "latest") {
|
||||
tag = system("git tag --sort=-creatordate | head -1", intern = TRUE)
|
||||
} else {
|
||||
# Retrieve all tags
|
||||
all_tags <- gert::git_tag_list(repo = sprintf("%s/%s", tempdir(), "tmp1"))
|
||||
# filter out tags that start with R- (= non-valid ones)
|
||||
all_tags <- all_tags[!grepl("R-", all_tags$name), ]
|
||||
|
||||
unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE)
|
||||
tag <- all_tags$name
|
||||
}
|
||||
package_name <- rep(package_name, length(tag))
|
||||
}
|
||||
|
||||
t1 <- Sys.time()
|
||||
cli::cli_h2("Building ({.pkg {package_name[1]}})")
|
||||
|
||||
cli::cli_alert("[{format(Sys.time(), format='%H:%M:%S')}] Building binaries for {.pkg {package_name[1]}} with tags {.field {tag}}.")
|
||||
|
||||
future::plan(future_strategy,
|
||||
workers = future_workers,
|
||||
rscript_startup = quote(options(crayon.enabled = TRUE))
|
||||
)
|
||||
|
||||
# 'cli' is slow -> https://github.com/HenrikBengtsson/progressr/issues/167
|
||||
if (debug) {
|
||||
progressr::handlers("debug")
|
||||
} else {
|
||||
progressr::handlers("progress")
|
||||
}
|
||||
p <- progressr::progressor(along = tag)
|
||||
|
||||
worker_fun <- function(x, y, p, debug) {
|
||||
p(message = sprintf("Building '%s'", y))
|
||||
tryCatch(
|
||||
{
|
||||
dump <- build_single_tag(x, y, dir_out_bin, local_clone_dir,
|
||||
platform = platform, arch = arch, debug = debug, force = force,
|
||||
install_system_dependencies = install_system_dependencies,
|
||||
deps_verbose = deps_verbose
|
||||
)
|
||||
|
||||
p(message = sprintf("Done building '%s'", y))
|
||||
|
||||
# if for some reason an underlying error didnt' get caught in the tryCatch calls, we check again here for the existence of the binary file on disk and mark the build as failed if it is not found
|
||||
tarball_name <- sprintf("%s_%s.tar.gz", x, y)
|
||||
if (fs::file_exists(sprintf("%s/%s", local_bin_path, tarball_name))) {
|
||||
cli::cli_alert_success("Successfully built package {.pkg {x}} with tag {.field {y}}.")
|
||||
} else {
|
||||
cli::cli_alert_warning("Error in building package {.pkg {x}} with tag {.field {y}}: Uncommon/unspecific error during build.")
|
||||
store_build_metadata(x, y, platform, error_occurred = TRUE, force = TRUE, arch = arch, error = "Uncommon/unspecific error during build")
|
||||
}
|
||||
},
|
||||
error = function(e) {
|
||||
cli::cli_alert_warning("Error in building package {.pkg {package_name}} with tag {.field {tag}}: {e}")
|
||||
local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, x, y)
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
# only stderr contains the important information why the build failed
|
||||
store_build_metadata(x, y, platform, error_occurred = TRUE, arch = arch, force = TRUE, error = e$stderr)
|
||||
}
|
||||
)
|
||||
p(message = sprintf("Finished building %s %s", x, y))
|
||||
}
|
||||
|
||||
|
||||
if (debug) {
|
||||
mapply(worker_fun, package_name, tag, MoreArgs = list(p, debug))
|
||||
} else {
|
||||
future.apply::future_mapply(worker_fun, package_name, tag,
|
||||
future.seed = TRUE, MoreArgs = list(p, debug)
|
||||
)
|
||||
}
|
||||
|
||||
total_build_time <- round(Sys.time() - t1, 2)
|
||||
cli::cli_alert("Execution time ({.pkg {package_name[1]}}) ({length(tag)} tags): {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.")
|
||||
|
||||
# out <- progressr::with_progress({
|
||||
# p <- progressr::progressor(along = tag)
|
||||
# future.apply::future_mapply(function(x, y) {
|
||||
mapply(function(x, y) {
|
||||
tryCatch(
|
||||
{
|
||||
# p()
|
||||
dump <- upload_single_binary_to_s3(package_name = x, tag = y, force = force, build_for_minor = build_for_minor, debug = debug)
|
||||
},
|
||||
error = function(e) {
|
||||
message(sprintf("Error in uploading package %s with tag %s: %s", x, y, e))
|
||||
}
|
||||
)
|
||||
# }, package_name, tag, future.seed = TRUE)
|
||||
}, package_name, tag)
|
||||
|
||||
if (archive) {
|
||||
archive_package(package_name[1], debug = debug)
|
||||
}
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
||||
#' @importFrom cli cli_alert
|
||||
#' @importFrom pkgbuild build
|
||||
#' @importFrom fs file_size file_move
|
||||
#' @export
|
||||
build_single_tag <- function(
|
||||
package_name,
|
||||
tag = NULL,
|
||||
platform,
|
||||
arch,
|
||||
dir_out_bin,
|
||||
local_clone_dir,
|
||||
debug = FALSE,
|
||||
force = FALSE,
|
||||
install_system_dependencies = TRUE,
|
||||
deps_verbose = FALSE) {
|
||||
cli::cli_alert("{.fun build_single_tag}: (1/3) Cloning package {.pkg {package_name}} with tag {.field {tag}}.")
|
||||
|
||||
local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag)
|
||||
|
||||
# Using system git here as {gert} does not provide this functionality to checkout a branch by tag
|
||||
system2("git", args = c(
|
||||
"clone", "-q", sprintf("--branch=%s", tag),
|
||||
sprintf("https://github.com/cran/%s", package_name), local_clone_dir_single
|
||||
))
|
||||
|
||||
### Install system dependencies
|
||||
if (install_system_dependencies) {
|
||||
tryCatch(
|
||||
{
|
||||
install_package_system_dependencies(package_name, tag, platform, local_clone_dir_single, deps_verbose)
|
||||
},
|
||||
# NB: here we need to use conditionMessage() to extract the actual error - as opposed to using $stderr for errors within the tryCatch used in the future* calls
|
||||
error = function(e) {
|
||||
cli::cli_alert_warning("Error in installing dependencies for package {.pkg {package_name[1]}} with tag {.field {tag[1]}}: {e}")
|
||||
store_build_metadata(package_name[1], tag[1], platform, arch = arch, error_occurred = TRUE, force = TRUE, error = conditionMessage(e))
|
||||
return(TRUE)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
if (file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
|
||||
cli::cli_alert("{.fun build_single_tag}: (2/3) Tarball for package {.pkg {package_name}} with tag {.field {tag}} already exists. Skipping build.")
|
||||
} else {
|
||||
cli::cli_alert("{.fun build_single_tag}: (2/3) Building package {.pkg {package_name}} with tag {.field {tag}}.")
|
||||
|
||||
if (debug) {
|
||||
quiet <- FALSE
|
||||
} else {
|
||||
quiet <- TRUE
|
||||
}
|
||||
t1 <- Sys.time()
|
||||
tryCatch(
|
||||
{
|
||||
if (debug) {
|
||||
message(sprintf("DEBUG1: Printing 'dir_out_bin': %s", dir_out_bin))
|
||||
}
|
||||
pkgbuild::build(
|
||||
path = sprintf("%s", local_clone_dir_single),
|
||||
binary = TRUE, vignettes = FALSE,
|
||||
dest_path = dir_out_bin, quiet = quiet
|
||||
)
|
||||
if (debug) {
|
||||
message(sprintf("DEBUG: Listing dir 'dir_out_bin': %s", dir_out_bin))
|
||||
print(fs::dir_ls(dir_out_bin))
|
||||
}
|
||||
},
|
||||
error = function(e) {
|
||||
cli::cli_alert_warning("Error in starting build command for package {.pkg {package_name}} with tag {.field {tag}}: {e}")
|
||||
local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name, tag)
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
store_build_metadata(package_name, tag, platform, arch = arch, error_occurred = TRUE, force = TRUE, error = sprintf("Error trying to initiate pkgbuild - likely a non-valid R package structure. Full error: %s", e))
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
)
|
||||
|
||||
if (any(grepl("alpine", system2("cat", args = c("/etc/os-release"), stdout = TRUE)))) {
|
||||
linux_suffix <- "musl"
|
||||
} else {
|
||||
linux_suffix <- "gnu"
|
||||
}
|
||||
|
||||
# set tarball id for arch
|
||||
local_arch <- Sys.info()[["machine"]]
|
||||
if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
|
||||
tarball_id <- "unknown"
|
||||
tarball_arch <- "aarch64"
|
||||
} else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
|
||||
tarball_id <- "pc"
|
||||
tarball_arch <- "x86_64"
|
||||
}
|
||||
|
||||
if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
|
||||
if (debug) {
|
||||
cli::cli_alert_info('{.fun build_single_tag}: DEBUG: Moving package from {.path {sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, tarball_arch, tarball_id, linux_suffix)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
|
||||
}
|
||||
# double-check that file exists (some packages like https://github.com/cran/BACCO/tree/1.0-14 don't include R/ and hence don't procude a valid binary)
|
||||
if (fs::file_exists(sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, tarball_arch, tarball_id, linux_suffix))) {
|
||||
# remove _aarch64-unknown-linux-gnu/musl part in filename
|
||||
fs::file_move(
|
||||
sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, tarball_arch, tarball_id, linux_suffix),
|
||||
sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)
|
||||
)
|
||||
} else {
|
||||
cli::cli_alert_info('{.fun build_single_tag}: File for package {.pkg {package_name}} {.field {tag}} at {.path {sprintf("%s/%s_%s_R_%s-%s-linux-%s.tar.gz", dir_out_bin, package_name, tag, tarball_arch, tarball_id, linux_suffix)}} does not exist - skipping.')
|
||||
if (debug) {
|
||||
message(sprintf("DEBUG: Listing dir 'dir_out_bin': %s", dir_out_bin))
|
||||
message(fs::dir_ls(dir_out_bin))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.')
|
||||
}
|
||||
unlink(sprintf("%s/%s_%s_R*.tar.gz", dir_out_bin, package_name, tag))
|
||||
|
||||
cli::cli_alert("{.fun build_single_tag}: (3/3) Removing {.path {local_clone_dir_single}}.")
|
||||
unlink(local_clone_dir_single, force = TRUE, recursive = TRUE)
|
||||
|
||||
total_build_time <- round(as.numeric(difftime(Sys.time(), t1, units = "secs")), 2)
|
||||
|
||||
# bytes to MB in binary format
|
||||
file_size <- round(as.numeric(fs::file_size(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) / (1024^2), 2)
|
||||
|
||||
if (debug) {
|
||||
cli::cli_alert_warning("DEBUG: total_build_time: {total_build_time}")
|
||||
cli::cli_alert_warning("DEBUG: file_size: {file_size}")
|
||||
}
|
||||
|
||||
tarball_name <- sprintf("%s_%s.tar.gz", package_name, tag)
|
||||
if (fs::file_exists(sprintf("%s/%s", dir_out_bin, tarball_name))) {
|
||||
store_build_metadata(package_name, tag, platform, arch = arch, error_occurred = FALSE, force = force, build_duration = total_build_time, size = file_size)
|
||||
}
|
||||
}
|
||||
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
11
R/db.R
11
R/db.R
|
|
@ -1,11 +0,0 @@
|
|||
#' @importFrom DBI dbConnect
|
||||
#' @importFrom RPostgres Postgres
|
||||
#' @export
|
||||
dbcon <- function() {
|
||||
DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS")
|
||||
)
|
||||
}
|
||||
|
||||
dbcon_mem <- memoise::memoise(dbcon)
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
#' Get updated CRAN packages
|
||||
#' @importFrom magrittr %>%
|
||||
#' @importFrom purrr map list_rbind
|
||||
#' @importFrom lubridate today
|
||||
#' @examples
|
||||
#' get_updated_cran_packages(lubridate::interval(today() - 7, today()))
|
||||
#' @export
|
||||
get_updated_cran_packages <- function(date = lubridate::today()) {
|
||||
feed <- get_cranberries_feed()
|
||||
|
||||
# Apply the function to each item and bind the results into a data frame
|
||||
results <- purrr::map(feed, ~ process_cranberries_rss(.x, date)) %>%
|
||||
purrr::list_rbind()
|
||||
|
||||
return(results)
|
||||
}
|
||||
|
||||
#' Get new CRAN packages
|
||||
#' @importFrom magrittr %>%
|
||||
#' @importFrom purrr map list_rbind
|
||||
#' @importFrom lubridate today
|
||||
#' @export
|
||||
#' @examples
|
||||
#' # last week
|
||||
#' get_new_cran_packages(lubridate::interval(today() - 7, today()))
|
||||
#'
|
||||
get_new_cran_packages <- function(date = lubridate::today()) {
|
||||
feed <- get_cranberries_feed(type = "new")
|
||||
|
||||
# Apply the function to each item and bind the results into a data frame
|
||||
results <- purrr::map(feed, ~ process_cranberries_rss(.x, date)) %>%
|
||||
purrr::list_rbind()
|
||||
|
||||
return(results)
|
||||
}
|
||||
|
||||
#' Get removed CRAN packages
|
||||
#' @importFrom magrittr %>%
|
||||
#' @importFrom purrr map list_rbind
|
||||
#' @importFrom lubridate today
|
||||
#' @export
|
||||
#' @examples
|
||||
#' get_removed_cran_packages()
|
||||
#' get_removed_cran_packages(lubridate::interval(today() - 7, today()))
|
||||
#'
|
||||
get_removed_cran_packages <- function(date = lubridate::today()) {
|
||||
feed <- get_cranberries_feed(type = "removed")
|
||||
|
||||
# Apply the function to each item and bind the results into a data frame
|
||||
results <- purrr::map(feed, ~ process_cranberries_rss(.x, date)) %>%
|
||||
purrr::list_rbind()
|
||||
|
||||
if (nrow(results == 0)) {
|
||||
cli::cli_alert_info("{.fun archive_package}: No packages to archive for interval {.field {date}}.")
|
||||
}
|
||||
|
||||
return(results)
|
||||
}
|
||||
|
||||
#' Get Cranberries fieed
|
||||
#' @importFrom xml2 read_xml xml_find_all
|
||||
#' @export
|
||||
#' @param type Which type of packages to query. Allowed are `"updated"`, `"new"` and `"removed"`
|
||||
get_cranberries_feed <- function(type = "updated") {
|
||||
feed <- sprintf("https://dirk.eddelbuettel.com/cranberries/cran/%s/index.rss", type)
|
||||
|
||||
# Fetch and parse the RSS feed
|
||||
rss_content <- xml2::read_xml(feed)
|
||||
|
||||
# Extract item nodes
|
||||
items <- xml2::xml_find_all(rss_content, "//item")
|
||||
|
||||
return(items)
|
||||
}
|
||||
|
||||
#' @importFrom lubridate dmy_hms as_date interval %within%
|
||||
#' @importFrom xml2 xml_text xml_find_first
|
||||
#' @export
|
||||
process_cranberries_rss <- function(feed, date = lubridate::today()) {
|
||||
title <- xml2::xml_text(xml_find_first(feed, "title"))
|
||||
pub_date_text <- xml2::xml_text(xml_find_first(feed, "pubDate"))
|
||||
pub_date <- lubridate::as_date(dmy_hms(pub_date_text))
|
||||
if (class(date) == "Date") {
|
||||
date <- lubridate::interval(date, date)
|
||||
}
|
||||
|
||||
if (pub_date %within% date) {
|
||||
if (grepl("updated", title)) {
|
||||
package_info <- strsplit(title, " ")[[1]]
|
||||
package_name <- package_info[2]
|
||||
package_version_new <- package_info[7]
|
||||
package_version_old <- package_info[12]
|
||||
previous_update_date <- package_info[14]
|
||||
|
||||
return(data.frame(
|
||||
"name" = package_name,
|
||||
"version" = package_version_new,
|
||||
"date" = pub_date,
|
||||
"version_old" = package_version_old,
|
||||
"previous_update_date" = previous_update_date
|
||||
))
|
||||
} else if (grepl("New package", title)) {
|
||||
package_info <- strsplit(title, " ")[[1]]
|
||||
package_name <- package_info[3]
|
||||
package_version_new <- package_info[7]
|
||||
|
||||
return(data.frame(
|
||||
"name" = package_name,
|
||||
"version" = package_version_new,
|
||||
"date" = pub_date
|
||||
))
|
||||
} else if (grepl("was removed", title)) {
|
||||
package_info <- strsplit(title, " ")[[1]]
|
||||
package_name <- package_info[2]
|
||||
|
||||
return(data.frame(
|
||||
"name" = package_name,
|
||||
"date" = pub_date
|
||||
))
|
||||
}
|
||||
} else {
|
||||
return(NULL)
|
||||
}
|
||||
}
|
||||
84
R/helpers.R
84
R/helpers.R
|
|
@ -1,84 +0,0 @@
|
|||
#' @export
|
||||
set_codename <- function(codename) {
|
||||
if (is.null(codename)) {
|
||||
if (Sys.info()["sysname"] == "Linux") {
|
||||
if (any(grepl("alpine", system2("cat", args = c("/etc/os-release"), stdout = TRUE)))) {
|
||||
version <- system2("grep",
|
||||
args = c("'^VERSION_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '\"'"), stdout = TRUE
|
||||
)
|
||||
version_stripped <- substr(gsub("\\.", "", version), 1, 3)
|
||||
codename <- paste0("alpine", version_stripped)
|
||||
} else {
|
||||
dist_fam <- system2("grep",
|
||||
args = c("'^ID_LIKE=' /etc/os-release | cut -d'=' -f2 | tr -d '\"'"), stdout = TRUE
|
||||
)
|
||||
if (dist_fam == "debian") {
|
||||
codename <- system2("grep",
|
||||
args = c("'^VERSION_CODENAME=' /etc/os-release | cut -d'=' -f2 | tr -d '\"'"), stdout = TRUE
|
||||
)
|
||||
} else if (grepl("rhel|fedora", dist_fam)) {
|
||||
platform_id <- system2("grep",
|
||||
args = c("'^PLATFORM_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '\"'"), stdout = TRUE
|
||||
)
|
||||
if (platform_id == "platform:el9") {
|
||||
codename <- "rhel9"
|
||||
} else if (platform_id == "platform:el8") {
|
||||
codename <- "rhel8"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return(codename)
|
||||
} else {
|
||||
return(codename)
|
||||
}
|
||||
}
|
||||
|
||||
#' Helper function to set the path for binary package outputs
|
||||
#' @export
|
||||
set_bin_path <- function(
|
||||
r_minor_version, build_for_minor,
|
||||
local_build_root, codename) {
|
||||
|
||||
local_arch = Sys.info()[["machine"]]
|
||||
if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
|
||||
arch <- "arm64"
|
||||
} else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
|
||||
arch <- "amd64"
|
||||
}
|
||||
|
||||
if (!build_for_minor) {
|
||||
path <- sprintf(
|
||||
"%s/%s/%s/latest/src/contrib",
|
||||
local_build_root, arch, codename
|
||||
)
|
||||
} else {
|
||||
path <- sprintf(
|
||||
"%s/%s/%s/%s/latest/src/contrib",
|
||||
local_build_root, arch, codename, r_minor_version
|
||||
)
|
||||
}
|
||||
return(path)
|
||||
}
|
||||
|
||||
add_column_if_not_exists <- function(db_name, table_name, column_name, column_type) {
|
||||
# Connect to the database
|
||||
conn <- dbConnect(SQLite(), dbname = db_name)
|
||||
|
||||
# Get the list of columns in the table
|
||||
columns <- dbGetQuery(conn, paste0("PRAGMA table_info(", table_name, ");"))
|
||||
|
||||
# Check if the column already exists
|
||||
column_exists <- any(columns$name == column_name)
|
||||
|
||||
# Add the column if it doesn't exist
|
||||
if (!column_exists) {
|
||||
dbExecute(conn, paste0("ALTER TABLE ", table_name, " ADD COLUMN ", column_name, " ", column_type, ";"))
|
||||
message(paste("Column '", column_name, "' added to table '", table_name, "'.", sep = ""))
|
||||
} else {
|
||||
message(paste("Column '", column_name, "' already exists in table '", table_name, "'.", sep = ""))
|
||||
}
|
||||
|
||||
# Disconnect from the database
|
||||
dbDisconnect(conn)
|
||||
}
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
#' @export
|
||||
install_package_system_dependencies <- function(package_name,
|
||||
tag,
|
||||
platform = platform,
|
||||
local_clone_dir,
|
||||
deps_verbose = FALSE) {
|
||||
cli::cli_alert("{.fun install_package_system_dependencies}: Cloning package {.pkg {package_name[1]}} with tag {.field {tail(tag, 1)}}.")
|
||||
|
||||
t1 <- Sys.time()
|
||||
|
||||
local_clone_dir_single <- sprintf("%s/%s_%s", local_clone_dir, package_name[1], tail(tag, 1))
|
||||
|
||||
if (!dir.exists(local_clone_dir_single)) {
|
||||
system2("git", args = c(
|
||||
"clone", "-q", sprintf("--branch=%s", tail(tag, 1)),
|
||||
sprintf("https://github.com/cran/%s", package_name[1]), local_clone_dir_single
|
||||
))
|
||||
}
|
||||
|
||||
if (grepl("alpine", platform)) {
|
||||
platform <- "alpine"
|
||||
}
|
||||
|
||||
cli::cli_alert("{.fun install_package_system_dependencies}: Installing R package dependencies for {.pkg {package_name[1]}} with tag {.field {tail(tag, 1)}}.")
|
||||
# Sys.setenv(PKG_SYSREQS_PLATFORM = platform)
|
||||
Sys.setenv(PKG_SYSREQS = TRUE)
|
||||
Sys.setenv(PKG_SYSREQS_VERBOSE = TRUE)
|
||||
# FIXME: https://github.com/r-lib/pak/issues/610
|
||||
if (grepl("rhel-9", pak::system_r_platform())) {
|
||||
Sys.setenv(PKG_SYSREQS_PLATFORM = "redhat-9")
|
||||
# otherwise pak::pkg_history() fails with 'error setting certificate verify locations'
|
||||
Sys.setenv(CURL_CA_BUNDLE = "/etc/pki/tls/certs/ca-bundle.crt")
|
||||
} else if (grepl("rhel-8", pak::system_r_platform())) {
|
||||
Sys.setenv(PKG_SYSREQS_PLATFORM = "redhat-8")
|
||||
# otherwise pak::pkg_history() fails with 'error setting certificate verify locations'
|
||||
Sys.setenv(CURL_CA_BUNDLE = "/etc/pki/tls/certs/ca-bundle.crt")
|
||||
}
|
||||
if (deps_verbose) {
|
||||
pak::local_install_deps(sprintf("%s", local_clone_dir_single))
|
||||
} else {
|
||||
suppressMessages(pak::local_install_deps(sprintf("%s", local_clone_dir_single)))
|
||||
}
|
||||
|
||||
cli::cli_alert("{.fun install_package_system_dependencies}: Removing temporary clone dir at {.path {local_clone_dir_single}}.")
|
||||
|
||||
total_build_time <- round(Sys.time() - t1, 2)
|
||||
cli::cli_alert("Dependency installation time ({.pkg {package_name[[1]]}}): {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.")
|
||||
|
||||
unlink(sprintf("%s", local_clone_dir_single), recursive = TRUE, force = TRUE)
|
||||
}
|
||||
|
|
@ -1,117 +0,0 @@
|
|||
#' Stores build metadata of single binary builds in a Postgres DB
|
||||
#' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute
|
||||
store_build_metadata <- function(
|
||||
package_name, tag, platform, error_occurred, arch,
|
||||
force = FALSE, error = NA, build_duration = NA, size = NA) {
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
# Check if an entry with the same package_name and tag already exists
|
||||
existing_entries <- dbGetQuery(con, paste0(
|
||||
"SELECT * FROM single_builds WHERE name = '",
|
||||
package_name, "' AND tag = '", tag, "' AND platform = '", arch, "' AND arch = '", platform, "'"
|
||||
))
|
||||
|
||||
if (nrow(existing_entries) >= 1 && !force) {
|
||||
cli::cli_alert("{.fun store_build_metadata}: Build metadata for {.field {.pkg {package_name}}} {.field {tag}} already exists.")
|
||||
} else if (nrow(existing_entries) >= 1 && force) {
|
||||
cli::cli_alert_info("{.fun store_build_metadata}: Force overwriting build metadata for {.pkg {package_name}} {.field {tag}} ({platform}) because {.code force = TRUE} was set.")
|
||||
|
||||
DBI::dbExecute(con, "UPDATE single_builds SET timestamp = $1, error_occurred = $2, error_text = $3, duration = $4, size = $5, removed = $9 WHERE name = $6 and tag = $7 and platform = $8",
|
||||
params = list(format(Sys.time(), "%Y-%m-%d %H:%M:%S"), error_occurred, error, build_duration, size, package_name, tag, platform, FALSE)
|
||||
)
|
||||
} else if (nrow(existing_entries) == 0) {
|
||||
cli::cli_alert("{.fun store_build_metadata}: Storing build metadata for {.pkg {package_name}} {.field {tag}}.")
|
||||
# if no entry exists already, we can insert the info via dbWriteTable by passing a DF
|
||||
metadata <- data.frame(
|
||||
name = package_name,
|
||||
tag = tag,
|
||||
platform = platform,
|
||||
arch = arch,
|
||||
error_occurred = error_occurred,
|
||||
error_text = error,
|
||||
size = size,
|
||||
timestamp = format(Sys.time(), "%Y-%m-%d %H:%M:%S"),
|
||||
duration = build_duration,
|
||||
removed = FALSE
|
||||
)
|
||||
# Write the data frame to the SQLite database
|
||||
dbWriteTable(con, "single_builds", metadata, append = TRUE)
|
||||
}
|
||||
dbDisconnect(con)
|
||||
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
||||
remove_from_metadata <- function(package) {
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
|
||||
DBI::dbExecute(con, "UPDATE single_builds SET removed = true WHERE name = $1",
|
||||
params = list(package)
|
||||
)
|
||||
}
|
||||
|
||||
#' @importFrom DBI dbConnect dbDisconnect dbWriteTable dbGetQuery dbExecute dbListTables
|
||||
#' @importFrom dplyr group_by summarise filter n
|
||||
build_metadata_summary <- function(package_name, platform) {
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS")
|
||||
)
|
||||
|
||||
metadata <- dbGetQuery(con, sprintf(
|
||||
"SELECT * FROM single_builds WHERE name = '%s' AND platform = '%s'", package_name, platform
|
||||
))
|
||||
|
||||
cli::cli_alert("{.fun build_binary_package}: Building/updating metadata summary for {.pkg {package_name[[1]]}}.")
|
||||
|
||||
# add average_build_time_per_tag to metadata summary table
|
||||
|
||||
# If the entry does not exist or is not valid, entry will be NULL or NA
|
||||
# entry_exists <- !is.null(entry$average_build_time_per_tag) && !is.na(entry$average_build_time_per_tag)
|
||||
|
||||
# If the entry does not exist, update the average_build_time_per_tag
|
||||
# if (!entry_exists) {
|
||||
# dbExecute(con, paste0(
|
||||
# "UPDATE metadata_summary SET average_build_time_per_tag = ", average_build_time_per_tag,
|
||||
# " WHERE package_name = '", package_name[[1]], "' AND platform = '", platform, "'"
|
||||
# ))
|
||||
# }
|
||||
|
||||
# Calculate the summary
|
||||
summary <- metadata %>%
|
||||
group_by(package_name, platform) %>%
|
||||
summarise(
|
||||
successful_builds = sum(!error_occurred),
|
||||
unsuccessful_builds = sum(error_occurred),
|
||||
total_builds = n(),
|
||||
percentage_successful_builds = round(successful_builds / total_builds * 100, 2),
|
||||
average_build_time_per_tag = round(sum(as.numeric(build_duration)) / total_builds, 2)
|
||||
)
|
||||
|
||||
# Check if the metadata_summary table exists
|
||||
if ("metadata_summary" %in% dbListTables(con)) {
|
||||
dbExecute(con, paste0(
|
||||
"UPDATE metadata_summary SET successful_builds = ", summary$successful_builds,
|
||||
", unsuccessful_builds = ", summary$unsuccessful_builds,
|
||||
", total_builds = ", summary$total_builds,
|
||||
", percentage_successful_builds = ", summary$percentage_successful_builds,
|
||||
", average_build_time_per_tag = '", summary$average_build_time_per_tag,
|
||||
"', package_name = '", package_name,
|
||||
"', platform = '", platform, "'",
|
||||
"', arch = '", platform, "'",
|
||||
))
|
||||
} else {
|
||||
dbWriteTable(con, "metadata_summary", summary)
|
||||
}
|
||||
|
||||
# Close the SQLite connection
|
||||
dbDisconnect(con)
|
||||
}
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
#' Query a specific table in the Postgres database containing the metadata
|
||||
#' @param table Table name. See [list_metadata_tables()] for valid tables.
|
||||
#' @importFrom DBI dbConnect dbGetQuery
|
||||
#' @importFrom RPostgres Postgres
|
||||
#' @importFrom dplyr tbl
|
||||
#' @export
|
||||
query_metadata_table <- function(table = "single_builds") {
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
metadata <- tbl(con, table)
|
||||
return(metadata)
|
||||
}
|
||||
|
||||
#' List existing database tables
|
||||
#' @export
|
||||
list_metadata_tables <- function() {
|
||||
con <- DBI::dbConnect(RPostgres::Postgres(),
|
||||
dbname = "build_metadata", host = "r-binaries.devxy.io",
|
||||
port = 15432, user = "r_binaries", password = Sys.getenv("PGPASS"),
|
||||
sslmode = "require"
|
||||
)
|
||||
dbListTables(con)
|
||||
}
|
||||
|
||||
query_metadata_summary <- function(package_name = NULL, platform = NULL) {
|
||||
# s3 <- paws.storage::s3(config = list(
|
||||
# endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
# region = "eu-central-003"
|
||||
# ))
|
||||
# db_local <- s3$get_object(Bucket = "devxy-arm64-r-binaries-db", Key = "metadata.sqlite")
|
||||
# writeBin(db_local$Body, "/tmp/metadata.sqlite")
|
||||
|
||||
# Create a new SQLite connection
|
||||
con <- dbcon_mem()
|
||||
|
||||
# If both package_name and platform are NULL, return the full table
|
||||
if (is.null(package_name) && is.null(platform)) {
|
||||
summary <- dbGetQuery(con, "SELECT * FROM metadata_summary")
|
||||
} else {
|
||||
# Query the metadata_summary table for the specified package and platform
|
||||
summary <- dbGetQuery(con, paste0(
|
||||
"SELECT * FROM metadata_summary WHERE package_name = '",
|
||||
package_name, "' AND platform = '", platform, "'"
|
||||
))
|
||||
}
|
||||
|
||||
# Close the SQLite connection
|
||||
dbDisconnect(con)
|
||||
|
||||
return(summary)
|
||||
}
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
#' @importFrom cranlike add_PACKAGES
|
||||
#' @importFrom s3fs s3_dir_ls s3_file_system
|
||||
#' @export
|
||||
add_to_package_index <- function(
|
||||
package_name = NULL,
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries",
|
||||
local_build_root = "/mnt/cache/binaries",
|
||||
codename = NULL, r_minor_version = NULL,
|
||||
build_for_minor = FALSE,
|
||||
debug = FALSE) {
|
||||
codename <- set_codename(codename)
|
||||
if (is.null(r_minor_version)) {
|
||||
r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
|
||||
}
|
||||
|
||||
local_arch <- Sys.info()[["machine"]]
|
||||
if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
|
||||
arch <- "arm64"
|
||||
} else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
|
||||
arch <- "amd64"
|
||||
}
|
||||
|
||||
if (!build_for_minor) {
|
||||
local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename)
|
||||
} else {
|
||||
dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version)
|
||||
}
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
endpoint = endpoint,
|
||||
region_name = region,
|
||||
)
|
||||
|
||||
# get latest PACKAGES file from S3
|
||||
if (!s3fs::s3_file_exists(sprintf("%s/PACKAGES", remote_bin_dir))) {
|
||||
s3fs::s3_file_download(sprintf("%s/PACKAGES", remote_bin_dir), sprintf("%s/PACKAGES", local_bin_dir))
|
||||
}
|
||||
file_names <- list.files(local_bin_dir, pattern = sprintf("%s*", package_name))
|
||||
|
||||
# list all tarballs for the given package
|
||||
cranlike::add_PACKAGES(file_names, local_bin_dir)
|
||||
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
||||
#' @importFrom s3fs s3_file_upload s3_dir_ls
|
||||
#' @importFrom cranlike update_PACKAGES
|
||||
#' @export
|
||||
upload_package_index <- function(
|
||||
package_name = NULL,
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries",
|
||||
local_build_root = ".",
|
||||
codename = NULL, r_minor_version = NULL,
|
||||
build_for_minor = FALSE,
|
||||
debug = FALSE) {
|
||||
cli::cli_alert("{.fun upload_package_index}: Updating PACKAGES* files in S3.")
|
||||
|
||||
codename <- set_codename(codename)
|
||||
if (is.null(r_minor_version)) {
|
||||
r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
|
||||
}
|
||||
|
||||
local_arch <- Sys.info()[["machine"]]
|
||||
if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
|
||||
arch <- "arm64"
|
||||
} else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
|
||||
arch <- "amd64"
|
||||
}
|
||||
|
||||
if (!build_for_minor) {
|
||||
local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename)
|
||||
} else {
|
||||
dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version)
|
||||
}
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
endpoint = endpoint,
|
||||
region_name = region,
|
||||
)
|
||||
|
||||
cli::cli_alert("{.fun upload_package_index}: Started listing remote packages")
|
||||
pkgs <- s3fs::s3_dir_ls(remote_bin_dir)
|
||||
cli::cli_alert_success("{.fun upload_package_index}: Finished listing remote packages")
|
||||
# We remove 4 from the count as we don't want to count the PACKAGES* files + Archive/ dir
|
||||
pkg_count <- length(pkgs) - 5
|
||||
unique_pkgs <- length(unique(sapply(strsplit(basename(pkgs), "_"), function(x) x[1]))) - 5
|
||||
|
||||
t1 <- Sys.time()
|
||||
cranlike::update_PACKAGES(sprintf("s3://%s", remote_bin_dir))
|
||||
total_build_time <- round(Sys.time() - t1, 2)
|
||||
cli::cli_alert("{.fun upload_package_index}: Time updating PACKAGES index for {.field {pkg_count}} ({.field {unique_pkgs}} unique) packages: {.strong {total_build_time} {units(difftime(Sys.time(), t1))}}.")
|
||||
|
||||
purrr::walk2(
|
||||
c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz"),
|
||||
sprintf("%s/%s", remote_bin_dir, c("PACKAGES", "PACKAGES.db", "PACKAGES.rds", "PACKAGES.gz")),
|
||||
\(x, y) s3fs::s3_file_upload(x, y, overwrite = TRUE, CacheControl = "no-cache")
|
||||
)
|
||||
|
||||
return(invisible(TRUE))
|
||||
}
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
#' @title Process updated and new CRAN packages
|
||||
#' @description
|
||||
#' Packages which got removed from CRAN can be deleted by setting `prune = TRUE`.
|
||||
#' Argument `interval` allows to specify a range which should be processed.
|
||||
#'
|
||||
#' @importFrom dplyr bind_rows
|
||||
#' @importFrom purrr walk2
|
||||
#' @examples
|
||||
#' process_cran_updates(
|
||||
#' interval = lubridate::interval(lubridate::today() - 2, lubridate::today() - 20),
|
||||
#' process_updated = FALSE, process_new = FALSE
|
||||
#' )
|
||||
#'
|
||||
#' @export
|
||||
process_cran_updates <- function(
|
||||
package_name,
|
||||
tag,
|
||||
platform,
|
||||
local_clone_dir,
|
||||
interval = lubridate::today(),
|
||||
build_for_minor = FALSE,
|
||||
codename = NULL,
|
||||
r_minor_version = NULL,
|
||||
local_build_root = ".",
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries",
|
||||
process_updated = TRUE,
|
||||
process_new = TRUE,
|
||||
process_removed = TRUE) {
|
||||
# Get list of updated and new packages for a specific day
|
||||
updated_pkgs <- get_updated_cran_packages(interval)
|
||||
# FIXME: temporary hickup for package redatam
|
||||
updated_pkgs <- updated_pkgs[updated_pkgs$name != 'redatam', ]
|
||||
new_pkgs <- get_new_cran_packages(interval)
|
||||
|
||||
all_pkgs <- dplyr::bind_rows(updated_pkgs, new_pkgs)
|
||||
|
||||
if (any(c(process_updated, process_new))) {
|
||||
cli::cli_alert_success("{.fun process_cran_updates}: Updated packages:")
|
||||
print(updated_pkgs)
|
||||
cli::cli_alert_success("{.fun process_cran_updates}: New packages:")
|
||||
print(new_pkgs)
|
||||
purrr::walk2(all_pkgs$name, all_pkgs$version, ~ {
|
||||
build_binary_package(.x, .y, platform = platform)
|
||||
archive_package(.x)
|
||||
})
|
||||
}
|
||||
|
||||
if (process_removed) {
|
||||
removed_pkgs <- get_removed_cran_packages(interval)
|
||||
cli::cli_alert_success("{.fun process_cran_updates}: Removed packages:")
|
||||
print(removed_pkgs)
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
endpoint = endpoint,
|
||||
region_name = region,
|
||||
)
|
||||
|
||||
codename <- set_codename(codename)
|
||||
|
||||
local_arch = Sys.info()[["machine"]]
|
||||
if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
|
||||
arch <- "arm64"
|
||||
} else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
|
||||
arch <- "amd64"
|
||||
}
|
||||
|
||||
if (!build_for_minor) {
|
||||
local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename)
|
||||
} else {
|
||||
dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
|
||||
remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version)
|
||||
}
|
||||
|
||||
files <- s3fs::s3_dir_ls(remote_bin_dir)
|
||||
|
||||
foo <- lapply(removed_pkgs$name, function(.x) {
|
||||
cli::cli_alert("{.fun process_cran_updates}: Removing package {.pkg {.x}} from S3.")
|
||||
files_filtered <- grep(sprintf("/%s_", .x), files, value = TRUE)
|
||||
if (length(files_filtered) > 0) {
|
||||
s3fs::s3_file_delete(files_filtered)
|
||||
cli::cli_alert_success("{.fun process_cran_updates}: Successfully removed {.pkg {basename(files_filtered)}} from S3.")
|
||||
remove_from_metadata(.x)
|
||||
cli::cli_alert_success("{.fun process_cran_updates}: Successfully set {.pkg {basename(files_filtered)}} as 'removed' in metadata table.")
|
||||
} else {
|
||||
cli::cli_alert("{.fun process_cran_updates}: No tarballs found for package {.pkg {.x}} - already removed?")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
72
R/upload.R
72
R/upload.R
|
|
@ -1,72 +0,0 @@
|
|||
#' @importFrom s3fs s3_file_exists s3_file_upload s3_file_system
|
||||
#' @export
|
||||
upload_single_binary_to_s3 <- function(
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries",
|
||||
local_build_root = "/root",
|
||||
codename = NULL,
|
||||
package_name, tag,
|
||||
r_minor_version = NULL, build_for_minor = FALSE,
|
||||
force = FALSE,
|
||||
debug = FALSE) {
|
||||
codename <- set_codename(codename)
|
||||
|
||||
cli::cli_h2("Uploading ({.pkg {package_name[1]}})")
|
||||
|
||||
if (is.null(r_minor_version)) {
|
||||
r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
|
||||
}
|
||||
|
||||
local_bin_path <- set_bin_path(r_minor_version, build_for_minor, local_build_root = local_build_root, codename)
|
||||
remote_bin_path <- set_bin_path(r_minor_version, build_for_minor, local_build_root = bucket, codename)
|
||||
|
||||
tarball_name <- sprintf("%s_%s.tar.gz", package_name, tag)
|
||||
|
||||
if (!file.exists(sprintf("%s/%s", local_bin_path, tarball_name))) {
|
||||
cli::cli_alert("{.fun upload_single_binary_to_s3}: File {.pkg {package_name}} {.field {tag}} does not exist locally - skipping upload.")
|
||||
return(TRUE)
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
cli::cli_alert_warning("DEBUG: local_bin_path: {local_bin_path}")
|
||||
cli::cli_alert_warning("DEBUG: remote_bin_path: {remote_bin_path}")
|
||||
}
|
||||
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
endpoint = endpoint,
|
||||
region_name = region,
|
||||
)
|
||||
|
||||
exists <- s3fs::s3_file_exists(sprintf("%s/%s", remote_bin_path, tarball_name))
|
||||
|
||||
# suppress progressr output here
|
||||
progressr::handlers("void")
|
||||
# don't parallelise
|
||||
future::plan("sequential")
|
||||
|
||||
if ((!exists && !force) || (!exists && force)) {
|
||||
cli::cli_alert("{.fun upload_single_binary_to_s3}: Uploading {.pkg {package_name}} {.field {tag}} to {.path {sprintf('%s/%s', remote_bin_path, tarball_name)}}.")
|
||||
s3fs::s3_file_upload(
|
||||
sprintf("%s/%s", local_bin_path, tarball_name),
|
||||
sprintf("%s/%s", remote_bin_path, tarball_name)
|
||||
)
|
||||
cli::cli_alert_success("Successfully uploaded package {.pkg {package_name}} with tag {.field {tag}}.")
|
||||
cli::cli_alert("{.fun upload_single_binary_to_s3}: Deleting binary for {.pkg {package_name}} {.field {tag}} at path {.path {sprintf('%s/%s', local_bin_path, tarball_name)}}.")
|
||||
file.remove(sprintf("%s/%s", local_bin_path, tarball_name))
|
||||
} else if (exists && force) {
|
||||
cli::cli_alert_info("{.fun upload_single_binary_to_s3}: Force uploading package {.pkg {package_name}} {.field {tag}} to {.path {sprintf('%s/%s', remote_bin_path, tarball_name)}} because {.code force = TRUE} was set.")
|
||||
s3fs::s3_file_upload(
|
||||
sprintf("%s/%s", local_bin_path, tarball_name),
|
||||
sprintf("%s/%s", remote_bin_path, tarball_name),
|
||||
overwrite = TRUE
|
||||
)
|
||||
cli::cli_alert_success("Successfully uploaded package {.pkg {package_name}} with tag {.field {tag}}.")
|
||||
cli::cli_alert("{.fun upload_single_binary_to_s3}: Deleting binary for {.pkg {package_name}} {.field {tag}} at path {.path {sprintf('%s/%s', local_bin_path, tarball_name)}}.")
|
||||
file.remove(sprintf("%s/%s", local_bin_path, tarball_name))
|
||||
} else if (exists && !force) {
|
||||
cli::cli_alert("{.fun upload_single_binary_to_s3}: Package {.pkg {package_name}} {.field {tag}} already exists in S3. Skipping upload.")
|
||||
}
|
||||
}
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
% Generated by roxygen2: do not edit by hand
|
||||
% Please edit documentation in R/archive_packages.R
|
||||
\name{archive_package}
|
||||
\alias{archive_package}
|
||||
\title{Archive packages in CRAN-like repositories}
|
||||
\usage{
|
||||
archive_package(
|
||||
package_name,
|
||||
build_for_minor = FALSE,
|
||||
codename = NULL,
|
||||
r_minor_version = NULL,
|
||||
local_build_root = ".",
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region = "eu-central-003",
|
||||
bucket = "devxy-arm64-r-binaries",
|
||||
pause = NULL,
|
||||
arch = NULL,
|
||||
debug = FALSE
|
||||
)
|
||||
}
|
||||
\description{
|
||||
Archive packages in CRAN-like repositories
|
||||
}
|
||||
\examples{
|
||||
archive_package("AATtools", codename = "rhel9")
|
||||
archive_package("adw", codename = "rhel8", arch = "amd64")
|
||||
|
||||
}
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
% Generated by roxygen2: do not edit by hand
|
||||
% Please edit documentation in R/build_binaries.R
|
||||
\name{build_binary_package}
|
||||
\alias{build_binary_package}
|
||||
\title{Build R binary packages}
|
||||
\usage{
|
||||
build_binary_package(
|
||||
package_name,
|
||||
tag = NULL,
|
||||
codename = NULL,
|
||||
r_minor_version = NULL,
|
||||
build_for_minor = FALSE,
|
||||
local_build_root = "/root",
|
||||
local_clone_dir = "/tmp",
|
||||
platform = NULL,
|
||||
arch = NULL,
|
||||
install_system_dependencies = TRUE,
|
||||
deps_verbose = FALSE,
|
||||
debug = FALSE,
|
||||
force = FALSE,
|
||||
archive = TRUE,
|
||||
future_strategy = "multisession",
|
||||
future_workers = 2
|
||||
)
|
||||
}
|
||||
\description{
|
||||
Build R binary packages
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
% Generated by roxygen2: do not edit by hand
|
||||
% Please edit documentation in R/get_updated_packages.R
|
||||
\name{get_cranberries_feed}
|
||||
\alias{get_cranberries_feed}
|
||||
\title{Get Cranberries fieed}
|
||||
\usage{
|
||||
get_cranberries_feed(type = "updated")
|
||||
}
|
||||
\arguments{
|
||||
\item{type}{Which type of packages to query. Allowed are \code{"updated"}, \code{"new"} and \code{"removed"}}
|
||||
}
|
||||
\description{
|
||||
Get Cranberries fieed
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
% Generated by roxygen2: do not edit by hand
|
||||
% Please edit documentation in R/get_updated_packages.R
|
||||
\name{get_new_cran_packages}
|
||||
\alias{get_new_cran_packages}
|
||||
\title{Get new CRAN packages}
|
||||
\usage{
|
||||
get_new_cran_packages(date = lubridate::today())
|
||||
}
|
||||
\description{
|
||||
Get new CRAN packages
|
||||
}
|
||||
\examples{
|
||||
# last week
|
||||
get_new_cran_packages(lubridate::interval(today() - 7, today()))
|
||||
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
% Generated by roxygen2: do not edit by hand
|
||||
% Please edit documentation in R/get_updated_packages.R
|
||||
\name{get_removed_cran_packages}
|
||||
\alias{get_removed_cran_packages}
|
||||
\title{Get removed CRAN packages}
|
||||
\usage{
|
||||
get_removed_cran_packages(date = lubridate::today())
|
||||
}
|
||||
\description{
|
||||
Get removed CRAN packages
|
||||
}
|
||||
\examples{
|
||||
get_removed_cran_packages()
|
||||
get_removed_cran_packages(lubridate::interval(today() - 7, today()))
|
||||
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue