diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index c55a714..605f46c 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -17,11 +17,11 @@ matrix: # - PLATFORM: ubuntu-2204 # BLOCK: 20099:21169 # ARCH: arm64 - - PLATFORM: alpine-320 - BLOCK: 5881:length(pkgs) - ARCH: arm64 - NCPUS: 1 - STRATEGY: sequential + # - PLATFORM: alpine-320 + # BLOCK: 5881:length(pkgs) + # ARCH: arm64 + # NCPUS: 1 + # STRATEGY: sequential - PLATFORM: alpine-320 BLOCK: 1:length(pkgs) ARCH: amd64 @@ -57,7 +57,7 @@ steps: # more complicated system dependencies which pak cannot resolve - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point - - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))' + - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR", "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = TRUE, force = TRUE, platform = "${PLATFORM}"))' backend_options: kubernetes: ### PROD-parallel diff --git a/R/archive_packages.R b/R/archive_packages.R index fcbb529..d159923 100644 --- a/R/archive_packages.R +++ b/R/archive_packages.R @@ -22,12 +22,20 @@ archive_package <- function( ) codename <- set_codename(codename) + + local_arch = Sys.info()[["machine"]] + if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) { + arch <- "arm64" + } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) { + arch <- "amd64" + } + if (!build_for_minor) { - local_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", local_build_root, codename) - remote_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", bucket, codename) + local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename) + remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename) } else { - dir_out_bin <- sprintf("%s/arm64/%s/%s/latest/src/contrib", local_build_root, codename, r_minor_version) - remote_bin_dir <- sprintf("%s/arm64/%s/%s/latest/src/contrib", bucket, codename, r_minor_version) + dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename) + remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version) } files <- s3fs::s3_dir_ls(remote_bin_dir) diff --git a/R/build_binaries.R b/R/build_binaries.R index a61406f..496eda3 100644 --- a/R/build_binaries.R +++ b/R/build_binaries.R @@ -192,19 +192,26 @@ build_single_tag <- function( linux_suffix <- "gnu" } + local_arch <- Sys.info()[["machine"]] + if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) { + arch <- "aarch64" + } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) { + arch <- "x86_64" + } + if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) { if (debug) { - cli::cli_alert_info('{.fun build_single_tag}: Moving package from {.path {sprintf("%s/%s_%s_R_aarch64-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, linux_suffix)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}') + cli::cli_alert_info('{.fun build_single_tag}: Moving package from {.path {sprintf("%s/%s_%s_R_%s-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, linux_suffix)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}') } # double-check that file exists (some packages like https://github.com/cran/BACCO/tree/1.0-14 don't include R/ and hence don't procude a valid binary) - if (fs::file_exists(sprintf("%s/%s_%s_R_aarch64-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, linux_suffix))) { + if (fs::file_exists(sprintf("%s/%s_%s_R_%s-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, linux_suffix))) { # remove _aarch64-unknown-linux-gnu/musl part in filename fs::file_move( - sprintf("%s/%s_%s_R_aarch64-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, linux_suffix), + sprintf("%s/%s_%s_R_%s-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, linux_suffix), sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag) ) } else { - cli::cli_alert_info('{.fun build_single_tag}: File for package {.pkg {package_name}} {.field {tag}} at {.path {sprintf("%s/%s_%s_R_aarch64-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, linux_suffix)}} does not exist - skipping.') + cli::cli_alert_info('{.fun build_single_tag}: File for package {.pkg {package_name}} {.field {tag}} at {.path {sprintf("%s/%s_%s_R_%s-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, linux_suffix)}} does not exist - skipping.') } } else { cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.') diff --git a/R/helpers.R b/R/helpers.R index 97a4464..049c77d 100644 --- a/R/helpers.R +++ b/R/helpers.R @@ -39,15 +39,23 @@ set_codename <- function(codename) { set_bin_path <- function( r_minor_version, build_for_minor, local_build_root, codename) { + + local_arch = Sys.info()[["machine"]] + if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) { + arch <- "arm64" + } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) { + arch <- "amd64" + } + if (!build_for_minor) { path <- sprintf( - "%s/arm64/%s/latest/src/contrib", - local_build_root, codename + "%s/%s/%s/latest/src/contrib", + local_build_root, arch, codename ) } else { path <- sprintf( - "%s/arm64/%s/%s/latest/src/contrib", - local_build_root, codename, r_minor_version + "%s/%s/%s/%s/latest/src/contrib", + local_build_root, arch, codename, r_minor_version ) } return(path) diff --git a/R/package_index.R b/R/package_index.R index 8e9b287..246b70e 100644 --- a/R/package_index.R +++ b/R/package_index.R @@ -15,12 +15,19 @@ add_to_package_index <- function( r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string) } + local_arch <- Sys.info()[["machine"]] + if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) { + arch <- "arm64" + } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) { + arch <- "amd64" + } + if (!build_for_minor) { - local_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib/", local_build_root, codename) - remote_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib/", bucket, codename) + local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename) + remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename) } else { - dir_out_bin <- sprintf("%s/arm64/%s/%s/latest/src/contrib/", local_build_root, codename, r_minor_version) - remote_bin_dir <- sprintf("%s/arm64/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version) + dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename) + remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version) } s3fs::s3_file_system( @@ -61,12 +68,19 @@ upload_package_index <- function( r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string) } + local_arch <- Sys.info()[["machine"]] + if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) { + arch <- "arm64" + } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) { + arch <- "amd64" + } + if (!build_for_minor) { - local_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", local_build_root, codename) - remote_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", bucket, codename) + local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename) + remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename) } else { - dir_out_bin <- sprintf("%s/arm64/%s/%s/latest/src/contrib", local_build_root, codename, r_minor_version) - remote_bin_dir <- sprintf("%s/arm64/%s/%s/latest/src/contrib", bucket, codename, r_minor_version) + dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename) + remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version) } s3fs::s3_file_system( diff --git a/R/process_cran_updates.R b/R/process_cran_updates.R index 8727470..bbf831e 100644 --- a/R/process_cran_updates.R +++ b/R/process_cran_updates.R @@ -58,12 +58,20 @@ process_cran_updates <- function( ) codename <- set_codename(codename) + + local_arch = Sys.info()[["machine"]] + if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) { + arch <- "arm64" + } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) { + arch <- "amd64" + } + if (!build_for_minor) { - local_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", local_build_root, codename) - remote_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", bucket, codename) + local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename) + remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename) } else { - dir_out_bin <- sprintf("%s/arm64/%s/%s/latest/src/contrib", local_build_root, codename, r_minor_version) - remote_bin_dir <- sprintf("%s/arm64/%s/%s/latest/src/contrib", bucket, codename, r_minor_version) + dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename) + remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version) } files <- s3fs::s3_dir_ls(remote_bin_dir)