account for architecture when building and uploading

2024-09-05 13:44:02 +02:00 · 2024-09-05 13:44:02 +02:00 · 533fa440aa
commit 533fa440aa
parent e8e69306a4
6 changed files with 75 additions and 30 deletions
--- a/.woodpecker/build.yaml
+++ b/.woodpecker/build.yaml
@ -17,11 +17,11 @@ matrix:
    # - PLATFORM: ubuntu-2204
    #   BLOCK: 20099:21169
    #   ARCH: arm64
-    - PLATFORM: alpine-320
-      BLOCK: 5881:length(pkgs)
-      ARCH: arm64
-      NCPUS: 1
-      STRATEGY: sequential
+    # - PLATFORM: alpine-320
+    #   BLOCK: 5881:length(pkgs)
+    #   ARCH: arm64
+    #   NCPUS: 1
+    #   STRATEGY: sequential
    - PLATFORM: alpine-320
      BLOCK: 1:length(pkgs)
      ARCH: amd64
@ -57,7 +57,7 @@ steps:
      # more complicated system dependencies which pak cannot resolve
      - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
      # set additional repos: important as otherwise some packages cannot be resolved (e.g. INLA). Also: setting our own binary repos so we can make use of them for dep installation at some point
-      - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR",  "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "${PLATFORM}"))'
+      - R -q -e 'options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"))); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs[${BLOCK}], c("biplotbootGUI", "cncaGUI", "dynBiplotGUI", "ade4TkGUI", "BlockmodelingGUI", "cncaGUI", "EventDetectGUI", "irtGUI", "KappaGUI", "magickGUI", "miniGUI", "multibiplotGUI", "TextMiningGUI", "later", "RclusTool", "Rcmdr", "RcmdrMisc", "RcmdrPlugin.aRnova", "RcmdrPlugin.BiclustGUI", "RcmdrPlugin.BWS1", "RcmdrPlugin.BWS2", "RcmdrPlugin.BWS3", "RcmdrPlugin.cpd", "RcmdrPlugin.DCCV", "RcmdrPlugin.DCE", "RcmdrPlugin.depthTools", "RcmdrPlugin.DoE", "RcmdrPlugin.EACSPIR", "RcmdrPlugin.EBM", "RcmdrPlugin.EcoVirtual", "RcmdrPlugin.Export", "RcmdrPlugin.EZR",  "RcmdrPlugin.FactoMineR", "RcmdrPlugin.GWRM", "RcmdrPlugin.HH", "RcmdrPlugin.KMggplot2", "RcmdrPlugin.MA", "RcmdrPlugin.MPAStats", "RcmdrPlugin.NMBU", "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "sf", "StratigrapheR", "tcltk2", "TVMM", "uHMM", "RInno")); library(bincraftR); future::plan("${STRATEGY}", workers = ${NCPUS}, rscript_startup = quote(options(crayon.enabled = TRUE))); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = TRUE, force = TRUE, platform = "${PLATFORM}"))'
    backend_options:
      kubernetes:
        ### PROD-parallel
--- a/R/archive_packages.R
+++ b/R/archive_packages.R
@ -22,12 +22,20 @@ archive_package <- function(
  )

  codename <- set_codename(codename)
+
+  local_arch = Sys.info()[["machine"]]
+  if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
+    arch <- "arm64" 
+  } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
+    arch <- "amd64"
+  }
+
  if (!build_for_minor) {
-    local_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", local_build_root, codename)
-    remote_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", bucket, codename)
+    local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
+    remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename)
  } else {
-    dir_out_bin <- sprintf("%s/arm64/%s/%s/latest/src/contrib", local_build_root, codename, r_minor_version)
-    remote_bin_dir <- sprintf("%s/arm64/%s/%s/latest/src/contrib", bucket, codename, r_minor_version)
+    dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
+    remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version)
  }

  files <- s3fs::s3_dir_ls(remote_bin_dir)
--- a/R/build_binaries.R
+++ b/R/build_binaries.R
@ -192,19 +192,26 @@ build_single_tag <- function(
      linux_suffix <- "gnu"
    }

+    local_arch <- Sys.info()[["machine"]]
+    if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
+      arch <- "aarch64"
+    } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
+      arch <- "x86_64"
+    }
+
    if (!file.exists(sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag))) {
      if (debug) {
-        cli::cli_alert_info('{.fun build_single_tag}: Moving package from {.path {sprintf("%s/%s_%s_R_aarch64-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, linux_suffix)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
+        cli::cli_alert_info('{.fun build_single_tag}: Moving package from {.path {sprintf("%s/%s_%s_R_%s-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, linux_suffix)}} to {.path {sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)}}')
      }
      # double-check that file exists (some packages like https://github.com/cran/BACCO/tree/1.0-14 don't include R/ and hence don't procude a valid binary)
-      if (fs::file_exists(sprintf("%s/%s_%s_R_aarch64-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, linux_suffix))) {
+      if (fs::file_exists(sprintf("%s/%s_%s_R_%s-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, linux_suffix))) {
        # remove _aarch64-unknown-linux-gnu/musl part in filename
        fs::file_move(
-          sprintf("%s/%s_%s_R_aarch64-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, linux_suffix),
+          sprintf("%s/%s_%s_R_%s-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, linux_suffix),
          sprintf("%s/%s_%s.tar.gz", dir_out_bin, package_name, tag)
        )
      } else {
-        cli::cli_alert_info('{.fun build_single_tag}: File for package {.pkg {package_name}} {.field {tag}} at {.path {sprintf("%s/%s_%s_R_aarch64-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, linux_suffix)}} does not exist - skipping.')
+        cli::cli_alert_info('{.fun build_single_tag}: File for package {.pkg {package_name}} {.field {tag}} at {.path {sprintf("%s/%s_%s_R_%s-unknown-linux-%s.tar.gz", dir_out_bin, package_name, tag, arch, linux_suffix)}} does not exist - skipping.')
      }
    } else {
      cli::cli_alert_warning('{.fun build_single_tag}: Binary {sprintf("%s_%s.tar.gz", package_name, tag)} already exists. Skipping copy.')
--- a/R/helpers.R
+++ b/R/helpers.R
@ -39,15 +39,23 @@ set_codename <- function(codename) {
 set_bin_path <- function(
    r_minor_version, build_for_minor,
    local_build_root, codename) {
+  
+  local_arch = Sys.info()[["machine"]]
+  if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
+    arch <- "arm64" 
+  } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
+    arch <- "amd64"
+  }
+
  if (!build_for_minor) {
    path <- sprintf(
-      "%s/arm64/%s/latest/src/contrib",
-      local_build_root, codename
+      "%s/%s/%s/latest/src/contrib",
+      local_build_root, arch, codename
    )
  } else {
    path <- sprintf(
-      "%s/arm64/%s/%s/latest/src/contrib",
-      local_build_root, codename, r_minor_version
+      "%s/%s/%s/%s/latest/src/contrib",
+      local_build_root, arch, codename, r_minor_version
    )
  }
  return(path)
--- a/R/package_index.R
+++ b/R/package_index.R
@ -15,12 +15,19 @@ add_to_package_index <- function(
    r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
  }

+  local_arch <- Sys.info()[["machine"]]
+  if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
+    arch <- "arm64"
+  } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
+    arch <- "amd64"
+  }
+
  if (!build_for_minor) {
-    local_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib/", local_build_root, codename)
-    remote_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib/", bucket, codename)
+    local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
+    remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename)
  } else {
-    dir_out_bin <- sprintf("%s/arm64/%s/%s/latest/src/contrib/", local_build_root, codename, r_minor_version)
-    remote_bin_dir <- sprintf("%s/arm64/%s/%s/latest/src/contrib/", bucket, codename, r_minor_version)
+    dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
+    remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version)
  }

  s3fs::s3_file_system(
@ -61,12 +68,19 @@ upload_package_index <- function(
    r_minor_version <- sub("R version (\\d+\\.\\d+).*", "\\1", R.Version()$version.string)
  }

+  local_arch <- Sys.info()[["machine"]]
+  if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
+    arch <- "arm64"
+  } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
+    arch <- "amd64"
+  }
+
  if (!build_for_minor) {
-    local_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", local_build_root, codename)
-    remote_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", bucket, codename)
+    local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
+    remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename)
  } else {
-    dir_out_bin <- sprintf("%s/arm64/%s/%s/latest/src/contrib", local_build_root, codename, r_minor_version)
-    remote_bin_dir <- sprintf("%s/arm64/%s/%s/latest/src/contrib", bucket, codename, r_minor_version)
+    dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
+    remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version)
  }

  s3fs::s3_file_system(
--- a/R/process_cran_updates.R
+++ b/R/process_cran_updates.R
@ -58,12 +58,20 @@ process_cran_updates <- function(
    )

    codename <- set_codename(codename)
+
+  local_arch = Sys.info()[["machine"]]
+  if (grepl("arm64", local_arch) || grepl("aarch64", local_arch)) {
+    arch <- "arm64" 
+  } else if (grepl("amd64", local_arch) || grepl("x86_64", local_arch)) {
+    arch <- "amd64"
+  }
+
    if (!build_for_minor) {
-      local_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", local_build_root, codename)
-      remote_bin_dir <- sprintf("%s/arm64/%s/latest/src/contrib", bucket, codename)
+      local_bin_dir <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
+      remote_bin_dir <- sprintf("%s/%s/%s/latest/src/contrib", bucket, arch, codename)
    } else {
-      dir_out_bin <- sprintf("%s/arm64/%s/%s/latest/src/contrib", local_build_root, codename, r_minor_version)
-      remote_bin_dir <- sprintf("%s/arm64/%s/%s/latest/src/contrib", bucket, codename, r_minor_version)
+      dir_out_bin <- set_bin_path(r_minor_version, build_for_minor, local_build_root, codename)
+      remote_bin_dir <- sprintf("%s/%s/%s/%s/latest/src/contrib", bucket, arch, codename, r_minor_version)
    }

    files <- s3fs::s3_dir_ls(remote_bin_dir)