new build approach

.crow/alpine-322-arm64-install-deps.yaml

@@ -37,8 +37,8 @@ steps:
       - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
       - mkdir -p /mnt/cache/R-pkgs
       - rm -rf /mnt/cache/R-pkgs/00LOCK-* /mnt/cache/R-pkgs/bincraft
-      - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'
-      - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")'
+      - R -q -e 'install.packages("remotes")'
+      - R -q -e 'remotes::install_git("https://codefloe.com/rpkgs/bincraft.git")'
       - R -q -e 'packageVersion("bincraft")'
     backend_options:
       kubernetes:

.crow/build-alpine-322-amd64-1.yaml

@@ -56,7 +56,7 @@ steps:
       - R -q -e 'packageVersion("bincraft")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), 5, labels = F))[[1]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
       # hanging: 'doBy', 'IDPmisc', 'frailtypack', 'afex'
     backend_options:
       kubernetes:

.crow/build-alpine-322-amd64-2.yaml

@@ -4,7 +4,7 @@ when:
   - event: manual
     evaluate: 'build == "alpine-322-amd64"'
   - event: manual
-    evaluate: 'build == "alpine-322-amd64-1"'
+    evaluate: 'build == "alpine-322-amd64-2"'
   - event: manual
     evaluate: 'build == "all-amd64"'
 
@@ -56,7 +56,7 @@ steps:
       - R -q -e 'packageVersion("bincraft")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), 5, labels = F))[[2]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
     backend_options:
       kubernetes:
         resources:

.crow/build-alpine-322-amd64-3.yaml

@@ -56,7 +56,7 @@ steps:
       - R -q -e 'packageVersion("bincraft")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), 5, labels = F))[[3]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
     backend_options:
       kubernetes:
         resources:

.crow/build-alpine-322-amd64-4.yaml

@@ -56,7 +56,7 @@ steps:
       - R -q -e 'packageVersion("bincraft")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), 5, labels = F))[[4]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
     backend_options:
       kubernetes:
         resources:

.crow/build-alpine-322-amd64-5.yaml

@@ -44,8 +44,6 @@ steps:
       R_LIBS_USER: /mnt/cache/R-pkgs
       CCACHE_DIR: /mnt/cache/ccache
       PLATFORM: alpine-322
-      BLOCK_START: 18001
-      BLOCK_END: 20000
       ARCH: amd64
       NCPUS: 2
       STRATEGY: sequential
@@ -56,7 +54,7 @@ steps:
       - R -q -e 'packageVersion("bincraft")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), 5, labels = F))[[5]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
     backend_options:
       kubernetes:
         resources:

.crow/build-alpine-322-amd64-trigger-1.yaml

@@ -0,0 +1,68 @@
+when:
+  - event: manual
+    evaluate: 'trigger == "true"'
+
+skip_clone: true
+
+steps:
+  - name: 'Build binaries'
+    image: docker.io/devxygmbh/rpkgs-build-env-alpine:3.22-4.5.0
+    pull: true
+    environment:
+      RED_HAT_DEV_PW:
+        from_secret: RED_HAT_DEV_PW
+      HETZNER_S3_ACCESS_KEY_K3S:
+        from_secret: HETZNER_S3_ACCESS_KEY_K3S
+      HETZNER_S3_SECRET_KEY_K3S:
+        from_secret: HETZNER_S3_SECRET_KEY_K3S
+      PGPASS:
+        from_secret: PGPASS
+      GIT_RO_TOKEN:
+        from_secret: GIT_RO_TOKEN
+      GITHUB_PAT:
+        from_secret: GITHUB_PAT
+      # normal env vars
+      GIT_USER: pat-s
+      NTFY_SERVER:
+        from_secret: NTFY_SERVER
+      NTFY_TOPIC:
+        from_secret: NTFY_TOPIC
+      NTFY_AUTH: TRUE
+      NTFY_PASSWORD:
+        from_secret: ntfy_token
+      # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
+      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
+      R_LIBS_USER: /mnt/cache/R-pkgs
+      CCACHE_DIR: /mnt/cache/ccache
+      PLATFORM: alpine-322
+      BLOCK_START: 10001
+      BLOCK_END: 12000
+      ARCH: amd64
+      NCPUS: 2
+      STRATEGY: sequential
+    volumes:
+      - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache
+    commands:
+      - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
+      - R -q -e 'packageVersion("bincraft")'
+      - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
+      # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = c($PACKAGES); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      # hanging: 'doBy', 'IDPmisc', 'frailtypack', 'afex'
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: 5Gi
+            cpu: 1000m
+          limits:
+            memory: 20Gi
+            cpu: 2000m
+        nodeSelector:
+          kubernetes.io/arch: amd64
+          node.kubernetes.io/instance-type: AX42
+        tolerations:
+          - key: 'CI'
+            operator: 'Equal'
+            value: 'true'
+            effect: 'NoSchedule'

.crow/build-alpine-322-arm64-1.yaml

@@ -56,7 +56,7 @@ steps:
       - R -q -e 'packageVersion("bincraft")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), 5, labels = F))[[1]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
     backend_options:
       kubernetes:
         resources:

.crow/build-alpine-322-arm64-2.yaml

@@ -56,7 +56,7 @@ steps:
       - R -q -e 'packageVersion("bincraft")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), 5, labels = F))[[2]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
     backend_options:
       kubernetes:
         resources:

.crow/build-alpine-322-arm64-3.yaml

@@ -59,7 +59,7 @@ steps:
       - R -q -e 'packageVersion("bincraft")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), 5, labels = F))[[3]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
     backend_options:
       kubernetes:
         resources:

.crow/build-alpine-322-arm64-4.yaml

@@ -59,7 +59,7 @@ steps:
       - R -q -e 'packageVersion("bincraft")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), 5, labels = F))[[4]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
     backend_options:
       kubernetes:
         resources:

.crow/build-alpine-322-arm64-5.yaml

@@ -59,7 +59,7 @@ steps:
       - R -q -e 'packageVersion("bincraft")'
       - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
       # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews'
-      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[$BLOCK_START:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
+      - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), 5, labels = F))[[5]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'sf', 'Rfast')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))"
     backend_options:
       kubernetes:
         resources:

local/packages-to-build.R

@@ -0,0 +1,121 @@
+library(bincraft, quietly = TRUE)
+suppressPackageStartupMessages(library(dplyr))
+library(DBI, quietly = TRUE)
+library(purrr, quietly = TRUE)
+library(progressr, quietly = TRUE)
+suppressPackageStartupMessages(library(data.table))
+
+arch = Sys.getenv("ARCH")
+platform = Sys.getenv("PLATFORM")
+
+con <- DBI::dbConnect(
+  RPostgres::Postgres(),
+  dbname = "build_metadata",
+  host = "r-binaries.devxy.io",
+  port = 15432,
+  user = "r_binaries",
+  password = Sys.getenv("PGPASS"),
+  sslmode = "require"
+)
+
+cran_archive = tools::CRAN_archive_db()
+cran_release = tools::CRAN_package_db()
+# Subset cran_archive to only those packages
+cran_archive_in_release <- cran_archive[names(cran_archive) %in% cran_release$Package]
+
+archive_versions <- rbindlist(
+  lapply(names(cran_archive), function(pkg) {
+    df <- as.data.table(cran_archive_in_release[[pkg]])
+    file_names <- rownames(cran_archive_in_release[[pkg]]) # Get row names from the original data.frame!
+    versions <- sub(".*_(.*)\\.tar\\.gz$", "\\1", file_names)
+    data.table(
+      Package = pkg,
+      Version = versions,
+      mtime = df$mtime
+    )
+  }),
+  fill = TRUE
+)
+
+# Select the 9 most recent by mtime for each package
+archive_versions <- archive_versions[
+  order(Package, -as.numeric(mtime))
+][,
+  head(.SD, 9),
+  by = Package
+][,
+  .(Package, Version)
+]
+
+# Now get release versions (assuming cran_release has Package and Version columns)
+release_versions <- data.table(
+  Package = cran_release$Package,
+  Version = as.character(cran_release$Version)
+)
+
+pkgs_to_build <- unique(rbind(archive_versions, release_versions, fill = TRUE))
+setorder(pkgs_to_build, Package, Version)
+
+### Get all packages in S3
+s3fs::s3_file_system(
+  aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"),
+  aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"),
+  endpoint = "https://hel1.your-objectstorage.com",
+  region_name = "hel1",
+  refresh = TRUE
+)
+s3_pkgs = s3fs::s3_dir_ls(
+  sprintf("devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", arch, sub("-", "", platform)),
+  recurse = TRUE
+)
+
+file_names <- basename(s3_pkgs)
+matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names)
+parts <- regmatches(file_names, matches)
+parts <- parts[sapply(parts, length) == 3]
+s3_dt <- data.table(
+  Package = sapply(parts, `[`, 2),
+  Version = sapply(parts, `[`, 3)
+)
+
+### Get all packages with build errors
+
+sql_query <- paste0(
+  # nolint
+  "SELECT error_occurred FROM ",
+  "single_builds",
+  " WHERE name = $1 AND tag = $2 AND platform = $3 AND arch = $4"
+)
+# Function to query for a single package-version
+query_error <- function(pkg, ver) {
+  purrr::insistently(
+    ~ DBI::dbGetQuery(
+      con,
+      sql_query,
+      params = list(pkg, ver, platform, arch)
+    ),
+    rate = purrr::rate_backoff(
+      pause_base = 1L,
+      pause_cap = 60L,
+      pause_min = 1L,
+      max_times = 10L,
+      jitter = FALSE
+    ),
+    quiet = FALSE
+  )()
+}
+
+# Fetch all relevant columns from the database
+errored_pkgs <- DBI::dbGetQuery(
+  con,
+  sprintf("SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", platform, arch)
+)
+errored_pkgs <- as.data.table(errored_pkgs)
+setkey(pkgs_to_build, Package, Version)
+setnames(errored_pkgs, c("Package","Version"))
+setkey(errored_pkgs, Package, Version)
+
+### Final subsetting
+pkgs_no_error <- pkgs_to_build[!errored_pkgs]
+# only return package names
+pkgs <- unique(as.character(pkgs_no_error[!s3_dt, Package]))