diff --git a/.crow/archive-missed-packages.yaml b/.crow/archive-missed-packages.yaml index 1077b7d..c42f41b 100644 --- a/.crow/archive-missed-packages.yaml +++ b/.crow/archive-missed-packages.yaml @@ -41,10 +41,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: diff --git a/.crow/build-all-versions-amd64.yaml b/.crow/build-all-versions-amd64.yaml index d1b12a5..38203a1 100644 --- a/.crow/build-all-versions-amd64.yaml +++ b/.crow/build-all-versions-amd64.yaml @@ -3,9 +3,9 @@ # OS (alpine) # OS_VERSION (3.22) # R_VERSION (4.5) -# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.23 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 -# crow pipeline create --var ARCH=arm64 --var OS=redhat --var OS_VERSION=10 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 -# crow pipeline create --var ARCH=amd64 --var OS=redhat --var OS_VERSION=10 --var R_VERSION=4.5.3 --var task=build-all-amd64 --var K8S_INSTANCE_TYPE=AX42 --branch=main --log-level=info 5 +# woodpecker-cli pipeline create --var ARCH=amd64 --var OS=alpine --var OS_VERSION=3.23 --var R_VERSION=4.5 --var task=build-all-amd64 --var K8S_INSTANCE_TYPE=AX42 --branch=main --log-level=info 5 +# crow-dev pipeline create --var ARCH=amd64 --var OS=alpine --var OS_VERSION=3.23 --var R_VERSION=4.5 --var task=build-all-amd64 --var K8S_INSTANCE_TYPE=AX42 --branch=main --log-level=info 5 +# crow pipeline create --var ARCH=amd64 --var OS=redhat --var OS_VERSION=10 --var R_VERSION=4.5 --var task=build-all-amd64 --var K8S_INSTANCE_TYPE=AX42 --branch=main --log-level=info 5 when: - event: manual evaluate: 'task == "build-all-amd64"' @@ -14,7 +14,7 @@ skip_clone: true labels: platform: linux/amd64 - agent: artemis + backend: kubernetes matrix: include: @@ -34,8 +34,8 @@ matrix: SPLIT_INDEX: 5 BACKEND: kubernetes -depends_on: - - build-all-versions-install-deps-amd64 +# depends_on: +# - build-all-versions-install-deps steps: - name: 'Build binaries' @@ -44,10 +44,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -65,11 +65,11 @@ steps: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - R -q -e 'if (!requireNamespace("bincraft", quietly = TRUE)) pak::pak("git::https://codefloe.com/rpkgs/bincraft.git"); packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' # pkgs = readRDS('/mnt/cache/pkgs_amd64.rds'); - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = readRDS('/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), $SPLIT_INTO, labels = F))[[$SPLIT_INDEX]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'Rfast', 'compareGroups', 'ff', 'GsymPoint', 'RcppDynProg', 'comtradr', 'RcppDynProg', 'FD', 'PearsonDS', 'DCluster', 'gRc', 'mixlm', 'geospt', 'fdth', 'ffmanova', 'fiery', 'ffscrapr', 'cold', 'fiery', 'RcmdrPlugin.DoE', 'RcmdrPlugin.NMBU', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.UCA', 'RcmdrPlugin.WorldFlora', 'RcmdrPlugin.aRnova', 'RcmdrPlugin.depthTools', 'RcmdrPlugin.orloca', 'RcmdrPlugin.sos', 'RcmdrPlugin.survival', 'RcmdrPlugin.temis', 'GWlasso', 'GWmodelVis')); library(future); plan('sequential'); n = length(pkgs); foo = lapply(seq_along(pkgs), function(i) { x = pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); bincraft::build_binary_package(x, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', tag_limit = 5L, s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE) })" 2>&1 + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), $SPLIT_INTO, labels = F))[[$SPLIT_INDEX]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'doBy', 'IDPmisc', 'frailtypack', 'afex', 'FrF2', 'DoE.base', 'agricolae', 'doFuture', 'fscaret', 'PHYLOGR', 'seewave', 'pls', 'relaimpo', 'geepack', 'gggenes', 'NPCirc', 'repmis', 'PNDSIBGE', 'lidR', 'poismf', 'neonstore', 'MachineShop', 'mvst', 'MacBehaviour', 'mcmcderive', 'RGIFT', 'KnowBR', 'netmeta', 'spdep', 'Rfast', 'compareGroups', 'ff', 'GsymPoint', 'RcppDynProg', 'comtradr', 'RcppDynProg', 'FD', 'PearsonDS', 'DCluster', 'gRc', 'mixlm', 'geospt', 'fdth', 'ffmanova', 'fiery', 'ffscrapr', 'cold', 'fiery', 'RcmdrPlugin.DoE', 'RcmdrPlugin.NMBU', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.UCA', 'RcmdrPlugin.WorldFlora', 'RcmdrPlugin.aRnova', 'RcmdrPlugin.depthTools', 'RcmdrPlugin.orloca', 'RcmdrPlugin.sos', 'RcmdrPlugin.survival', 'RcmdrPlugin.temis', 'GWlasso', 'GWmodelVis')); library(future); plan('multisession'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, s3_endpoint = 'https://hel1.your-objectstorage.com', tag_limit = 5L, s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE))" # archive missed packages - R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" backend_options: diff --git a/.crow/build-all-versions-arm64.yaml b/.crow/build-all-versions-arm64.yaml index 1467da8..e3dc757 100644 --- a/.crow/build-all-versions-arm64.yaml +++ b/.crow/build-all-versions-arm64.yaml @@ -5,8 +5,9 @@ # R_VERSION (4.5) # echo "c('RInno','KeyboardSimulator','R2PPT','RWinEdt','blatr','excel.link','spectrino','taskscheduleR','MDSGUI','BiplotGUI','R2wd','rFUSION','MediaNews','doBy','IDPmisc','frailtypack','afex','FrF2','DoE.base','agricolae','doFuture','fscaret','PHYLOGR','seewave','pls','relaimpo','geepack','gggenes','NPCirc','repmis','PNDSIBGE','lidR','poismf','neonstore','MachineShop','mvst','MacBehaviour','mcmcderive','RGIFT','KnowBR','netmeta','spdep','sf','Rfast','compareGroups','ff','GsymPoint','RcppDynProg','comtradr','RcppDynProg','FD','PearsonDS','DCluster','gRc','mixlm','geospt','fdth','ffmanova','fiery','ffscrapr','cold','fiery','RcmdrPlugin.DoE','RcmdrPlugin.NMBU')" | base64 # crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.22 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --branch=main --var SKIP_PKGS=YygnUklubm8nLCdLZXlib2FyZFNpbXVsYXRvcicsJ1IyUFBUJywnUldpbkVkdCcsJ2JsYXRyJywnZXhjZWwubGluaycsJ3NwZWN0cmlubycsJ3Rhc2tzY2hlZHVsZVInLCdNRFNHVUknLCdCaXBsb3RHVUknLCdSMndkJywnckZVU0lPTicsJ01lZGlhTmV3cycsJ2RvQnknLCdJRFBtaXNjJywnZnJhaWx0eXBhY2snLCdhZmV4JywnRnJGMicsJ0RvRS5iYXNlJywnYWdyaWNvbGFlJywnZG9GdXR1cmUnLCdmc2NhcmV0JywnUEhZTE9HUicsJ3NlZXdhdmUnLCdwbHMnLCdyZWxhaW1wbycsJ2dlZXBhY2snLCdnZ2dlbmVzJywnTlBDaXJjJywncmVwbWlzJywnUE5EU0lCR0UnLCdsaWRSJywncG9pc21mJywnbmVvbnN0b3JlJywnTWFjaGluZVNob3AnLCdtdnN0JywnTWFjQmVoYXZpb3VyJywnbWNtY2Rlcml2ZScsJ1JHSUZUJywnS25vd0JSJywnbmV0bWV0YScsJ3NwZGVwJywnc2YnLCdSZmFzdCcsJ2NvbXBhcmVHcm91cHMnLCdmZicsJ0dzeW1Qb2ludCcsJ1JjcHBEeW5Qcm9nJywnY29tdHJhZHInLCdSY3BwRHluUHJvZycsJ0ZEJywnUGVhcnNvbkRTJywnRENsdXN0ZXInLCdnUmMnLCdtaXhsbScsJ2dlb3NwdCcsJ2ZkdGgnLCdmZm1hbm92YScsJ2ZpZXJ5JywnZmZzY3JhcHInLCdjb2xkJywnZmllcnknLCdSY21kclBsdWdpbi5Eb0UnLCdSY21kclBsdWdpbi5OTUJVJykK --log-level=info 7 +# crow pipeline create --var ARCH=arm64 --var OS=redhat --var OS_VERSION=10 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --var SKIP_PKGS="c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION')" --branch=main --log-level=info 7 +# woodpecker-cli pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.23 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --branch=main --log-level=info 5 # crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.23 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --branch=main --log-level=info 5 -# crow pipeline create --var ARCH=arm64 --var OS=redhat --var OS_VERSION=10 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5 when: - event: manual evaluate: 'task == "build-all-arm64"' @@ -49,8 +50,8 @@ matrix: # SPLIT_INDEX: 5 # BACKEND: docker -depends_on: - - build-all-versions-install-deps-arm64 +# depends_on: +# - build-all-versions-install-deps steps: - name: 'Build binaries' @@ -59,10 +60,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -80,9 +81,12 @@ steps: - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + # need to install bincraft here again as the docker backend cannot use the shared volumes + - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git", dependencies = TRUE)' + - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = readRDS('/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), $SPLIT_INTO, labels = F))[[$SPLIT_INDEX]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'RcmdrPlugin.DoE', 'RcmdrPlugin.NMBU', 'RcmdrPlugin.ROC', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.UCA', 'RcmdrPlugin.WorldFlora', 'RcmdrPlugin.aRnova', 'RcmdrPlugin.depthTools', 'RcmdrPlugin.orloca', 'RcmdrPlugin.sos', 'RcmdrPlugin.survival', 'RcmdrPlugin.temis')); library(future); plan('sequential'); n = length(pkgs); foo = lapply(seq_along(pkgs), function(i) { x = pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); bincraft::build_binary_package(x, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', tag_limit = 5L, s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE) })" 2>&1 + # pkgs = readRDS('/mnt/cache/pkgs_arm64.rds'); + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); source('local/packages-to-build.R'); sprintf('Total# of remaining packages:%s', length(pkgs)); pkgs <- split(pkgs, cut(seq_along(pkgs), $SPLIT_INTO, labels = F))[[$SPLIT_INDEX]]; sprintf('# of packages for this job:%s', length(pkgs)); pkgs = setdiff(pkgs, c('RInno', 'RcmdrPlugin.DoE', 'RcmdrPlugin.NMBU', 'RcmdrPlugin.ROC', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.UCA', 'RcmdrPlugin.WorldFlora', 'RcmdrPlugin.aRnova', 'RcmdrPlugin.depthTools', 'RcmdrPlugin.orloca', 'RcmdrPlugin.sos', 'RcmdrPlugin.survival', 'RcmdrPlugin.temis')); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, s3_endpoint = 'https://hel1.your-objectstorage.com', tag_limit = 5L, s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE));" # archive missed packages - R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)" backend_options: diff --git a/.crow/build-all-versions-install-deps-amd64.yaml b/.crow/build-all-versions-install-deps-amd64.yaml deleted file mode 100644 index a07ea12..0000000 --- a/.crow/build-all-versions-install-deps-amd64.yaml +++ /dev/null @@ -1,57 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-all-amd64"' - -skip_clone: true - -labels: - platform: linux/${ARCH} - agent: artemis - -steps: - - name: 'Install deps and bincraft' - image: reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}-${R_VERSION} - pull: true - environment: - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - volumes: - - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache - commands: - # one-time full wipe to fix corrupted .so files from previous failed builds - # - rm -rf /mnt/cache/R-pkgs - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft - - R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future")); packageVersion("bincraft")' - - R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Precomputed %s packages to build', length(pkgs))" - backend_options: - docker: - resources: - requests: - memory: 5Gi - cpu: 1000m - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-all-versions-install-deps-arm64.yaml b/.crow/build-all-versions-install-deps-arm64.yaml deleted file mode 100644 index 4639249..0000000 --- a/.crow/build-all-versions-install-deps-arm64.yaml +++ /dev/null @@ -1,56 +0,0 @@ -when: - - event: manual - evaluate: 'task == "build-all-arm64"' - -skip_clone: true - -labels: - agent: gaia - -steps: - - name: 'Install deps and bincraft' - image: reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}-${R_VERSION} - pull: true - environment: - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - volumes: - - ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache - commands: - # one-time full wipe to fix corrupted .so files from previous failed builds - # - rm -rf /mnt/cache/R-pkgs - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft - - R -q -e 'pak::sysreqs_db_update(); pak::local_install("/tmp/bincraft"); pak::pak(c("RPostgres", "s3fs", "data.table", "future")); packageVersion("bincraft")' - - R -q -e "source('local/packages-to-build.R'); saveRDS(pkgs, '/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Precomputed %s packages to build', length(pkgs))" - backend_options: - docker: - resources: - requests: - memory: 5Gi - cpu: 1000m - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 1000m - nodeSelector: - kubernetes.io/arch: ${ARCH} - node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE} - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/build-r-minor-sensitive-packages.yaml b/.crow/build-r-minor-sensitive-packages.yaml index 6e9d10c..8f41f43 100644 --- a/.crow/build-r-minor-sensitive-packages.yaml +++ b/.crow/build-r-minor-sensitive-packages.yaml @@ -31,10 +31,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -52,18 +52,15 @@ steps: from_secret: ntfy_token # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /tmp/R-libs:/mnt/cache/R-pkgs + R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache NCPUS: 2 STRATEGY: sequential commands: - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - mkdir -p /tmp/R-libs - - git clone -q https://codefloe.com/rpkgs/bincraft.git /tmp/bincraft && R CMD INSTALL --library=/tmp/R-libs /tmp/bincraft && R -q -e 'packageVersion("bincraft")' + - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run) - - $XVFB -- R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = bincraft::get_r_minor_sensitive_packages(r_minor_packages_issue_url = 'https://git.devxy.io/api/v1/repos/devxy/build-cran-binaries/issues/29'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, is_r_minor_sensitive = TRUE, force = FALSE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE))" 2>&1 + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); pkgs = bincraft::get_r_minor_sensitive_packages(r_minor_packages_issue_url = 'https://git.devxy.io/api/v1/repos/devxy/build-cran-binaries/issues/29'); foo = lapply(pkgs, function(x) bincraft::build_binary_package(x, is_debug = FALSE, is_r_minor_sensitive = TRUE, force = FALSE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE))" backend_options: kubernetes: resources: diff --git a/.crow/process-updates-alpine-321-amd64.yaml b/.crow/process-updates-alpine-321-amd64.yaml index b22fdd4..22585f2 100644 --- a/.crow/process-updates-alpine-321-amd64.yaml +++ b/.crow/process-updates-alpine-321-amd64.yaml @@ -13,15 +13,15 @@ labels: steps: - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.4.3 + image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.4.3 pull: true environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -55,7 +55,7 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" backend_options: kubernetes: resources: diff --git a/.crow/process-updates-alpine-321-arm64.yaml b/.crow/process-updates-alpine-321-arm64.yaml index 7e93dd2..d0eb651 100644 --- a/.crow/process-updates-alpine-321-arm64.yaml +++ b/.crow/process-updates-alpine-321-arm64.yaml @@ -13,15 +13,15 @@ labels: steps: - name: 'Processing Updates' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.4.3 + image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.4.3 pull: true environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -54,4 +54,6 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + + diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml index c76fd2e..29377cc 100644 --- a/.crow/process-updates-alpine-322-amd64.yaml +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -55,7 +55,7 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" backend_options: kubernetes: resources: diff --git a/.crow/process-updates-alpine-322-arm64.yaml b/.crow/process-updates-alpine-322-arm64.yaml index 81989b8..e598cc4 100644 --- a/.crow/process-updates-alpine-322-arm64.yaml +++ b/.crow/process-updates-alpine-322-arm64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -54,6 +54,6 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" diff --git a/.crow/process-updates-alpine-323-amd64.yaml b/.crow/process-updates-alpine-323-amd64.yaml index bbce076..0a3206a 100644 --- a/.crow/process-updates-alpine-323-amd64.yaml +++ b/.crow/process-updates-alpine-323-amd64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -55,7 +55,7 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-323', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" backend_options: kubernetes: resources: diff --git a/.crow/process-updates-alpine-323-arm64.yaml b/.crow/process-updates-alpine-323-arm64.yaml index 9dc33dc..b944ca0 100644 --- a/.crow/process-updates-alpine-323-arm64.yaml +++ b/.crow/process-updates-alpine-323-arm64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -54,6 +54,6 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'alpine-321', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" diff --git a/.crow/process-updates-redhat-8-amd64.yaml b/.crow/process-updates-redhat-8-amd64.yaml index 43b1781..d96042a 100644 --- a/.crow/process-updates-redhat-8-amd64.yaml +++ b/.crow/process-updates-redhat-8-amd64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -55,7 +55,7 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" backend_options: kubernetes: resources: diff --git a/.crow/process-updates-redhat-8-arm64.yaml b/.crow/process-updates-redhat-8-arm64.yaml index c204439..6683a09 100644 --- a/.crow/process-updates-redhat-8-arm64.yaml +++ b/.crow/process-updates-redhat-8-arm64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -54,6 +54,6 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-8', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" diff --git a/.crow/process-updates-redhat-9-amd64.yaml b/.crow/process-updates-redhat-9-amd64.yaml index d78a60e..1d2b80a 100644 --- a/.crow/process-updates-redhat-9-amd64.yaml +++ b/.crow/process-updates-redhat-9-amd64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -56,7 +56,7 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" backend_options: kubernetes: resources: diff --git a/.crow/process-updates-redhat-9-arm64.yaml b/.crow/process-updates-redhat-9-arm64.yaml index 49ce90c..d9b3c93 100644 --- a/.crow/process-updates-redhat-9-arm64.yaml +++ b/.crow/process-updates-redhat-9-arm64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -54,6 +54,6 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'redhat-9', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" diff --git a/.crow/process-updates-ubuntu-2204-amd64.yaml b/.crow/process-updates-ubuntu-2204-amd64.yaml index 54fb43c..bc16fb1 100644 --- a/.crow/process-updates-ubuntu-2204-amd64.yaml +++ b/.crow/process-updates-ubuntu-2204-amd64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -56,7 +56,7 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" backend_options: kubernetes: resources: diff --git a/.crow/process-updates-ubuntu-2204-arm64.yaml b/.crow/process-updates-ubuntu-2204-arm64.yaml index 666a8b3..40995f6 100644 --- a/.crow/process-updates-ubuntu-2204-arm64.yaml +++ b/.crow/process-updates-ubuntu-2204-arm64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -54,6 +54,6 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2204', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" diff --git a/.crow/process-updates-ubuntu-2404-amd64.yaml b/.crow/process-updates-ubuntu-2404-amd64.yaml index 1be58f4..67f416c 100644 --- a/.crow/process-updates-ubuntu-2404-amd64.yaml +++ b/.crow/process-updates-ubuntu-2404-amd64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -55,7 +55,7 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" backend_options: kubernetes: resources: diff --git a/.crow/process-updates-ubuntu-2404-arm64.yaml b/.crow/process-updates-ubuntu-2404-arm64.yaml index 0a4d9f2..a4d814c 100644 --- a/.crow/process-updates-ubuntu-2404-arm64.yaml +++ b/.crow/process-updates-ubuntu-2404-arm64.yaml @@ -18,10 +18,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -54,6 +54,6 @@ steps: - R -q -e 'packageVersion("bincraft")' - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise - - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" diff --git a/.crow/update-package-index-alpine-321-amd64.yaml b/.crow/update-package-index-alpine-321-amd64.yaml new file mode 100644 index 0000000..a9f0181 --- /dev/null +++ b/.crow/update-package-index-alpine-321-amd64.yaml @@ -0,0 +1,66 @@ +when: + - event: [cron] + cron: update-package-index-alpine-321-amd64 + - event: [cron] + cron: update-package-index + - event: manual + evaluate: 'update_package_index == "alpine-321-amd64"' + +skip_clone: true + +steps: + - name: Upload PACKAGES files + image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + # volumes: + # - amd64-binaries-r-dep-cache-alpine-321:/mnt/cache + commands: + - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . + - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' + - R -q -e 'pak::pak("pat-s/cranlike@s3")' + - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' + - R -q -e 'packageVersion("bincraft")' + # - cd /mnt/cache/packages + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine321", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' + backend_options: + kubernetes: + resources: + requests: + memory: 1800Mi + cpu: 2000m + limits: + memory: 1800Mi + cpu: 8000m + nodeSelector: + kubernetes.io/arch: 'amd64' + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/update-package-index-alpine-321-arm64.yaml b/.crow/update-package-index-alpine-321-arm64.yaml new file mode 100644 index 0000000..c7a6b5b --- /dev/null +++ b/.crow/update-package-index-alpine-321-arm64.yaml @@ -0,0 +1,61 @@ +when: + - event: [cron] + cron: update-package-index-alpine-321-arm64 + - event: [cron] + cron: update-package-index + - event: manual + evaluate: 'update_package_index == "alpine-321-arm64"' + +steps: + - name: Upload PACKAGES files + image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S + PGPASS: + from_secret: PGPASS + REPO_RO_TOKEN: + from_secret: REPO_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + commands: + - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' + - R -q -e 'pak::pak("pat-s/cranlike@s3")' + - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' + - R -q -e 'packageVersion("bincraft")' + - cd /mnt/cache/packages + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine321", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' + backend_options: + kubernetes: + resources: + requests: + memory: 1800Mi + cpu: 2000m + limits: + memory: 1800Mi + cpu: 8000m + nodeSelector: + kubernetes.io/arch: 'arm64' + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' diff --git a/.crow/update-package-index-alpine-322-amd64.yaml b/.crow/update-package-index-alpine-322-amd64.yaml index 9b3c157..9b5768b 100644 --- a/.crow/update-package-index-alpine-322-amd64.yaml +++ b/.crow/update-package-index-alpine-322-amd64.yaml @@ -15,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -47,15 +47,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' # - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine322", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' diff --git a/.crow/update-package-index-alpine-322-arm64.yaml b/.crow/update-package-index-alpine-322-arm64.yaml index b697462..387ea0c 100644 --- a/.crow/update-package-index-alpine-322-arm64.yaml +++ b/.crow/update-package-index-alpine-322-arm64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-alpine-322-amd64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -50,15 +47,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' # - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine322", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine322", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64' diff --git a/.crow/update-package-index-alpine-323-amd64.yaml b/.crow/update-package-index-alpine-323-amd64.yaml index fd496f2..ea5f861 100644 --- a/.crow/update-package-index-alpine-323-amd64.yaml +++ b/.crow/update-package-index-alpine-323-amd64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-alpine-322-arm64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -50,15 +47,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' # - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine323", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' diff --git a/.crow/update-package-index-alpine-323-arm64.yaml b/.crow/update-package-index-alpine-323-arm64.yaml index bbba71c..78aebbb 100644 --- a/.crow/update-package-index-alpine-323-arm64.yaml +++ b/.crow/update-package-index-alpine-323-arm64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-alpine-323-amd64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -50,15 +47,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' # - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine323", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine323", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64' diff --git a/.crow/update-package-index-redhat-8-amd64.yaml b/.crow/update-package-index-redhat-8-amd64.yaml index e3d5840..74f77a5 100644 --- a/.crow/update-package-index-redhat-8-amd64.yaml +++ b/.crow/update-package-index-redhat-8-amd64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-alpine-323-arm64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -47,15 +44,15 @@ steps: - R -q -e 'pak::pak("pat-s/cranlike@s3")' - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel8", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel8", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' diff --git a/.crow/update-package-index-redhat-8-arm64.yaml b/.crow/update-package-index-redhat-8-arm64.yaml index 0e3111d..8f6a488 100644 --- a/.crow/update-package-index-redhat-8-arm64.yaml +++ b/.crow/update-package-index-redhat-8-arm64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-redhat-8-amd64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -48,15 +45,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel8",s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel8",s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64' diff --git a/.crow/update-package-index-redhat-9-amd64.yaml b/.crow/update-package-index-redhat-9-amd64.yaml index 45a45b9..8b592b1 100644 --- a/.crow/update-package-index-redhat-9-amd64.yaml +++ b/.crow/update-package-index-redhat-9-amd64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-redhat-8-arm64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -48,15 +45,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel9", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' @@ -67,7 +64,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/proxy_cache/library/alpine:3.22 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/update-package-index-redhat-9-arm64.yaml b/.crow/update-package-index-redhat-9-arm64.yaml index 3dabf98..9b1a7b1 100644 --- a/.crow/update-package-index-redhat-9-arm64.yaml +++ b/.crow/update-package-index-redhat-9-arm64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-redhat-9-amd64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -48,15 +45,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel9", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "rhel9", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64' @@ -67,7 +64,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/proxy_cache/library/alpine:3.22 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/update-package-index-ubuntu-2204-amd64.yaml b/.crow/update-package-index-ubuntu-2204-amd64.yaml index d2b159a..503c541 100644 --- a/.crow/update-package-index-ubuntu-2204-amd64.yaml +++ b/.crow/update-package-index-ubuntu-2204-amd64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-redhat-9-arm64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -48,15 +45,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY")); warnings()' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "jammy", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S")); warnings()' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' @@ -67,7 +64,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/proxy_cache/library/alpine:3.22 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/update-package-index-ubuntu-2204-arm64.yaml b/.crow/update-package-index-ubuntu-2204-arm64.yaml index 57a8104..044ae82 100644 --- a/.crow/update-package-index-ubuntu-2204-arm64.yaml +++ b/.crow/update-package-index-ubuntu-2204-arm64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-ubuntu-2204-amd64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -48,15 +45,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "jammy", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "jammy", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64' @@ -67,7 +64,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/proxy_cache/library/alpine:3.22 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/update-package-index-ubuntu-2404-amd64.yaml b/.crow/update-package-index-ubuntu-2404-amd64.yaml index f507c0b..4ab14a1 100644 --- a/.crow/update-package-index-ubuntu-2404-amd64.yaml +++ b/.crow/update-package-index-ubuntu-2404-amd64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-ubuntu-2204-arm64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -50,15 +47,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "noble", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'amd64' @@ -69,7 +66,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/proxy_cache/library/alpine:3.22 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/update-package-index-ubuntu-2404-arm64.yaml b/.crow/update-package-index-ubuntu-2404-arm64.yaml index e052a66..c0e9f0a 100644 --- a/.crow/update-package-index-ubuntu-2404-arm64.yaml +++ b/.crow/update-package-index-ubuntu-2404-arm64.yaml @@ -8,9 +8,6 @@ when: skip_clone: true -depends_on: - - update-package-index-ubuntu-2404-amd64 - steps: - name: Upload PACKAGES files image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4 @@ -18,10 +15,10 @@ steps: environment: RED_HAT_DEV_PW: from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S PGPASS: from_secret: PGPASS REPO_RO_TOKEN: @@ -48,15 +45,15 @@ steps: - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' - R -q -e 'packageVersion("bincraft")' - cd /mnt/cache/packages - - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))' + - R -q -e 'options(crayon.enabled = TRUE); library(bincraft); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "noble", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' backend_options: kubernetes: resources: requests: - memory: 2600Mi + memory: 1800Mi cpu: 2000m limits: - memory: 2600Mi + memory: 1800Mi cpu: 8000m nodeSelector: kubernetes.io/arch: 'arm64' @@ -67,7 +64,7 @@ steps: effect: 'NoSchedule' - name: Purge CDN cache - image: reg.devxy.io/docker.io/library/alpine:3.23 + image: reg.devxy.io/proxy_cache/library/alpine:3.22 environment: BUNNYNET_API_KEY: from_secret: BUNNYNET_API_KEY diff --git a/.crow/weekly-audit-missing-alpine-321-amd64.yaml b/.crow/weekly-audit-missing-alpine-321-amd64.yaml deleted file mode 100644 index e2884eb..0000000 --- a/.crow/weekly-audit-missing-alpine-321-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-321-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-alpine-321-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: alpine-321 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-alpine321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-alpine-321-arm64.yaml b/.crow/weekly-audit-missing-alpine-321-arm64.yaml deleted file mode 100644 index 8711031..0000000 --- a/.crow/weekly-audit-missing-alpine-321-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-321-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-alpine-321-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: alpine-321 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-alpine321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-alpine-322-amd64.yaml b/.crow/weekly-audit-missing-alpine-322-amd64.yaml deleted file mode 100644 index 0f811e8..0000000 --- a/.crow/weekly-audit-missing-alpine-322-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-322-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-alpine-322-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: alpine-322 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-alpine322:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-alpine-322-arm64.yaml b/.crow/weekly-audit-missing-alpine-322-arm64.yaml deleted file mode 100644 index 2a1b821..0000000 --- a/.crow/weekly-audit-missing-alpine-322-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-322-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-alpine-322-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: alpine-322 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-alpine322:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-alpine-323-amd64.yaml b/.crow/weekly-audit-missing-alpine-323-amd64.yaml deleted file mode 100644 index 67546a1..0000000 --- a/.crow/weekly-audit-missing-alpine-323-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-323-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-alpine-323-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: alpine-323 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-alpine323:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-alpine-323-arm64.yaml b/.crow/weekly-audit-missing-alpine-323-arm64.yaml deleted file mode 100644 index 9f097c1..0000000 --- a/.crow/weekly-audit-missing-alpine-323-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-alpine-323-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-alpine-323-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: alpine-323 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-alpine323:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-8-amd64.yaml b/.crow/weekly-audit-missing-redhat-8-amd64.yaml deleted file mode 100644 index e30da4f..0000000 --- a/.crow/weekly-audit-missing-redhat-8-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-redhat-8-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-redhat-8-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: redhat-8 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-redhat8:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-8-arm64.yaml b/.crow/weekly-audit-missing-redhat-8-arm64.yaml deleted file mode 100644 index 7c6c97a..0000000 --- a/.crow/weekly-audit-missing-redhat-8-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-redhat-8-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-redhat-8-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: redhat-8 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-redhat8:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-9-amd64.yaml b/.crow/weekly-audit-missing-redhat-9-amd64.yaml deleted file mode 100644 index 9e273b0..0000000 --- a/.crow/weekly-audit-missing-redhat-9-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-redhat-9-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-redhat-9-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: redhat-9 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-redhat9:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-redhat-9-arm64.yaml b/.crow/weekly-audit-missing-redhat-9-arm64.yaml deleted file mode 100644 index a104d93..0000000 --- a/.crow/weekly-audit-missing-redhat-9-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-redhat-9-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-redhat-9-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: redhat-9 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-redhat9:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml b/.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml deleted file mode 100644 index 97008f6..0000000 --- a/.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-ubuntu-2204-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-ubuntu-2204-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: ubuntu-2204 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-ubuntu2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml b/.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml deleted file mode 100644 index b81c49c..0000000 --- a/.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-ubuntu-2204-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-ubuntu-2204-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: ubuntu-2204 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-ubuntu2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml b/.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml deleted file mode 100644 index 09162a6..0000000 --- a/.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-ubuntu-2404-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-ubuntu-2404-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: ubuntu-2404 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-ubuntu2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml b/.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml deleted file mode 100644 index 3be2a3b..0000000 --- a/.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml +++ /dev/null @@ -1,52 +0,0 @@ -when: - - event: cron - cron: weekly-audit-missing-ubuntu-2404-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-ubuntu-2404-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: ubuntu-2404 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-ubuntu2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-321-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-321-amd64.yaml deleted file mode 100644 index f43f1fd..0000000 --- a/.crow/weekly-rebuild-missing-alpine-321-amd64.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-321-amd64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-321-amd64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 - ARCH: amd64 - NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-alpine321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-321_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-321-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-321-arm64.yaml deleted file mode 100644 index c24f440..0000000 --- a/.crow/weekly-rebuild-missing-alpine-321-arm64.yaml +++ /dev/null @@ -1,58 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-321-arm64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-321-arm64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 - ARCH: arm64 - NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-alpine321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-321_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml deleted file mode 100644 index 84a7ffe..0000000 --- a/.crow/weekly-rebuild-missing-alpine-322-amd64.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-322-amd64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-322-amd64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - ARCH: amd64 - NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-alpine322:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-322_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml deleted file mode 100644 index a181aa4..0000000 --- a/.crow/weekly-rebuild-missing-alpine-322-arm64.yaml +++ /dev/null @@ -1,58 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-322-arm64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-322-arm64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-322 - ARCH: arm64 - NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-alpine322:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-322_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml b/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml deleted file mode 100644 index 1196f6c..0000000 --- a/.crow/weekly-rebuild-missing-alpine-323-amd64.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-323-amd64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-323-amd64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-323 - ARCH: amd64 - NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-alpine323:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-323_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml b/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml deleted file mode 100644 index 13eb4f9..0000000 --- a/.crow/weekly-rebuild-missing-alpine-323-arm64.yaml +++ /dev/null @@ -1,58 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-alpine-323-arm64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-alpine-323-arm64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-323 - ARCH: arm64 - NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-alpine323:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_alpine-323_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml deleted file mode 100644 index 10e3fa1..0000000 --- a/.crow/weekly-rebuild-missing-redhat-8-amd64.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-redhat-8-amd64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-redhat-8-amd64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - ARCH: amd64 - NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-redhat8:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_redhat-8_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml deleted file mode 100644 index 60e88ea..0000000 --- a/.crow/weekly-rebuild-missing-redhat-8-arm64.yaml +++ /dev/null @@ -1,58 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-redhat-8-arm64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-redhat-8-arm64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-8 - ARCH: arm64 - NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-redhat8:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_redhat-8_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml b/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml deleted file mode 100644 index ae2bf6c..0000000 --- a/.crow/weekly-rebuild-missing-redhat-9-amd64.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-redhat-9-amd64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-redhat-9-amd64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: amd64 - NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-redhat9:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_redhat-9_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml b/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml deleted file mode 100644 index eeb8a8b..0000000 --- a/.crow/weekly-rebuild-missing-redhat-9-arm64.yaml +++ /dev/null @@ -1,58 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-redhat-9-arm64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-redhat-9-arm64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: arm64 - NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-redhat9:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_redhat-9_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml deleted file mode 100644 index 696e7bf..0000000 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-ubuntu-2204-amd64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-ubuntu-2204-amd64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: amd64 - NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-ubuntu2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_ubuntu-2204_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml deleted file mode 100644 index 5de2a7b..0000000 --- a/.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml +++ /dev/null @@ -1,58 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-ubuntu-2204-arm64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-ubuntu-2204-arm64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: arm64 - NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-ubuntu2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_ubuntu-2204_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml deleted file mode 100644 index 5c8a0d5..0000000 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml +++ /dev/null @@ -1,59 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-ubuntu-2404-amd64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-ubuntu-2404-amd64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - ARCH: amd64 - NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-ubuntu2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_ubuntu-2404_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml b/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml deleted file mode 100644 index 52c8d46..0000000 --- a/.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml +++ /dev/null @@ -1,58 +0,0 @@ -when: - - event: cron - cron: weekly-rebuild-missing-ubuntu-2404-arm64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-ubuntu-2404-arm64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2404 - ARCH: arm64 - NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-ubuntu2404:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_ubuntu-2404_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s - %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index d2933d4..a1319cd 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -28,45 +28,45 @@ provider "registry.opentofu.org/hashicorp/http" { } provider "registry.terraform.io/bunnyway/bunnynet" { - version = "0.14.0" - constraints = "~> 0.14" + version = "0.13.4" + constraints = "~> 0.13" hashes = [ - "h1:5dK81yu2D9pLqhUJfT7iyzewDbxwdXGUuN2w7jurmKc=", - "h1:83Ro2f+3bHY79xUx/YXDsR1lPUoln3PRQFWsdvw1/xo=", - "h1:E8TqDL4A+Y7W1uI6zykNEfvJAmqkX1ZiRuAik90CZ3c=", - "h1:HLWBmzinzvwVUUJVCPb49/XUalWfkcgaP8w86dFYthw=", - "h1:JqlHrtekdtco6352QgPeIBtonmgtu4BppT6oCmzClKw=", - "h1:MSJldIkNDRjBwI4oag4s7RWXR9SwXcLHlnhPEl5XMb4=", - "h1:NWoevYv4hokmc8NPa1IJy1IoR49m26eBaX/x9IxKz94=", - "h1:UpGqBA0Tp2FaS4VVRlZpftL/mOlVhzqeoNdCdMJW/wI=", - "h1:YIYCwHDMtpWGWrGDn0ldaNB7Mn/Y9lFXbBBDEaZ8giw=", - "h1:YsusA72rg1RfVY+wssc2fjb3KM51zqoCh6zW3kIx9qA=", - "h1:eCJXYI6wMiIy1tie3vojTOAOJ0iUhy7KIvvZhImhrMM=", - "h1:fbwy3KSnZ57C+oNZiAwMq54QcGypkMWmBGQnJsYm2dM=", - "h1:ibwDPsYngqGf1NfQJ4RNG2JrMcUbtorJp0dz3QSK4ec=", - "h1:lV0MOJX23+sJJgRZPvgMe4aTwqVoK944gvmqpgJNUE4=", - "h1:naTBOoQEpSMZdxnrv6YPELsWkGXdlUZUpPUoPsTwZnQ=", - "h1:pEo7AkMij4CywNUzEL/gnrj2XakCLCaCXsHF5cQdBTo=", - "h1:qnDXOmPovQciqHjxbkyf9FK2IeqO0g+nMO6vT1pf/JA=", - "h1:zdYkIRTrC34OmnFmyGOyvUfDNpfN6wkXLHluGxzC4Ts=", - "zh:0e9b51ed64d17a65a9af9225fcaf93087698ba7b24f1aba4d789961b4d6b6e3b", - "zh:1ac06087182262069c5d9273e6ae346873f55c15aab4434a4f82c422cd99fda9", - "zh:251fc057494944f4c08b87f42960a7c1bb643c3df078aa1cc0debb99fca08d0d", - "zh:3843a46b59090fa64177ab0092ca9eb43df17e0706c972d47d75b437f07b596e", - "zh:3f56e8f3b1cb1855cc6aba36bc46664e4eb3f517d2ab09756258647029ff6cb4", - "zh:65be645ba621a38ace4673679c2ef47efa1d28538b9e2d40cc93e5920636aa00", - "zh:6a2cf02b745ec0588469d28e2f256616498843cd9f4a8e454b0e73ce609e7de1", - "zh:6d7aea79735c9b8ae6826d2b9f79895f0187f769a452b37a0c5984d5c37bf28f", - "zh:79f498296d1d3d3045275b1d9ef611586fbbdc7f5a14cd0ffb83feda153bcf3d", + "h1:+QVbWKiRIgDa79+R67+G+ySOY2NOh4XF+QUwqyD0J4k=", + "h1:+ltLNndD5k/rabC63ze1+BkJSaC9Ac1ehGmSwqNWI6U=", + "h1:A4e/OeVz2NlLy6KQ6q5bMSyA0UfWuqYB0ohM1RigGoU=", + "h1:C2FJP56+UcIkezMISrqeujQSHpF2QKZpl8kuj1ys/YI=", + "h1:EFdQc1h0NgX0udXy+2wdql55kcxhRKHVXZKf10E6Cg4=", + "h1:Iu55SQd1y8SNRJyVyhe9ogOdH3We1ZwzS5cQ5cR6Umc=", + "h1:K86Qiyp7K06VftyB12Ed1X/tb0zumIyrj8xWtq8pV+E=", + "h1:L7U3k3vzpD2+2ezosfDYdRlWJ110cdbKo94TZKJQmZg=", + "h1:Laeu1nC2ddKlfG93i5DMFUDyS8BSM9av34NlAQO2CR8=", + "h1:OYtriazjtHv2mDc6KU3XIqF2miO3a0KRxDR6zT4TZx8=", + "h1:QfVNEXsQ/ALjYe5QBtim3qpshIKOOweRzCpwSFGP0cw=", + "h1:T2PAr/O1JH/IG/Ka1e/EIsaDbhYiOm8h2gxXdSm4LfE=", + "h1:W4pDBoFOey25b/VRTewyAqzv+Xr+3zCVbnRRzPaXS2c=", + "h1:XYagIE0zHsSd9NlMVJNxE+Iz1UYl8Sike8dvkp4JruQ=", + "h1:abltdSwafUZvv0lNysgSQamfw7v+oNoMsjmzW8hOPwM=", + "h1:iWD13nK8IyG1V9E+M2p5++Cd5dgXCXg6FMe/Z7gX1VA=", + "h1:ks8OuNP6JHcP6ktm6BOzgDyAluawnfe7A9iwS903M68=", + "h1:oq+tSuDV1VmWIFUhaZl1F3cYvmRHqPnGv2EI8R7vZ7U=", + "zh:0e6df94293a7859959de39ca1d56b3d5c1b4ce55eea6e92b04f9ca8aafaba231", + "zh:11e882c60fd78bf503fa2bf27f2a38c95420c9175592a4304b6583977f1562ff", + "zh:254bced0a359115f474669f1e2ce7510d543547852f7d5f7418c61bf6fe3b6d4", + "zh:331ad3b5ffe986aaab766afd84fbc02d415e4d6387578f83045a088b272132b3", + "zh:431213f35b575d7ec5f5795bef6cda38f7d652a3eae2fdba8c6afbfe67d86c57", + "zh:51f54c160deb26670947af0ef19fc6b2f198e58e97c986a70386588db62c6326", + "zh:5e3d843ce3ed808898f459d8df2c711fb948e4f68a4383d8a29c5ff150ad620d", + "zh:69d2b75371bc71a55daff9b751e88a7b53a1acca805b570adbaaeb25fb29b1cb", + "zh:6c859258c7b6b0dd7857dec897656acbff12127b7ba1e6a2692a61c2125cc87b", + "zh:828b8a1f095436706883bffd8ffd2a522b9025fb57f34b4a9c003fc0d841ac7c", + "zh:839a56585ffb3702b1bda4e53220ea6e20c7590d645f9f2dfd238f2988598c20", "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:90d535db842477024f27d09da3ed8350dc3f6e23e9c20b59f8e9cc203bb68568", - "zh:a4cafc4c87ebc2ec016550066ce5bb35cd2be2ad604020ead97dd0ca1235fac0", - "zh:ab2d0fd1032a0bdbce7b25d0c30dfaf66874893742373ebd1e367e19e73f8fcd", - "zh:af81e05190ea94ba53e2f1a31b99bf498749284d3c4c8ea3a1b6494d2519b657", - "zh:b3c9d5b5423d93c942a32df65fab29bc87d19106970fab074e829a30db45e297", - "zh:b3d952f133653e58998324a3ac9405d733c00531283ede47124e73e5d72c0237", - "zh:b93406d06d2ba4c3eaec8d57eb8e18e2a6affcddaa806fbe68b2de0fd6456984", - "zh:d3183423fa2ebce97c3babf5ce25ef624978912d48ae4a2e078d22367c29fea7", - "zh:f0260f7119c4db1c7a1286a78ad27750fdbacee2a3e80e4213122af660d26a82", + "zh:9ac707f81ac6631bd6d301ab903b3cd599d246d38b4f4a3221ef7eed58396db7", + "zh:a0bd1b34aa77247000842552386aa82d4e29edc4f0038007666fc697c2fe572c", + "zh:a33e995039454e9a2961e1586a5981df4f63bd409f9726617747a52e1d0ea646", + "zh:b714bcb76ad39eb112268861d36316b6be2b1e75554e57756cf770bf837b07e8", + "zh:d81726a31fc7cc5c8b42651144b28532f633994898e96e495a7e20eddbae05bb", + "zh:e0441c4c9c1e9e0f180b375cc72079ae532b2f2028c4e38a648bf38b971ca42a", + "zh:f315dc180ebebcd371c8eed19c7756b29adac33c0d5b9d4880e3c0c6e4f3dd6a", ] } diff --git a/cdn.tf b/cdn.tf index 8bf55f1..cdd5f90 100644 --- a/cdn.tf +++ b/cdn.tf @@ -5,7 +5,7 @@ resource "bunnynet_pullzone" "devxy-r-binaries" { origin { type = "OriginUrl" - url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + url = "https://devxy-r-package-binaries-hel1.hel1.your-objectstorage.com" } routing { @@ -13,9 +13,9 @@ resource "bunnynet_pullzone" "devxy-r-binaries" { } s3_auth_enabled = true - s3_auth_key = var.B2_S3_ACCESS_KEY - s3_auth_secret = var.B2_S3_SECRET_KEY - s3_auth_region = "eu-central-003" + s3_auth_key = var.HETZNER_S3_ACCESS_KEY_K3S + s3_auth_secret = var.HETZNER_S3_SECRET_KEY_K3S + s3_auth_region = "hel1" cache_enabled = true cache_errors = true @@ -59,7 +59,7 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { origin { type = "OriginUrl" - url = "https://devxy-rpkgs-binaries.s3.eu-central-003.backblazeb2.com" + url = "https://devxy-r-package-binaries-hel1.hel1.your-objectstorage.com" } routing { @@ -67,9 +67,9 @@ resource "bunnynet_pullzone" "cran_rpkgs_com" { } s3_auth_enabled = true - s3_auth_key = var.B2_S3_ACCESS_KEY - s3_auth_secret = var.B2_S3_SECRET_KEY - s3_auth_region = "eu-central-003" + s3_auth_key = var.HETZNER_S3_ACCESS_KEY_K3S + s3_auth_secret = var.HETZNER_S3_SECRET_KEY_K3S + s3_auth_region = "hel1" cache_enabled = true cache_errors = true diff --git a/docs/superpowers/plans/2026-04-11-weekly-missing-binaries.md b/docs/superpowers/plans/2026-04-11-weekly-missing-binaries.md deleted file mode 100644 index d7a099c..0000000 --- a/docs/superpowers/plans/2026-04-11-weekly-missing-binaries.md +++ /dev/null @@ -1,905 +0,0 @@ -# Weekly Missing Binaries Audit & Rebuild — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a weekly CI workflow that audits CRAN packages missing binaries for their latest release version, reports them in Forgejo issues by OS family, and rebuilds those without a prior build failure. - -**Architecture:** Two independent workflow sets (audit + rebuild), each with one YAML per platform/arch (14 each). A shared R script does the audit logic. An excluded-packages JSON config is the shared blocklist. The audit writes per-platform RDS files consumed by rebuild workflows. - -**Tech Stack:** R (bincraft, s3fs, DBI/RPostgres, jsonlite, httr2), Crow/Woodpecker CI YAML, Forgejo API, PostgreSQL, Backblaze S3. - ---- - -## File Structure - -| File | Responsibility | -|------|---------------| -| `local/excluded-packages.json` | Excluded packages with reasons (single source of truth) | -| `local/weekly-missing-binaries-audit.R` | Audit logic: S3 check, DB check, issue update, RDS output | -| `.crow/weekly-audit-missing-{platform}-{arch}.yaml` (14 files) | Audit workflow per platform/arch | -| `.crow/weekly-rebuild-missing-{platform}-{arch}.yaml` (14 files) | Rebuild workflow per platform/arch | - -## Platform Reference - -Used throughout the plan. Each row is one workflow file pair (audit + rebuild). - -| PLATFORM env var | ARCH | S3 codename (sub("-","",PLATFORM)) | OS Family | Image | -|-----------------|------|-----------|-----------|-------| -| ubuntu-2204 | amd64 | ubuntu2204 | Ubuntu | reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 | -| ubuntu-2204 | arm64 | ubuntu2204 | Ubuntu | reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 | -| ubuntu-2404 | amd64 | ubuntu2404 | Ubuntu | reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 | -| ubuntu-2404 | arm64 | ubuntu2404 | Ubuntu | reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 | -| alpine-321 | amd64 | alpine321 | Alpine | reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 | -| alpine-321 | arm64 | alpine321 | Alpine | reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 | -| alpine-322 | amd64 | alpine322 | Alpine | reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 | -| alpine-322 | arm64 | alpine322 | Alpine | reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 | -| alpine-323 | amd64 | alpine323 | Alpine | reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 | -| alpine-323 | arm64 | alpine323 | Alpine | reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 | -| redhat-8 | amd64 | redhat8 | Red Hat | reg.devxy.io/rpkgs/build-env-redhat:8-4.4.3 | -| redhat-8 | arm64 | redhat8 | Red Hat | reg.devxy.io/rpkgs/build-env-redhat:8-4.4.3 | -| redhat-9 | amd64 | redhat9 | Red Hat | reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 | -| redhat-9 | arm64 | redhat9 | Red Hat | reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 | - ---- - -## Task 1: Create excluded-packages.json - -**Files:** -- Create: `local/excluded-packages.json` - -- [ ] **Step 1: Create the JSON file** - -Extract every package from the inline exclusion list in `.crow/build-all-versions-amd64.yaml` (line 72). The list has two categories: Windows-only packages (comment on line 69) and problematic packages (hang/OOM). Create `local/excluded-packages.json`: - -```json -[ - {"package": "RInno", "reason": "windows-only"}, - {"package": "KeyboardSimulator", "reason": "windows-only"}, - {"package": "R2PPT", "reason": "windows-only"}, - {"package": "RWinEdt", "reason": "windows-only"}, - {"package": "blatr", "reason": "windows-only"}, - {"package": "excel.link", "reason": "windows-only"}, - {"package": "spectrino", "reason": "windows-only"}, - {"package": "taskscheduleR", "reason": "windows-only"}, - {"package": "MDSGUI", "reason": "windows-only"}, - {"package": "BiplotGUI", "reason": "windows-only"}, - {"package": "R2wd", "reason": "windows-only"}, - {"package": "rFUSION", "reason": "windows-only"}, - {"package": "MediaNews", "reason": "windows-only"}, - {"package": "doBy", "reason": "hang"}, - {"package": "IDPmisc", "reason": "hang"}, - {"package": "frailtypack", "reason": "hang"}, - {"package": "afex", "reason": "hang"}, - {"package": "FrF2", "reason": "hang"}, - {"package": "DoE.base", "reason": "hang"}, - {"package": "agricolae", "reason": "hang"}, - {"package": "doFuture", "reason": "hang"}, - {"package": "fscaret", "reason": "hang"}, - {"package": "PHYLOGR", "reason": "hang"}, - {"package": "seewave", "reason": "hang"}, - {"package": "pls", "reason": "hang"}, - {"package": "relaimpo", "reason": "hang"}, - {"package": "geepack", "reason": "hang"}, - {"package": "gggenes", "reason": "hang"}, - {"package": "NPCirc", "reason": "hang"}, - {"package": "repmis", "reason": "hang"}, - {"package": "PNDSIBGE", "reason": "hang"}, - {"package": "lidR", "reason": "hang"}, - {"package": "poismf", "reason": "hang"}, - {"package": "neonstore", "reason": "hang"}, - {"package": "MachineShop", "reason": "hang"}, - {"package": "mvst", "reason": "hang"}, - {"package": "MacBehaviour", "reason": "hang"}, - {"package": "mcmcderive", "reason": "hang"}, - {"package": "RGIFT", "reason": "hang"}, - {"package": "KnowBR", "reason": "hang"}, - {"package": "netmeta", "reason": "hang"}, - {"package": "spdep", "reason": "hang"}, - {"package": "Rfast", "reason": "hang"}, - {"package": "compareGroups", "reason": "hang"}, - {"package": "ff", "reason": "hang"}, - {"package": "GsymPoint", "reason": "hang"}, - {"package": "RcppDynProg", "reason": "hang"}, - {"package": "comtradr", "reason": "hang"}, - {"package": "FD", "reason": "hang"}, - {"package": "PearsonDS", "reason": "hang"}, - {"package": "DCluster", "reason": "hang"}, - {"package": "gRc", "reason": "hang"}, - {"package": "mixlm", "reason": "hang"}, - {"package": "geospt", "reason": "hang"}, - {"package": "fdth", "reason": "hang"}, - {"package": "ffmanova", "reason": "hang"}, - {"package": "fiery", "reason": "hang"}, - {"package": "ffscrapr", "reason": "hang"}, - {"package": "cold", "reason": "hang"}, - {"package": "RcmdrPlugin.DoE", "reason": "hang"}, - {"package": "RcmdrPlugin.NMBU", "reason": "hang"}, - {"package": "RcmdrPlugin.RiskDemo", "reason": "hang"}, - {"package": "RcmdrPlugin.TeachStat", "reason": "hang"}, - {"package": "RcmdrPlugin.TeachingDemos", "reason": "hang"}, - {"package": "RcmdrPlugin.UCA", "reason": "hang"}, - {"package": "RcmdrPlugin.WorldFlora", "reason": "hang"}, - {"package": "RcmdrPlugin.aRnova", "reason": "hang"}, - {"package": "RcmdrPlugin.depthTools", "reason": "hang"}, - {"package": "RcmdrPlugin.orloca", "reason": "hang"}, - {"package": "RcmdrPlugin.sos", "reason": "hang"}, - {"package": "RcmdrPlugin.survival", "reason": "hang"}, - {"package": "RcmdrPlugin.temis", "reason": "hang"}, - {"package": "GWlasso", "reason": "hang"}, - {"package": "GWmodelVis", "reason": "hang"} -] -``` - -- [ ] **Step 2: Commit** - -```bash -git add local/excluded-packages.json -git commit -m "feat: add excluded-packages.json for weekly missing binaries workflow" -``` - ---- - -## Task 2: Create the audit R script - -**Files:** -- Create: `local/weekly-missing-binaries-audit.R` - -This is the core logic. It reads env vars `PLATFORM` and `ARCH`, checks S3 + DB, writes an RDS of rebuildable packages, and updates the Forgejo issue for the OS family. - -- [ ] **Step 1: Create the R script** - -Create `local/weekly-missing-binaries-audit.R` with the full content below: - -```r -options(error = function() { - cat("ERROR:", geterrmessage(), "\n", file = stdout()) - traceback(2) - q(status = 1) -}) - -library(DBI, quietly = TRUE) -suppressPackageStartupMessages(library(data.table)) - -platform <- Sys.getenv("PLATFORM") -arch <- Sys.getenv("ARCH") -stopifnot(nzchar(platform), nzchar(arch)) - -# S3 codename: remove hyphen from platform string (e.g. "ubuntu-2204" -> "ubuntu2204") -s3_codename <- gsub("-", "", platform) - -# OS family for issue grouping -os_family <- if (grepl("^ubuntu", platform)) { - "Ubuntu" -} else if (grepl("^alpine", platform)) { - "Alpine" -} else if (grepl("^redhat", platform)) { - "Red Hat" -} else { - stop("Unknown platform: ", platform) -} - -cat(sprintf("Audit: platform=%s, arch=%s, s3_codename=%s, os_family=%s\n", - platform, arch, s3_codename, os_family)) - -### 1. Get current CRAN release packages -cran_release <- as.data.table(tools::CRAN_package_db()[, c("Package", "Version")]) -cran_release[, Version := as.character(Version)] -cat(sprintf("CRAN release packages: %d\n", nrow(cran_release))) - -### 2. List S3 tarballs for this platform/arch -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE -) - -s3_path <- sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, s3_codename) -s3_files <- tryCatch( - s3fs::s3_dir_ls(s3_path), - error = function(e) { - cat(sprintf("Warning: could not list S3 path %s: %s\n", s3_path, conditionMessage(e))) - character(0) - } -) - -# Parse package name + version from tarball filenames -file_names <- basename(s3_files) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) -parts <- parts[lengths(parts) == 3] - -s3_dt <- if (length(parts) > 0) { - data.table( - Package = vapply(parts, `[`, character(1), 2), - Version = vapply(parts, `[`, character(1), 3) - ) -} else { - data.table(Package = character(0), Version = character(0)) -} -cat(sprintf("S3 packages for %s/%s: %d\n", arch, s3_codename, nrow(s3_dt))) - -### 3. Find CRAN release packages missing from S3 -setkey(cran_release, Package, Version) -setkey(s3_dt, Package, Version) -missing <- cran_release[!s3_dt] -cat(sprintf("Missing binaries (latest CRAN version): %d\n", nrow(missing))) - -### 4. Load excluded packages -excluded_json <- jsonlite::fromJSON("local/excluded-packages.json") -excluded_pkgs <- excluded_json$package -missing_not_excluded <- missing[!Package %in% excluded_pkgs] -missing_excluded <- missing[Package %in% excluded_pkgs] -cat(sprintf("Missing after excluding %d blocked packages: %d\n", - length(excluded_pkgs), nrow(missing_not_excluded))) - -### 5. Check DB for prior build failures on these exact versions -con <- DBI::dbConnect( - RPostgres::Postgres(), - dbname = "build_metadata", - host = "r-binaries.devxy.io", - port = 15432, - user = "rpkgs", - password = Sys.getenv("PGPASS"), - sslmode = "require" -) -on.exit(DBI::dbDisconnect(con), add = TRUE) - -errored_pkgs <- as.data.table(DBI::dbGetQuery( - con, - sprintf( - "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE AND platform = '%s' AND arch = '%s'", - platform, arch - ) -)) -setnames(errored_pkgs, c("Package", "Version")) -setkey(errored_pkgs, Package, Version) - -# Split: rebuildable vs known failures -known_failures <- missing_not_excluded[errored_pkgs, nomatch = 0, on = c("Package", "Version")] -rebuildable <- missing_not_excluded[!errored_pkgs, on = c("Package", "Version")] - -cat(sprintf("Known build failures: %d\n", nrow(known_failures))) -cat(sprintf("Rebuildable (no prior failure): %d\n", nrow(rebuildable))) - -### 6. Write RDS for rebuild workflow -cache_dir <- "/mnt/cache/packages" -if (dir.exists(cache_dir)) { - rds_path <- file.path(cache_dir, sprintf("weekly_rebuild_%s_%s.rds", platform, arch)) - saveRDS(rebuildable$Package, rds_path) - cat(sprintf("Wrote %d packages to %s\n", nrow(rebuildable), rds_path)) -} else { - cat(sprintf("Cache dir %s does not exist, skipping RDS write\n", cache_dir)) -} - -### 7. Update Forgejo issue -forgejo_token <- Sys.getenv("FORGEJO_TOKEN") -if (!nzchar(forgejo_token)) { - cat("FORGEJO_TOKEN not set, skipping issue update\n") -} else { - base_url <- "https://git.devxy.io/api/v1" - repo <- "devxy/build-cran-binaries" - issue_title <- sprintf("Missing package binaries for latest version (%s)", os_family) - - # Helper: make API request - forgejo_get <- function(path, query = list()) { - url <- paste0(base_url, path) - resp <- httr2::request(url) |> - httr2::req_headers(Authorization = paste("token", forgejo_token)) |> - httr2::req_url_query(!!!query) |> - httr2::req_perform() - httr2::resp_body_json(resp) - } - - forgejo_patch <- function(path, body) { - url <- paste0(base_url, path) - httr2::request(url) |> - httr2::req_headers(Authorization = paste("token", forgejo_token)) |> - httr2::req_method("PATCH") |> - httr2::req_body_json(body) |> - httr2::req_perform() - } - - forgejo_post <- function(path, body) { - url <- paste0(base_url, path) - httr2::request(url) |> - httr2::req_headers(Authorization = paste("token", forgejo_token)) |> - httr2::req_body_json(body) |> - httr2::req_perform() - } - - # Build the markdown section for this platform/arch - build_section <- function() { - n_missing <- nrow(rebuildable) + nrow(known_failures) - n_rebuild <- nrow(rebuildable) - header <- sprintf("### %s (%d missing, %d to rebuild)", arch, n_missing, n_rebuild) - - lines <- header - if (nrow(rebuildable) > 0) { - pkg_lines <- sprintf("- %s (%s)", rebuildable$Package, rebuildable$Version) - lines <- c(lines, "", pkg_lines) - } else if (nrow(known_failures) == 0) { - lines <- c(lines, "", "All binaries available.") - } - - if (nrow(known_failures) > 0) { - lines <- c(lines, "", "#### Known build failures", - sprintf("- %s (%s)", known_failures$Package, known_failures$Version)) - } - - paste(lines, collapse = "\n") - } - - # Build excluded packages footer - build_excluded_footer <- function() { - if (nrow(excluded_json) == 0) return("") - items <- sprintf("%s (%s)", excluded_json$package, excluded_json$reason) - paste0("\n---\n\n## Excluded packages\n", paste(items, collapse = ", ")) - } - - new_section <- build_section() - - # Search for existing issue - issues <- forgejo_get( - sprintf("/repos/%s/issues", repo), - query = list(type = "issues", state = "open", q = issue_title, limit = 50) - ) - - # Find exact title match - existing <- Filter(function(i) i$title == issue_title, issues) - - if (length(existing) > 0) { - issue <- existing[[1]] - body <- issue$body - - # Replace or insert the platform section + arch subsection - platform_header <- sprintf("## %s", platform) - arch_header <- sprintf("### %s", arch) - - # Split body into lines for manipulation - body_lines <- strsplit(body, "\n")[[1]] - - # Find the platform section - platform_start <- which(body_lines == platform_header) - - if (length(platform_start) > 0) { - platform_start <- platform_start[1] - # Find end of this platform section (next ## or --- or end) - remaining <- body_lines[(platform_start + 1):length(body_lines)] - platform_end_offset <- which(grepl("^## |^---$", remaining)) - platform_end <- if (length(platform_end_offset) > 0) { - platform_start + platform_end_offset[1] - 1 - } else { - length(body_lines) - } - - # Within platform section, find the arch subsection - section_lines <- body_lines[platform_start:platform_end] - arch_start_offset <- which(grepl(sprintf("^### %s", arch), section_lines)) - - if (length(arch_start_offset) > 0) { - arch_start <- arch_start_offset[1] - # Find end of arch subsection (next ### or ## or --- or end of platform section) - arch_remaining <- section_lines[(arch_start + 1):length(section_lines)] - arch_end_offset <- which(grepl("^###|^## |^---$", arch_remaining)) - arch_end <- if (length(arch_end_offset) > 0) { - arch_start + arch_end_offset[1] - 1 - } else { - length(section_lines) - } - # Replace arch subsection within platform section - section_lines <- c( - section_lines[1:(arch_start - 1)], - strsplit(new_section, "\n")[[1]], - if (arch_end < length(section_lines)) section_lines[(arch_end + 1):length(section_lines)] else character(0) - ) - } else { - # Append arch subsection to end of platform section - section_lines <- c(section_lines, "", strsplit(new_section, "\n")[[1]]) - } - - body_lines <- c( - body_lines[1:(platform_start - 1)], - section_lines, - if (platform_end < length(body_lines)) body_lines[(platform_end + 1):length(body_lines)] else character(0) - ) - } else { - # Insert new platform section before "---" (excluded packages footer) or at end - footer_line <- which(body_lines == "---") - insert_at <- if (length(footer_line) > 0) footer_line[1] - 1 else length(body_lines) - body_lines <- c( - body_lines[1:insert_at], - "", - platform_header, - "", - strsplit(new_section, "\n")[[1]], - if (insert_at < length(body_lines)) body_lines[(insert_at + 1):length(body_lines)] else character(0) - ) - } - - # Update timestamp - timestamp_pattern <- "^_Last updated:.*_$" - ts_line <- which(grepl(timestamp_pattern, body_lines)) - new_ts <- sprintf("_Last updated: %s_", Sys.Date()) - if (length(ts_line) > 0) { - body_lines[ts_line[1]] <- new_ts - } else { - body_lines <- c(new_ts, "", body_lines) - } - - # Rebuild excluded footer - footer_start <- which(body_lines == "---") - if (length(footer_start) > 0) { - body_lines <- c(body_lines[1:(footer_start[1] - 1)], - strsplit(build_excluded_footer(), "\n")[[1]]) - } else { - body_lines <- c(body_lines, strsplit(build_excluded_footer(), "\n")[[1]]) - } - - new_body <- paste(body_lines, collapse = "\n") - forgejo_patch( - sprintf("/repos/%s/issues/%d", repo, issue$number), - list(body = new_body) - ) - cat(sprintf("Updated issue #%d: %s\n", issue$number, issue_title)) - } else { - # Create new issue - body_lines <- c( - sprintf("_Last updated: %s_", Sys.Date()), - "", - sprintf("## %s", platform), - "", - new_section, - build_excluded_footer() - ) - new_body <- paste(body_lines, collapse = "\n") - forgejo_post( - sprintf("/repos/%s/issues", repo), - list(title = issue_title, body = new_body) - ) - cat(sprintf("Created new issue: %s\n", issue_title)) - } -} - -cat("Audit complete.\n") -``` - -- [ ] **Step 2: Commit** - -```bash -git add local/weekly-missing-binaries-audit.R -git commit -m "feat: add weekly missing binaries audit R script" -``` - ---- - -## Task 3: Create audit workflow for ubuntu-2204-amd64 - -**Files:** -- Create: `.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml` - -This is the template. All other audit workflows follow the same structure with substituted values. - -- [ ] **Step 1: Create the workflow YAML** - -Create `.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml`: - -```yaml -when: - - event: cron - cron: weekly-audit-missing-ubuntu-2204-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-ubuntu-2204-amd64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: ubuntu-2204 - ARCH: amd64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - amd64-binaries-r-dep-cache-ubuntu2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' -``` - -- [ ] **Step 2: Commit** - -```bash -git add .crow/weekly-audit-missing-ubuntu-2204-amd64.yaml -git commit -m "feat: add weekly audit workflow for ubuntu-2204-amd64" -``` - ---- - -## Task 4: Create remaining 13 audit workflows - -**Files:** -- Create: 13 files in `.crow/` (see substitution table below) - -Each file follows the exact same structure as Task 3 with these substitutions: - -| File suffix | PLATFORM | ARCH | Image | Volume | -|-------------|----------|------|-------|--------| -| ubuntu-2204-arm64 | ubuntu-2204 | arm64 | reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 | arm64-binaries-r-dep-cache-ubuntu2204 | -| ubuntu-2404-amd64 | ubuntu-2404 | amd64 | reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 | amd64-binaries-r-dep-cache-ubuntu2404 | -| ubuntu-2404-arm64 | ubuntu-2404 | arm64 | reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 | arm64-binaries-r-dep-cache-ubuntu2404 | -| alpine-321-amd64 | alpine-321 | amd64 | reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 | amd64-binaries-r-dep-cache-alpine321 | -| alpine-321-arm64 | alpine-321 | arm64 | reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 | arm64-binaries-r-dep-cache-alpine321 | -| alpine-322-amd64 | alpine-322 | amd64 | reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 | amd64-binaries-r-dep-cache-alpine322 | -| alpine-322-arm64 | alpine-322 | arm64 | reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 | arm64-binaries-r-dep-cache-alpine322 | -| alpine-323-amd64 | alpine-323 | amd64 | reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 | amd64-binaries-r-dep-cache-alpine323 | -| alpine-323-arm64 | alpine-323 | arm64 | reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 | arm64-binaries-r-dep-cache-alpine323 | -| redhat-8-amd64 | redhat-8 | amd64 | reg.devxy.io/rpkgs/build-env-redhat:8-4.4.3 | amd64-binaries-r-dep-cache-redhat8 | -| redhat-8-arm64 | redhat-8 | arm64 | reg.devxy.io/rpkgs/build-env-redhat:8-4.4.3 | arm64-binaries-r-dep-cache-redhat8 | -| redhat-9-amd64 | redhat-9 | amd64 | reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 | amd64-binaries-r-dep-cache-redhat9 | -| redhat-9-arm64 | redhat-9 | arm64 | reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 | arm64-binaries-r-dep-cache-redhat9 | - -**Key substitution points in each YAML (6 locations):** - -1. `cron:` value — `weekly-audit-missing-{suffix}` -2. `evaluate:` value — `'task == "weekly-audit-missing-{suffix}"'` -3. `image:` — use the Image column -4. `PLATFORM:` env var — use the PLATFORM column -5. `ARCH:` env var — use the ARCH column -6. `volumes:` — `{ARCH}-binaries-r-dep-cache-{PLATFORM-without-hyphens}:/mnt/cache` -7. `nodeSelector: kubernetes.io/arch:` — use the ARCH column - -- [ ] **Step 1: Create all 13 audit workflow files** - -Copy the template from Task 3 and substitute the values per the table above. Example for alpine-321-arm64: - -```yaml -when: - - event: cron - cron: weekly-audit-missing-alpine-321-arm64 - - event: manual - evaluate: 'task == "weekly-audit-missing-alpine-321-arm64"' - -skip_clone: true - -steps: - - name: 'Audit missing binaries' - image: reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 - pull: true - environment: - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - FORGEJO_TOKEN: - from_secret: FORGEJO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - PLATFORM: alpine-321 - ARCH: arm64 - R_LIBS_USER: /mnt/cache/R-pkgs - volumes: - - arm64-binaries-r-dep-cache-alpine321:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/packages /mnt/cache/R-pkgs - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak(c("git::https://codefloe.com/rpkgs/bincraft.git", "httr2", "jsonlite"))' - - R -q -e 'source("local/weekly-missing-binaries-audit.R")' - backend_options: - kubernetes: - resources: - requests: - memory: 2Gi - cpu: 2000m - limits: - memory: 4Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' -``` - -Repeat for all 13 remaining suffixes from the table, substituting the 7 locations. - -- [ ] **Step 2: Commit** - -```bash -git add .crow/weekly-audit-missing-*.yaml -git commit -m "feat: add remaining 13 weekly audit workflows for all platform/arch combos" -``` - ---- - -## Task 5: Create rebuild workflow for ubuntu-2204-amd64 - -**Files:** -- Create: `.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml` - -This is the template for rebuild workflows. It reads the RDS written by the audit and builds each package. - -- [ ] **Step 1: Create the workflow YAML** - -Create `.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml`: - -```yaml -when: - - event: cron - cron: weekly-rebuild-missing-ubuntu-2204-amd64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-ubuntu-2204-amd64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: ubuntu-2204 - ARCH: amd64 - NCPUS: 2 - volumes: - - amd64-binaries-r-dep-cache-ubuntu2204:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_ubuntu-2204_amd64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s: %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: AX42 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' -``` - -- [ ] **Step 2: Commit** - -```bash -git add .crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml -git commit -m "feat: add weekly rebuild workflow for ubuntu-2204-amd64" -``` - ---- - -## Task 6: Create remaining 13 rebuild workflows - -**Files:** -- Create: 13 files in `.crow/` (see substitution table below) - -Each file follows the exact same structure as Task 5 with these substitutions: - -| File suffix | PLATFORM | ARCH | Image | Volume | nodeSelector instance-type | -|-------------|----------|------|-------|--------|--------------------------| -| ubuntu-2204-arm64 | ubuntu-2204 | arm64 | reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 | arm64-binaries-r-dep-cache-ubuntu2204 | (omit) | -| ubuntu-2404-amd64 | ubuntu-2404 | amd64 | reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 | amd64-binaries-r-dep-cache-ubuntu2404 | AX42 | -| ubuntu-2404-arm64 | ubuntu-2404 | arm64 | reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 | arm64-binaries-r-dep-cache-ubuntu2404 | (omit) | -| alpine-321-amd64 | alpine-321 | amd64 | reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 | amd64-binaries-r-dep-cache-alpine321 | AX42 | -| alpine-321-arm64 | alpine-321 | arm64 | reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 | arm64-binaries-r-dep-cache-alpine321 | (omit) | -| alpine-322-amd64 | alpine-322 | amd64 | reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 | amd64-binaries-r-dep-cache-alpine322 | AX42 | -| alpine-322-arm64 | alpine-322 | arm64 | reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 | arm64-binaries-r-dep-cache-alpine322 | (omit) | -| alpine-323-amd64 | alpine-323 | amd64 | reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 | amd64-binaries-r-dep-cache-alpine323 | AX42 | -| alpine-323-arm64 | alpine-323 | arm64 | reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 | arm64-binaries-r-dep-cache-alpine323 | (omit) | -| redhat-8-amd64 | redhat-8 | amd64 | reg.devxy.io/rpkgs/build-env-redhat:8-4.4.3 | amd64-binaries-r-dep-cache-redhat8 | AX42 | -| redhat-8-arm64 | redhat-8 | arm64 | reg.devxy.io/rpkgs/build-env-redhat:8-4.4.3 | arm64-binaries-r-dep-cache-redhat8 | (omit) | -| redhat-9-amd64 | redhat-9 | amd64 | reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 | amd64-binaries-r-dep-cache-redhat9 | AX42 | -| redhat-9-arm64 | redhat-9 | arm64 | reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 | arm64-binaries-r-dep-cache-redhat9 | (omit) | - -**Key substitution points (8 locations):** - -1. `cron:` value — `weekly-rebuild-missing-{suffix}` -2. `evaluate:` value — `'task == "weekly-rebuild-missing-{suffix}"'` -3. `image:` — use the Image column -4. `PLATFORM:` env var — use the PLATFORM column -5. `ARCH:` env var — use the ARCH column -6. `volumes:` — `{ARCH}-binaries-r-dep-cache-{PLATFORM-without-hyphens}:/mnt/cache` -7. `rds_path` in the R command — `weekly_rebuild_{PLATFORM}_{ARCH}.rds` -8. `nodeSelector` — `kubernetes.io/arch: {ARCH}`, plus `node.kubernetes.io/instance-type: AX42` only for amd64 (omit the instance-type line entirely for arm64) - -- [ ] **Step 1: Create all 13 rebuild workflow files** - -Copy the template from Task 5 and substitute per the table. Example for redhat-9-arm64: - -```yaml -when: - - event: cron - cron: weekly-rebuild-missing-redhat-9-arm64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-redhat-9-arm64"' - -skip_clone: true - -steps: - - name: 'Rebuild missing binaries' - image: reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 - pull: true - environment: - RED_HAT_DEV_PW: - from_secret: RED_HAT_DEV_PW - B2_S3_ACCESS_KEY: - from_secret: B2_S3_ACCESS_KEY - B2_S3_SECRET_KEY: - from_secret: B2_S3_SECRET_KEY - PGPASS: - from_secret: PGPASS - REPO_RO_TOKEN: - from_secret: REPO_RO_TOKEN - GITHUB_PAT: - from_secret: GITHUB_PAT - GIT_USER: pat-s - R_PKG_CACHE_DIR: /mnt/cache/pkgcache - R_LIBS_USER: /mnt/cache/R-pkgs - CCACHE_DIR: /mnt/cache/ccache - PLATFORM: redhat-9 - ARCH: arm64 - NCPUS: 2 - volumes: - - arm64-binaries-r-dep-cache-redhat9:/mnt/cache - commands: - - git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git . - - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages - - rm -rf /mnt/cache/R-pkgs/00LOCK-* - - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")' - - R -q -e 'packageVersion("bincraft")' - - XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi - - $XVFB $XVFB_ARGS -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); rds_path <- '/mnt/cache/packages/weekly_rebuild_redhat-9_arm64.rds'; if (!file.exists(rds_path)) { cat('No RDS file found at', rds_path, '- nothing to rebuild\n'); q('no') }; pkgs <- readRDS(rds_path); if (length(pkgs) == 0) { cat('RDS is empty - nothing to rebuild\n'); q('no') }; excluded <- jsonlite::fromJSON('local/excluded-packages.json')[['package']]; pkgs <- setdiff(pkgs, excluded); cat(sprintf('Rebuilding %d packages\n', length(pkgs))); n <- length(pkgs); for (i in seq_along(pkgs)) { x <- pkgs[i]; cat(sprintf('[%d/%d] %s\n', i, n, x)); tryCatch(bincraft::build_binary_package(x, tag_limit = 1L, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE), error = function(e) cat(sprintf('ERROR building %s: %s\n', x, conditionMessage(e)))) }" 2>&1 - backend_options: - kubernetes: - resources: - requests: - memory: 5Gi - cpu: 3000m - limits: - memory: 18Gi - cpu: 3000m - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' -``` - -Note: arm64 workflows do NOT include `node.kubernetes.io/instance-type: AX42` in nodeSelector. - -Repeat for all 13 remaining suffixes from the table, substituting the 8 locations. - -- [ ] **Step 2: Commit** - -```bash -git add .crow/weekly-rebuild-missing-*.yaml -git commit -m "feat: add remaining 13 weekly rebuild workflows for all platform/arch combos" -``` - ---- - -## Task 7: Final verification - -- [ ] **Step 1: Verify all files exist** - -Run: -```bash -ls -1 local/excluded-packages.json local/weekly-missing-binaries-audit.R -ls -1 .crow/weekly-audit-missing-*.yaml | wc -l -ls -1 .crow/weekly-rebuild-missing-*.yaml | wc -l -``` - -Expected: both files exist, 14 audit workflows, 14 rebuild workflows. - -- [ ] **Step 2: Validate JSON** - -Run: -```bash -python3 -c "import json; json.load(open('local/excluded-packages.json')); print('JSON valid')" -``` - -Expected: `JSON valid` - -- [ ] **Step 3: Validate YAML syntax** - -Run: -```bash -python3 -c " -import yaml, glob -for f in sorted(glob.glob('.crow/weekly-*-missing-*.yaml') + glob.glob('.crow/weekly-audit-missing-*.yaml') + glob.glob('.crow/weekly-rebuild-missing-*.yaml')): - yaml.safe_load(open(f)) - print(f'OK: {f}') -" -``` - -Expected: all files print `OK`. - -- [ ] **Step 4: Verify unique cron names** - -Run: -```bash -grep -h 'cron: weekly-' .crow/weekly-*.yaml | sort | uniq -c | sort -rn | head -``` - -Expected: all counts are 1 (no duplicates). - -- [ ] **Step 5: Verify all 14 platform/arch combos covered** - -Run: -```bash -for suffix in ubuntu-2204-amd64 ubuntu-2204-arm64 ubuntu-2404-amd64 ubuntu-2404-arm64 alpine-321-amd64 alpine-321-arm64 alpine-322-amd64 alpine-322-arm64 alpine-323-amd64 alpine-323-arm64 redhat-8-amd64 redhat-8-arm64 redhat-9-amd64 redhat-9-arm64; do - test -f ".crow/weekly-audit-missing-${suffix}.yaml" || echo "MISSING audit: ${suffix}" - test -f ".crow/weekly-rebuild-missing-${suffix}.yaml" || echo "MISSING rebuild: ${suffix}" -done -echo "All checks passed if no MISSING lines above" -``` - -Expected: no MISSING lines, just "All checks passed". diff --git a/docs/superpowers/specs/2026-04-11-weekly-missing-binaries-design.md b/docs/superpowers/specs/2026-04-11-weekly-missing-binaries-design.md deleted file mode 100644 index 8ddb71b..0000000 --- a/docs/superpowers/specs/2026-04-11-weekly-missing-binaries-design.md +++ /dev/null @@ -1,246 +0,0 @@ -# Weekly Missing Binaries Audit & Rebuild - -## Goal - -A weekly CI workflow that identifies CRAN packages whose latest release version has no binary available, reports them in Forgejo issues grouped by OS family and arch, and rebuilds those that have no prior build failure recorded in the database. - -## Architecture Overview - -Two independent workflow sets, each with one file per platform/arch combo (14 files each), plus a shared R script and an excluded-packages config file. - -``` -weekly-audit-missing-*-*.yaml (14 files) - │ - ▼ -local/weekly-missing-binaries-audit.R - │ - ├── Updates Forgejo issues (3 issues, one per OS family) - └── Writes RDS files to /mnt/cache/packages/ - -weekly-rebuild-missing-*-*.yaml (14 files) - │ - ▼ - Reads RDS, builds missing packages via bincraft::build_binary_package() -``` - -The audit and build workflows are fully independent. -Either can be triggered on its own via cron or manually. - -## Platforms - -All current platform/arch combinations: - -| Platform | Arch | Image | Codename | -|-------------|-------|-------------------------------------------------|------------| -| ubuntu-2204 | amd64 | reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 | jammy | -| ubuntu-2204 | arm64 | reg.devxy.io/rpkgs/build-env-ubuntu:jammy-4.4.3 | jammy | -| ubuntu-2404 | amd64 | reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 | noble | -| ubuntu-2404 | arm64 | reg.devxy.io/rpkgs/build-env-ubuntu:noble-4.4.3 | noble | -| alpine-321 | amd64 | reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 | alpine321 | -| alpine-321 | arm64 | reg.devxy.io/rpkgs/build-env-alpine:3.21-4.5 | alpine321 | -| alpine-322 | amd64 | reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 | alpine322 | -| alpine-322 | arm64 | reg.devxy.io/rpkgs/build-env-alpine:3.22-4.5 | alpine322 | -| alpine-323 | amd64 | reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 | alpine323 | -| alpine-323 | arm64 | reg.devxy.io/rpkgs/build-env-alpine:3.23-4.5 | alpine323 | -| redhat-8 | amd64 | reg.devxy.io/rpkgs/build-env-redhat:8-4.4.3 | rhel8 | -| redhat-8 | arm64 | reg.devxy.io/rpkgs/build-env-redhat:8-4.4.3 | rhel8 | -| redhat-9 | amd64 | reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 | rhel9 | -| redhat-9 | arm64 | reg.devxy.io/rpkgs/build-env-redhat:9-4.4.3 | rhel9 | - -## Component 1: Excluded Packages Config - -**File:** `local/excluded-packages.json` - -A JSON array of objects with `package` and `reason` fields: - -```json -[ - {"package": "RInno", "reason": "windows-only"}, - {"package": "KeyboardSimulator", "reason": "windows-only"}, - {"package": "doBy", "reason": "hang"}, - {"package": "frailtypack", "reason": "hang"}, - ... -] -``` - -This file is the single source of truth for packages that should be skipped. -Both the audit script and rebuild workflows read from it. - -The existing `build-all-versions-*.yaml` workflows retain their inline lists for now (migration is out of scope). - -## Component 2: Audit R Script - -**File:** `local/weekly-missing-binaries-audit.R` - -**Environment variables consumed:** -- `PLATFORM` — e.g. `ubuntu-2204`, `alpine-321`, `redhat-9` -- `ARCH` — `amd64` or `arm64` -- `B2_S3_ACCESS_KEY`, `B2_S3_SECRET_KEY` — S3 credentials -- `PGPASS` — PostgreSQL password -- `FORGEJO_TOKEN` — API token for issue updates - -The workflow commands (not the R script) also use: -- `REPO_RO_TOKEN` — for `git clone` in the workflow commands - -**Logic:** - -1. Parse `PLATFORM` to derive OS family (`Ubuntu`, `Alpine`, `Red Hat`) and S3 codename (e.g. `ubuntu-2204` -> `jammy`, `redhat-9` -> `rhel9`). -2. Fetch CRAN release packages via `tools::CRAN_package_db()` — extract `Package` and `Version`. -3. List S3 tarballs at `devxy-rpkgs-binaries/{arch}/{codename}/latest/src/contrib/` and parse `{name}_{version}.tar.gz`. -4. Find packages where the CRAN release version is missing from S3. -5. Read `local/excluded-packages.json` and remove those packages from the missing list. -6. Query the `single_builds` DB table: for each missing package+version+platform+arch, check if `error_occurred = TRUE`. Split into: - - **Rebuildable:** missing, not excluded, no prior failure for this version - - **Known failures:** missing, not excluded, but has a recorded failure for this version -7. Write the rebuildable package list (names only) to `/mnt/cache/packages/weekly_rebuild_{platform}_{arch}.rds`. -8. Update the Forgejo issue for this OS family. - -**Issue update logic:** - -- Issue title: `Missing package binaries for latest version ()` where OS family is `Ubuntu`, `Alpine`, or `Red Hat`. -- Search for existing open issue via `GET /api/v1/repos/devxy/build-cran-binaries/issues?type=issues&state=open&q=`. Match by exact title. -- If found, read the existing body, replace the section for this platform/arch, and `PATCH` the issue. -- If not found, `POST` a new issue with just this platform/arch section. - -**Issue body format:** - -```markdown -_Last updated: 2026-04-11_ - -## ubuntu-2204 - -### amd64 (12 missing, 8 to rebuild) -- ggplot2 (3.5.2) -- dplyr (1.1.5) -- ... - -#### Known build failures -- somepkg (1.0.0) - -### arm64 (5 missing, 5 to rebuild) -- ... - -## ubuntu-2404 - -### amd64 (3 missing, 3 to rebuild) -- ... - -### arm64 (0 missing) -All binaries available. - ---- - -## Excluded packages -doBy (hang), frailtypack (hang), RInno (windows-only), ... -``` - -Each audit workflow run updates only its own platform/arch section within the issue. -The "Excluded packages" section and "Last updated" timestamp are rewritten on every run. - -**Section replacement strategy:** -The script parses the existing issue body as markdown, finds the `## {platform}` + `### {arch}` section, replaces it, and writes back the full body. -If the section doesn't exist yet, it's appended under the correct `## {platform}` header (or a new one is created). - -## Component 3: Audit Workflows - -**Files:** 14 files, named `weekly-audit-missing-{platform}-{arch}.yaml` - -Example: `.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml` - -**Trigger:** -```yaml -when: - - event: cron - cron: weekly-audit-missing-ubuntu-2204-amd64 - - event: manual - evaluate: 'task == "weekly-audit-missing-ubuntu-2204-amd64"' -``` - -**Step:** Lightweight — clones repo, installs bincraft + dependencies, runs the audit R script. - -**Container:** Uses the platform-appropriate build image (needed for correct platform identification), but with minimal resource requests since no building happens. - -**Resources:** ~2Gi memory, 2 CPUs. - -## Component 4: Rebuild Workflows - -**Files:** 14 files, named `weekly-rebuild-missing-{platform}-{arch}.yaml` - -Example: `.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml` - -**Trigger:** -```yaml -when: - - event: cron - cron: weekly-rebuild-missing-ubuntu-2204-amd64 - - event: manual - evaluate: 'task == "weekly-rebuild-missing-ubuntu-2204-amd64"' -``` - -**Step:** - -1. Clone repo, install bincraft. -2. Read `local/excluded-packages.json` as a safety net. -3. Read `/mnt/cache/packages/weekly_rebuild_{platform}_{arch}.rds`. If missing or empty, exit 0. -4. Filter out excluded packages (double-check). -5. Iterate and call `bincraft::build_binary_package()` for each package with `tag_limit = 1L`. - Same S3/DB parameters as existing build workflows. -6. Uses `xvfb-run` / `xwfb-run` for graphical packages (same pattern as existing builds). - -**Resources:** Same as `process-updates` workflows — 5Gi request, 18Gi limit, 3 CPUs. - -**Cache volume:** Maps `${ARCH}-binaries-r-dep-cache-${PLATFORM}:/mnt/cache` (same volumes as existing builds, so the RDS files written by audit are visible). - -## Secrets Required - -All existing secrets are reused: -- `B2_S3_ACCESS_KEY`, `B2_S3_SECRET_KEY` — S3 access -- `PGPASS` — PostgreSQL -- `REPO_RO_TOKEN` — Git clone -- `GITHUB_PAT` — For bincraft GitHub mirror access - -New secret needed: -- `FORGEJO_TOKEN` — API token for creating/updating issues on git.devxy.io - -## Cron Schedule - -The audit and rebuild workflows each get their own cron names. -The cron schedule itself is configured in the Crow/Woodpecker server, not in the YAML. -Intended cadence: once per week (e.g. Sunday morning). - -## File Inventory - -| File | Type | Description | -|------|------|-------------| -| `local/excluded-packages.json` | Config | Excluded packages with reasons | -| `local/weekly-missing-binaries-audit.R` | R script | Audit logic, parameterized by env vars | -| `.crow/weekly-audit-missing-ubuntu-2204-amd64.yaml` | Workflow | Audit for ubuntu-2204/amd64 | -| `.crow/weekly-audit-missing-ubuntu-2204-arm64.yaml` | Workflow | Audit for ubuntu-2204/arm64 | -| `.crow/weekly-audit-missing-ubuntu-2404-amd64.yaml` | Workflow | Audit for ubuntu-2404/amd64 | -| `.crow/weekly-audit-missing-ubuntu-2404-arm64.yaml` | Workflow | Audit for ubuntu-2404/arm64 | -| `.crow/weekly-audit-missing-alpine-321-amd64.yaml` | Workflow | Audit for alpine-321/amd64 | -| `.crow/weekly-audit-missing-alpine-321-arm64.yaml` | Workflow | Audit for alpine-321/arm64 | -| `.crow/weekly-audit-missing-alpine-322-amd64.yaml` | Workflow | Audit for alpine-322/amd64 | -| `.crow/weekly-audit-missing-alpine-322-arm64.yaml` | Workflow | Audit for alpine-322/arm64 | -| `.crow/weekly-audit-missing-alpine-323-amd64.yaml` | Workflow | Audit for alpine-323/amd64 | -| `.crow/weekly-audit-missing-alpine-323-arm64.yaml` | Workflow | Audit for alpine-323/arm64 | -| `.crow/weekly-audit-missing-redhat-8-amd64.yaml` | Workflow | Audit for redhat-8/amd64 | -| `.crow/weekly-audit-missing-redhat-8-arm64.yaml` | Workflow | Audit for redhat-8/arm64 | -| `.crow/weekly-audit-missing-redhat-9-amd64.yaml` | Workflow | Audit for redhat-9/amd64 | -| `.crow/weekly-audit-missing-redhat-9-arm64.yaml` | Workflow | Audit for redhat-9/arm64 | -| `.crow/weekly-rebuild-missing-ubuntu-2204-amd64.yaml` | Workflow | Rebuild for ubuntu-2204/amd64 | -| `.crow/weekly-rebuild-missing-ubuntu-2204-arm64.yaml` | Workflow | Rebuild for ubuntu-2204/arm64 | -| `.crow/weekly-rebuild-missing-ubuntu-2404-amd64.yaml` | Workflow | Rebuild for ubuntu-2404/amd64 | -| `.crow/weekly-rebuild-missing-ubuntu-2404-arm64.yaml` | Workflow | Rebuild for ubuntu-2404/arm64 | -| `.crow/weekly-rebuild-missing-alpine-321-amd64.yaml` | Workflow | Rebuild for alpine-321/amd64 | -| `.crow/weekly-rebuild-missing-alpine-321-arm64.yaml` | Workflow | Rebuild for alpine-321/arm64 | -| `.crow/weekly-rebuild-missing-alpine-322-amd64.yaml` | Workflow | Rebuild for alpine-322/amd64 | -| `.crow/weekly-rebuild-missing-alpine-322-arm64.yaml` | Workflow | Rebuild for alpine-322/arm64 | -| `.crow/weekly-rebuild-missing-alpine-323-amd64.yaml` | Workflow | Rebuild for alpine-323/amd64 | -| `.crow/weekly-rebuild-missing-alpine-323-arm64.yaml` | Workflow | Rebuild for alpine-323/arm64 | -| `.crow/weekly-rebuild-missing-redhat-8-amd64.yaml` | Workflow | Rebuild for redhat-8/amd64 | -| `.crow/weekly-rebuild-missing-redhat-8-arm64.yaml` | Workflow | Rebuild for redhat-8/arm64 | -| `.crow/weekly-rebuild-missing-redhat-9-amd64.yaml` | Workflow | Rebuild for redhat-9/amd64 | -| `.crow/weekly-rebuild-missing-redhat-9-arm64.yaml` | Workflow | Rebuild for redhat-9/arm64 | - -**Total: 29 new files** (1 JSON config + 1 R script + 14 audit workflows + 14 rebuild workflows + the design doc itself) diff --git a/local/excluded-packages.json b/local/excluded-packages.json deleted file mode 100644 index 4bfa2de..0000000 --- a/local/excluded-packages.json +++ /dev/null @@ -1,76 +0,0 @@ -[ - {"package": "RInno", "reason": "windows-only"}, - {"package": "KeyboardSimulator", "reason": "windows-only"}, - {"package": "R2PPT", "reason": "windows-only"}, - {"package": "RWinEdt", "reason": "windows-only"}, - {"package": "blatr", "reason": "windows-only"}, - {"package": "excel.link", "reason": "windows-only"}, - {"package": "spectrino", "reason": "windows-only"}, - {"package": "taskscheduleR", "reason": "windows-only"}, - {"package": "MDSGUI", "reason": "windows-only"}, - {"package": "BiplotGUI", "reason": "windows-only"}, - {"package": "R2wd", "reason": "windows-only"}, - {"package": "rFUSION", "reason": "windows-only"}, - {"package": "MediaNews", "reason": "windows-only"}, - {"package": "doBy", "reason": "hang"}, - {"package": "IDPmisc", "reason": "hang"}, - {"package": "frailtypack", "reason": "hang"}, - {"package": "afex", "reason": "hang"}, - {"package": "FrF2", "reason": "hang"}, - {"package": "DoE.base", "reason": "hang"}, - {"package": "agricolae", "reason": "hang"}, - {"package": "doFuture", "reason": "hang"}, - {"package": "fscaret", "reason": "hang"}, - {"package": "PHYLOGR", "reason": "hang"}, - {"package": "seewave", "reason": "hang"}, - {"package": "pls", "reason": "hang"}, - {"package": "relaimpo", "reason": "hang"}, - {"package": "geepack", "reason": "hang"}, - {"package": "gggenes", "reason": "hang"}, - {"package": "NPCirc", "reason": "hang"}, - {"package": "repmis", "reason": "hang"}, - {"package": "PNDSIBGE", "reason": "hang"}, - {"package": "lidR", "reason": "hang"}, - {"package": "poismf", "reason": "hang"}, - {"package": "neonstore", "reason": "hang"}, - {"package": "MachineShop", "reason": "hang"}, - {"package": "mvst", "reason": "hang"}, - {"package": "MacBehaviour", "reason": "hang"}, - {"package": "mcmcderive", "reason": "hang"}, - {"package": "RGIFT", "reason": "hang"}, - {"package": "KnowBR", "reason": "hang"}, - {"package": "netmeta", "reason": "hang"}, - {"package": "spdep", "reason": "hang"}, - {"package": "Rfast", "reason": "hang"}, - {"package": "compareGroups", "reason": "hang"}, - {"package": "ff", "reason": "hang"}, - {"package": "GsymPoint", "reason": "hang"}, - {"package": "RcppDynProg", "reason": "hang"}, - {"package": "comtradr", "reason": "hang"}, - {"package": "FD", "reason": "hang"}, - {"package": "PearsonDS", "reason": "hang"}, - {"package": "DCluster", "reason": "hang"}, - {"package": "gRc", "reason": "hang"}, - {"package": "mixlm", "reason": "hang"}, - {"package": "geospt", "reason": "hang"}, - {"package": "fdth", "reason": "hang"}, - {"package": "ffmanova", "reason": "hang"}, - {"package": "fiery", "reason": "hang"}, - {"package": "ffscrapr", "reason": "hang"}, - {"package": "cold", "reason": "hang"}, - {"package": "RcmdrPlugin.DoE", "reason": "hang"}, - {"package": "RcmdrPlugin.NMBU", "reason": "hang"}, - {"package": "RcmdrPlugin.RiskDemo", "reason": "hang"}, - {"package": "RcmdrPlugin.TeachStat", "reason": "hang"}, - {"package": "RcmdrPlugin.TeachingDemos", "reason": "hang"}, - {"package": "RcmdrPlugin.UCA", "reason": "hang"}, - {"package": "RcmdrPlugin.WorldFlora", "reason": "hang"}, - {"package": "RcmdrPlugin.aRnova", "reason": "hang"}, - {"package": "RcmdrPlugin.depthTools", "reason": "hang"}, - {"package": "RcmdrPlugin.orloca", "reason": "hang"}, - {"package": "RcmdrPlugin.sos", "reason": "hang"}, - {"package": "RcmdrPlugin.survival", "reason": "hang"}, - {"package": "RcmdrPlugin.temis", "reason": "hang"}, - {"package": "GWlasso", "reason": "hang"}, - {"package": "GWmodelVis", "reason": "hang"} -] diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 9d19b0b..3a2135e 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -1,9 +1,3 @@ -options(error = function() { - cat("ERROR:", geterrmessage(), "\n", file = stdout()) - traceback(2) - q(status = 1) -}) - library(bincraft, quietly = TRUE) suppressPackageStartupMessages(library(dplyr)) library(DBI, quietly = TRUE) @@ -16,8 +10,10 @@ suppressPackageStartupMessages(library(data.table)) # Sys.setenv("ARCH" = "arm64") arch = Sys.getenv("ARCH") -# target: alpine-322, ubuntu-2404, redhat-9, etc. -platform = paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-") +if (Sys.getenv("OS") == "alpine") { + # target: alpine-322 + platform = paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-") +} con <- DBI::dbConnect( RPostgres::Postgres(), @@ -69,14 +65,14 @@ setorder(pkgs_to_build, Package, Version) ### Get all packages in S3 s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", + aws_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), + aws_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"), + endpoint = "https://hel1.your-objectstorage.com", + region_name = "hel1", refresh = TRUE ) s3_pkgs = s3fs::s3_dir_ls( - sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, sub("-", "", platform)), + sprintf("devxy-r-package-binaries-hel1/%s/%s/latest/src/contrib", arch, sub("-", "", platform)), recurse = TRUE ) diff --git a/local/weekly-missing-binaries-audit.R b/local/weekly-missing-binaries-audit.R deleted file mode 100644 index 9c2b4ae..0000000 --- a/local/weekly-missing-binaries-audit.R +++ /dev/null @@ -1,351 +0,0 @@ -options(error = function() { - cat("ERROR:", geterrmessage(), "\n", file = stdout()) - traceback(2) - q(status = 1) -}) - -suppressPackageStartupMessages(library(data.table)) -library(DBI, quietly = TRUE) -library(RPostgres, quietly = TRUE) -library(s3fs, quietly = TRUE) -library(jsonlite, quietly = TRUE) -library(httr2, quietly = TRUE) - -# --------------------------------------------------------------------------- -# Environment / config -# --------------------------------------------------------------------------- -platform <- Sys.getenv("PLATFORM") -arch <- Sys.getenv("ARCH") - -if (nchar(platform) == 0) stop("PLATFORM env var is not set") -if (nchar(arch) == 0) stop("ARCH env var is not set") - -# e.g. "ubuntu-2204" -> "ubuntu2204" -s3_codename <- gsub("-", "", platform) - -# Derive OS family for the Forgejo issue title -os_family <- if (grepl("^ubuntu", platform)) { - "Ubuntu" -} else if (grepl("^alpine", platform)) { - "Alpine" -} else if (grepl("^redhat", platform)) { - "Red Hat" -} else { - platform -} - -cat(sprintf("Platform: %s | Arch: %s | S3 codename: %s | OS family: %s\n", - platform, arch, s3_codename, os_family)) - -# --------------------------------------------------------------------------- -# 1. CRAN release packages -# --------------------------------------------------------------------------- -cat("Fetching CRAN package database...\n") -cran_release <- tools::CRAN_package_db() -cran_dt <- data.table( - Package = as.character(cran_release$Package), - Version = as.character(cran_release$Version) -) - -# --------------------------------------------------------------------------- -# 2. S3 tarballs -# --------------------------------------------------------------------------- -cat("Connecting to S3...\n") -s3fs::s3_file_system( - aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - endpoint = "https://s3.eu-central-003.backblazeb2.com", - region_name = "eu-central-003", - refresh = TRUE -) - -s3_path <- sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, s3_codename) -cat(sprintf("Listing S3 path: %s\n", s3_path)) - -s3_pkgs <- tryCatch( - s3fs::s3_dir_ls(s3_path, recurse = FALSE), - error = function(e) { - cat(sprintf("WARNING: Could not list S3 path %s: %s\n", s3_path, conditionMessage(e))) - character(0) - } -) - -file_names <- basename(s3_pkgs) -matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names) -parts <- regmatches(file_names, matches) -parts <- parts[sapply(parts, length) == 3] -s3_dt <- data.table( - Package = sapply(parts, `[`, 2), - Version = sapply(parts, `[`, 3) -) - -cat(sprintf("S3 contains %d tarballs for %s/%s\n", nrow(s3_dt), arch, s3_codename)) - -# --------------------------------------------------------------------------- -# 3. Find missing packages (CRAN release version not in S3) -# --------------------------------------------------------------------------- -setkey(cran_dt, Package, Version) -setkey(s3_dt, Package, Version) -missing_dt <- cran_dt[!s3_dt] -cat(sprintf("%d CRAN release packages missing from S3\n", nrow(missing_dt))) - -# --------------------------------------------------------------------------- -# 4. Remove excluded packages -# --------------------------------------------------------------------------- -script_dir <- tryCatch( - dirname(normalizePath( - if (exists("ofile", envir = sys.frame(1), inherits = FALSE)) sys.frame(1)$ofile - else commandArgs(trailingOnly = FALSE)[grepl("--file=", commandArgs(trailingOnly = FALSE))] |> - sub("--file=", "", x = _), - mustWork = FALSE - )), - error = function(e) "local" -) -excluded_path <- file.path(script_dir, "excluded-packages.json") - -excluded_pkgs <- data.table(package = character(0), reason = character(0)) -if (file.exists(excluded_path)) { - excluded_raw <- jsonlite::fromJSON(excluded_path) - excluded_pkgs <- as.data.table(excluded_raw) - cat(sprintf("Loaded %d excluded packages from %s\n", nrow(excluded_pkgs), excluded_path)) - missing_dt <- missing_dt[!Package %in% excluded_pkgs$package] - cat(sprintf("%d packages remain after removing exclusions\n", nrow(missing_dt))) -} else { - cat(sprintf("No excluded-packages.json found at %s -- skipping exclusion step\n", excluded_path)) -} - -# --------------------------------------------------------------------------- -# 5. Query PostgreSQL for known build failures -# --------------------------------------------------------------------------- -cat("Connecting to PostgreSQL...\n") -con <- DBI::dbConnect( - RPostgres::Postgres(), - dbname = "build_metadata", - host = "r-binaries.devxy.io", - port = 15432, - user = "rpkgs", - password = Sys.getenv("PGPASS"), - sslmode = "require" -) -on.exit(DBI::dbDisconnect(con), add = TRUE) - -errored_pkgs <- DBI::dbGetQuery( - con, - sprintf( - "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE AND platform = '%s' AND arch = '%s'", - platform, arch - ) -) -errored_dt <- as.data.table(errored_pkgs) -if (nrow(errored_dt) > 0) { - setnames(errored_dt, c("Package", "Version")) - setkey(errored_dt, Package, Version) -} - -cat(sprintf("Found %d known build failures for %s/%s\n", nrow(errored_dt), platform, arch)) - -# --------------------------------------------------------------------------- -# 6. Split into rebuildable vs known failures -# --------------------------------------------------------------------------- -if (nrow(errored_dt) > 0) { - known_failures_dt <- missing_dt[errored_dt, nomatch = 0] - rebuildable_dt <- missing_dt[!errored_dt] -} else { - known_failures_dt <- missing_dt[0] - rebuildable_dt <- missing_dt -} - -cat(sprintf("Rebuildable: %d | Known failures: %d\n", - nrow(rebuildable_dt), nrow(known_failures_dt))) - -# --------------------------------------------------------------------------- -# 7. Write rebuildable package names to cache RDS -# --------------------------------------------------------------------------- -cache_dir <- "/mnt/cache/packages" -cache_file <- file.path(cache_dir, sprintf("weekly_rebuild_%s_%s.rds", platform, arch)) - -if (dir.exists(cache_dir)) { - saveRDS(rebuildable_dt$Package, cache_file) - cat(sprintf("Wrote %d rebuildable packages to %s\n", nrow(rebuildable_dt), cache_file)) -} else { - cat(sprintf("Cache dir %s does not exist -- skipping RDS write\n", cache_dir)) -} - -# --------------------------------------------------------------------------- -# 8. Update Forgejo issue -# --------------------------------------------------------------------------- -forgejo_token <- Sys.getenv("FORGEJO_TOKEN") -if (nchar(forgejo_token) == 0) { - cat("FORGEJO_TOKEN not set -- skipping issue update\n") -} else { - issue_title <- sprintf("Missing package binaries for latest version (%s)", os_family) - - n_missing <- nrow(missing_dt) - n_rebuild <- nrow(rebuildable_dt) - - arch_lines <- sprintf("### %s (%d missing, %d to rebuild)", arch, n_missing, n_rebuild) - if (nrow(rebuildable_dt) > 0) { - arch_lines <- c(arch_lines, - paste0("- ", rebuildable_dt$Package, " (", rebuildable_dt$Version, ")") - ) - } else { - arch_lines <- c(arch_lines, "_None_") - } - if (nrow(known_failures_dt) > 0) { - arch_lines <- c(arch_lines, - "", - "#### Known build failures", - paste0("- ", known_failures_dt$Package, " (", known_failures_dt$Version, ")") - ) - } - - build_excluded_footer <- function() { - if (nrow(excluded_pkgs) == 0) return(character(0)) - entries <- paste( - paste0(excluded_pkgs$package, " (", excluded_pkgs$reason, ")"), - collapse = ", " - ) - c("---", "", "## Excluded packages", entries) - } - - forgejo_base <- "https://git.devxy.io/api/v1" - repo <- "devxy/build-cran-binaries" - - search_url <- sprintf( - "%s/repos/%s/issues?type=issues&state=open&q=%s&limit=50", - forgejo_base, repo, - utils::URLencode(issue_title, reserved = TRUE) - ) - search_resp <- httr2::request(search_url) |> - httr2::req_headers(Authorization = paste("token", forgejo_token)) |> - httr2::req_perform() - - existing_issues <- httr2::resp_body_json(search_resp, simplifyVector = FALSE) - - match_idx <- which(sapply(existing_issues, function(x) x$title) == issue_title) - - today_str <- format(Sys.Date(), "%Y-%m-%d") - - if (length(match_idx) > 0) { - # ---- Update existing issue ---- - issue_number <- existing_issues[[match_idx[1]]]$number - old_body <- existing_issues[[match_idx[1]]]$body - if (is.null(old_body)) old_body <- "" - - lines <- strsplit(old_body, "\n", fixed = TRUE)[[1]] - - # Update timestamp - ts_idx <- which(grepl("^_Last updated:", lines)) - if (length(ts_idx) > 0) { - lines[ts_idx[1]] <- sprintf("_Last updated: %s_", today_str) - } else { - lines <- c(sprintf("_Last updated: %s_", today_str), "", lines) - } - - plat_header <- sprintf("## %s", platform) - arch_header <- sprintf("### %s", arch) - - plat_idx <- which(lines == plat_header) - - if (length(plat_idx) == 0) { - # Platform section missing -- insert before --- footer - footer_idx <- which(lines == "---") - insert_at <- if (length(footer_idx) > 0) footer_idx[length(footer_idx)] else length(lines) + 1 - - new_plat_block <- c(sprintf("## %s", platform), "", arch_lines, "") - lines <- c( - lines[seq_len(insert_at - 1)], - new_plat_block, - lines[seq(insert_at, length(lines))] - ) - } else { - pi <- plat_idx[1] - - # End of platform section: next ## or --- at a higher level, or EOF - next_section <- which(grepl("^## |^---", lines) & seq_along(lines) > pi) - plat_end <- if (length(next_section) > 0) next_section[1] - 1 else length(lines) - - plat_lines <- lines[seq(pi, plat_end)] - arch_local_idx <- which(plat_lines == arch_header) - - if (length(arch_local_idx) == 0) { - # Append arch subsection at end of platform block - lines <- c( - lines[seq_len(plat_end)], - "", - arch_lines, - lines[seq(plat_end + 1, length(lines))] - ) - } else { - ai <- pi + arch_local_idx[1] - 1 # absolute line index - - # End of arch subsection - next_arch <- which(grepl("^### |^## |^---", lines) & seq_along(lines) > ai) - arch_end <- if (length(next_arch) > 0) next_arch[1] - 1 else plat_end - - lines <- c( - lines[seq_len(ai - 1)], - arch_lines, - lines[seq(arch_end + 1, length(lines))] - ) - } - } - - # Rebuild excluded footer - excl_hdr_idx <- which(lines == "## Excluded packages") - if (length(excl_hdr_idx) > 0) { - pre_dash <- which(lines == "---" & seq_along(lines) < excl_hdr_idx[1]) - remove_from <- if (length(pre_dash) > 0) pre_dash[length(pre_dash)] else excl_hdr_idx[1] - lines <- lines[seq_len(remove_from - 1)] - } - footer <- build_excluded_footer() - if (length(footer) > 0) { - lines <- c(lines, "", footer) - } - - new_body <- paste(lines, collapse = "\n") - - patch_url <- sprintf("%s/repos/%s/issues/%d", forgejo_base, repo, issue_number) - httr2::request(patch_url) |> - httr2::req_headers( - Authorization = paste("token", forgejo_token), - `Content-Type` = "application/json" - ) |> - httr2::req_body_json(list(body = new_body)) |> - httr2::req_method("PATCH") |> - httr2::req_perform() - - cat(sprintf("Updated Forgejo issue #%d: %s\n", issue_number, issue_title)) - - } else { - # ---- Create new issue ---- - body_lines <- c( - sprintf("_Last updated: %s_", today_str), - "", - sprintf("## %s", platform), - "", - arch_lines - ) - footer <- build_excluded_footer() - if (length(footer) > 0) { - body_lines <- c(body_lines, "", footer) - } - - post_url <- sprintf("%s/repos/%s/issues", forgejo_base, repo) - create_resp <- httr2::request(post_url) |> - httr2::req_headers( - Authorization = paste("token", forgejo_token), - `Content-Type` = "application/json" - ) |> - httr2::req_body_json(list( - title = issue_title, - body = paste(body_lines, collapse = "\n") - )) |> - httr2::req_perform() - - new_issue <- httr2::resp_body_json(create_resp) - cat(sprintf("Created Forgejo issue #%d: %s\n", new_issue$number, issue_title)) - } -} - -cat("Done.\n") diff --git a/provider.tf b/provider.tf index 0a9ba15..ef48532 100644 --- a/provider.tf +++ b/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { bunnynet = { source = "registry.terraform.io/BunnyWay/bunnynet" - version = "~> 0.14" + version = "~> 0.13" } } } diff --git a/vars.tf b/vars.tf index aad56ff..8168262 100644 --- a/vars.tf +++ b/vars.tf @@ -7,13 +7,3 @@ variable "HETZNER_S3_SECRET_KEY_K3S" { type = string sensitive = true } - -variable "B2_S3_ACCESS_KEY" { - type = string - sensitive = true -} - -variable "B2_S3_SECRET_KEY" { - type = string - sensitive = true -}