diff --git a/.crow/alpine-321-amd64-install-deps.yaml b/.crow/alpine-322-amd64-install-deps.yaml similarity index 91% rename from .crow/alpine-321-amd64-install-deps.yaml rename to .crow/alpine-322-amd64-install-deps.yaml index 6ece9ee..514bf0b 100644 --- a/.crow/alpine-321-amd64-install-deps.yaml +++ b/.crow/alpine-322-amd64-install-deps.yaml @@ -2,7 +2,7 @@ when: - event: manual evaluate: 'build == "all"' - event: manual - evaluate: 'build == "alpine-321-amd64"' + evaluate: 'build == "alpine-322-amd64"' - event: manual evaluate: 'build == "all-amd64"' @@ -10,7 +10,7 @@ skip_clone: true steps: - name: 'Install common R deps' - image: docker.io/devxygmbh/amd64-binaries-r-alpine-321 + image: docker.io/devxygmbh/amd64-binaries-r-alpine-322 pull: true environment: GIT_RO_TOKEN: @@ -30,9 +30,9 @@ steps: R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache - PLATFORM: alpine-321 + PLATFORM: alpine-322 # volumes: - # - amd64-binaries-r-dep-cache-alpine-321:/mnt/cache + # - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache commands: - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/bincraftR /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache diff --git a/.crow/build-alpine-322-amd64-1.yaml b/.crow/build-alpine-322-amd64-1.yaml new file mode 100644 index 0000000..b1de59b --- /dev/null +++ b/.crow/build-alpine-322-amd64-1.yaml @@ -0,0 +1,85 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "alpine-322-amd64"' + - event: manual + evaluate: 'build == "alpine-322-amd64-1"' + - event: manual + evaluate: 'build == "all-amd64"' + +skip_clone: true + +depends_on: + - alpine-322-amd64-install-deps + +steps: + - name: 'Build binaries' + image: docker.io/devxygmbh/amd64-binaries-r-alpine-322 + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S + PGPASS: + from_secret: PGPASS + GIT_RO_TOKEN: + from_secret: GIT_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: alpine-322 + BLOCK_START: 1 + BLOCK_END: 2200 + ARCH: amd64 + NCPUS: 2 + STRATEGY: sequential + commands: + - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . + - R -q -e 'packageVersion("bincraftR")' + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); if (file.exists(sprintf('/mnt/cache/%s.txt', Sys.getenv('CI_WORKFLOW_NAME')))) block_start = as.character(data.table::fread(sprintf('/mnt/cache/%s.txt', Sys.getenv('CI_WORKFLOW_NAME')), sep = '\t')) else block_start = Sys.getenv('BLOCK_START'); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[block_start:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" + backend_options: + kubernetes: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: AX42 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: binwiederhier/ntfy:v2.11.0 + when: + - status: [failure] + environment: + NTFY_TOKEN: + from_secret: ntfy_token + commands: | + BUILD_INDEX=$(cat build_index.txt) + ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-amd64-2.yaml b/.crow/build-alpine-322-amd64-2.yaml new file mode 100644 index 0000000..4bc0cfb --- /dev/null +++ b/.crow/build-alpine-322-amd64-2.yaml @@ -0,0 +1,85 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "alpine-322-amd64"' + - event: manual + evaluate: 'build == "alpine-322-amd64-1"' + - event: manual + evaluate: 'build == "all-amd64"' + +skip_clone: true + +depends_on: + - alpine-322-amd64-install-deps + +steps: + - name: 'Build binaries' + image: docker.io/devxygmbh/amd64-binaries-r-alpine-322 + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S + PGPASS: + from_secret: PGPASS + GIT_RO_TOKEN: + from_secret: GIT_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: alpine-322 + BLOCK_START: 2200 + BLOCK_END: 4400 + ARCH: amd64 + NCPUS: 2 + STRATEGY: sequential + commands: + - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . + - R -q -e 'packageVersion("bincraftR")' + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); if (file.exists(sprintf('/mnt/cache/%s.txt', Sys.getenv('CI_WORKFLOW_NAME')))) block_start = as.character(data.table::fread(sprintf('/mnt/cache/%s.txt', Sys.getenv('CI_WORKFLOW_NAME')), sep = '\t')) else block_start = Sys.getenv('BLOCK_START'); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[block_start:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" + backend_options: + kubernetes: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: AX42 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: binwiederhier/ntfy:v2.11.0 + when: + - status: [failure] + environment: + NTFY_TOKEN: + from_secret: ntfy_token + commands: | + BUILD_INDEX=$(cat build_index.txt) + ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-amd64-3.yaml b/.crow/build-alpine-322-amd64-3.yaml new file mode 100644 index 0000000..2f51b6c --- /dev/null +++ b/.crow/build-alpine-322-amd64-3.yaml @@ -0,0 +1,85 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "alpine-322-amd64"' + - event: manual + evaluate: 'build == "alpine-322-amd64-1"' + - event: manual + evaluate: 'build == "all-amd64"' + +skip_clone: true + +depends_on: + - alpine-322-amd64-install-deps + +steps: + - name: 'Build binaries' + image: docker.io/devxygmbh/amd64-binaries-r-alpine-322 + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S + PGPASS: + from_secret: PGPASS + GIT_RO_TOKEN: + from_secret: GIT_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: alpine-322 + BLOCK_START: 4401 + BLOCK_END: 6600 + ARCH: amd64 + NCPUS: 2 + STRATEGY: sequential + commands: + - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . + - R -q -e 'packageVersion("bincraftR")' + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); if (file.exists(sprintf('/mnt/cache/%s.txt', Sys.getenv('CI_WORKFLOW_NAME')))) block_start = as.character(data.table::fread(sprintf('/mnt/cache/%s.txt', Sys.getenv('CI_WORKFLOW_NAME')), sep = '\t')) else block_start = Sys.getenv('BLOCK_START'); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[block_start:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" + backend_options: + kubernetes: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: AX42 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: binwiederhier/ntfy:v2.11.0 + when: + - status: [failure] + environment: + NTFY_TOKEN: + from_secret: ntfy_token + commands: | + BUILD_INDEX=$(cat build_index.txt) + ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-amd64-4.yaml b/.crow/build-alpine-322-amd64-4.yaml new file mode 100644 index 0000000..f012c20 --- /dev/null +++ b/.crow/build-alpine-322-amd64-4.yaml @@ -0,0 +1,85 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "alpine-322-amd64"' + - event: manual + evaluate: 'build == "alpine-322-amd64-1"' + - event: manual + evaluate: 'build == "all-amd64"' + +skip_clone: true + +depends_on: + - alpine-322-amd64-install-deps + +steps: + - name: 'Build binaries' + image: docker.io/devxygmbh/amd64-binaries-r-alpine-322 + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S + PGPASS: + from_secret: PGPASS + GIT_RO_TOKEN: + from_secret: GIT_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: alpine-322 + BLOCK_START: 6601 + BLOCK_END: 8800 + ARCH: amd64 + NCPUS: 2 + STRATEGY: sequential + commands: + - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . + - R -q -e 'packageVersion("bincraftR")' + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); if (file.exists(sprintf('/mnt/cache/%s.txt', Sys.getenv('CI_WORKFLOW_NAME')))) block_start = as.character(data.table::fread(sprintf('/mnt/cache/%s.txt', Sys.getenv('CI_WORKFLOW_NAME')), sep = '\t')) else block_start = Sys.getenv('BLOCK_START'); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[block_start:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" + backend_options: + kubernetes: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: AX42 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: binwiederhier/ntfy:v2.11.0 + when: + - status: [failure] + environment: + NTFY_TOKEN: + from_secret: ntfy_token + commands: | + BUILD_INDEX=$(cat build_index.txt) + ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/build-alpine-322-amd64-5.yaml b/.crow/build-alpine-322-amd64-5.yaml new file mode 100644 index 0000000..b5cc45f --- /dev/null +++ b/.crow/build-alpine-322-amd64-5.yaml @@ -0,0 +1,85 @@ +when: + - event: manual + evaluate: 'build == "all"' + - event: manual + evaluate: 'build == "alpine-322-amd64"' + - event: manual + evaluate: 'build == "alpine-322-amd64-1"' + - event: manual + evaluate: 'build == "all-amd64"' + +skip_clone: true + +depends_on: + - alpine-322-amd64-install-deps + +steps: + - name: 'Build binaries' + image: docker.io/devxygmbh/amd64-binaries-r-alpine-322 + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S + PGPASS: + from_secret: PGPASS + GIT_RO_TOKEN: + from_secret: GIT_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: alpine-322 + BLOCK_START: 8801 + BLOCK_END: 10000 + ARCH: amd64 + NCPUS: 2 + STRATEGY: sequential + commands: + - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . + - R -q -e 'packageVersion("bincraftR")' + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # Windows-only: 'RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews', 'rFUSION', 'MediaNews' + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pak::sysreqs_db_update(); if (file.exists(sprintf('/mnt/cache/%s.txt', Sys.getenv('CI_WORKFLOW_NAME')))) block_start = as.character(data.table::fread(sprintf('/mnt/cache/%s.txt', Sys.getenv('CI_WORKFLOW_NAME')), sep = '\t')) else block_start = Sys.getenv('BLOCK_START'); pkgs = as.character(available.packages('cloud.r-project.org/src/contrib')[, 'Package'])[block_start:$BLOCK_END]; pkgs = setdiff(pkgs, c('RInno', 'KeyboardSimulator', 'R2PPT', 'RWinEdt', 'blatr', 'excel.link', 'spectrino', 'taskscheduleR', 'MDSGUI', 'BiplotGUI', 'R2wd', 'rFUSION', 'MediaNews')); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs, function(x) bincraftR::build_binary_package(x, debug = FALSE, force = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE, platform = 'alpine-322'))" + backend_options: + kubernetes: + resources: + requests: + memory: 5Gi + cpu: 1000m + limits: + memory: 20Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: AX42 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: binwiederhier/ntfy:v2.11.0 + when: + - status: [failure] + environment: + NTFY_TOKEN: + from_secret: ntfy_token + commands: | + BUILD_INDEX=$(cat build_index.txt) + ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed at $BUILD_INDEX" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/images.yaml b/.crow/images.yaml index 9cb235f..6e2bf9a 100644 --- a/.crow/images.yaml +++ b/.crow/images.yaml @@ -20,13 +20,13 @@ matrix: ARCH: arm64 - PLATFORM: ubuntu-2204 ARCH: amd64 - - PLATFORM: alpine-320 + - PLATFORM: alpine-321 ARCH: arm64 - - PLATFORM: alpine-320 + - PLATFORM: alpine-321 ARCH: amd64 - - PLATFORM: alpine-321 + - PLATFORM: alpine-322 ARCH: arm64 - - PLATFORM: alpine-321 + - PLATFORM: alpine-322 ARCH: amd64 steps: diff --git a/.crow/process-updates-alpine-322-amd64.yaml b/.crow/process-updates-alpine-322-amd64.yaml new file mode 100644 index 0000000..621b926 --- /dev/null +++ b/.crow/process-updates-alpine-322-amd64.yaml @@ -0,0 +1,84 @@ +when: + - event: [cron] + cron: process-cran-updates-alpine-322-amd64 + - event: manual + evaluate: 'process_cran_updates == "all"' + - event: manual + evaluate: 'process_cran_updates == "alpine-322-amd64"' + +skip_clone: true + +steps: + - name: 'Processing Updates' + image: docker.io/devxygmbh/amd64-binaries-r-alpine-322 + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S + PGPASS: + from_secret: PGPASS + GIT_RO_TOKEN: + from_secret: GIT_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + CCACHE_DIR: /mnt/cache/ccache + PLATFORM: alpine-322 + ARCH: amd64 + NCPUS: 2 + STRATEGY: sequential + INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3) + # volumes: + # - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache + commands: + - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . + - rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraftR /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres + - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' + # force install pak devel as otherwise sysdeps on Alpine are not resolved correctly + - R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' + - R -q -e 'packageVersion("bincraftR")' + - mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages + # options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraftR::process_cran_updates(interval = $INTERVAL, platform = 'alpine-322', process_updated = TRUE, process_new = FALSE, process_removed = TRUE, s3_endpoint = 'https://hel1.your-objectstorage.com', s3_region = 'hel1', s3_bucket = 'devxy-r-package-binaries-hel1', s3_access_key_id = Sys.getenv('HETZNER_S3_ACCESS_KEY_K3S'), s3_secret_access_key = Sys.getenv('HETZNER_S3_SECRET_KEY_K3S'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'r_binaries', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)" + backend_options: + kubernetes: + resources: + requests: + memory: 5Gi + cpu: 3000m + limits: + memory: 18Gi + cpu: 3000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: AX42 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: 'ntfy' + image: binwiederhier/ntfy:v2.11.0 + when: + - status: [failure] + environment: + NTFY_TOKEN: + from_secret: ntfy_token + commands: | + ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/.crow/update-package-index-alpine-322-amd64.yaml b/.crow/update-package-index-alpine-322-amd64.yaml new file mode 100644 index 0000000..f4afa99 --- /dev/null +++ b/.crow/update-package-index-alpine-322-amd64.yaml @@ -0,0 +1,89 @@ +when: + - event: [cron] + cron: update-package-index-alpine-322-amd64 + - event: [cron] + cron: update-package-index + - event: manual + evaluate: 'update_package_index == "alpine-322-amd64"' + +skip_clone: true + +steps: + - name: Upload PACKAGES files + image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2404 + pull: true + environment: + RED_HAT_DEV_PW: + from_secret: RED_HAT_DEV_PW + HETZNER_S3_ACCESS_KEY_K3S: + from_secret: HETZNER_S3_ACCESS_KEY_K3S + HETZNER_S3_SECRET_KEY_K3S: + from_secret: HETZNER_S3_SECRET_KEY_K3S + PGPASS: + from_secret: PGPASS + GIT_RO_TOKEN: + from_secret: GIT_RO_TOKEN + GITHUB_PAT: + from_secret: GITHUB_PAT + # normal env vars + GIT_USER: pat-s + NTFY_SERVER: + from_secret: NTFY_SERVER + NTFY_TOPIC: + from_secret: NTFY_TOPIC + NTFY_AUTH: TRUE + NTFY_PASSWORD: + from_secret: ntfy_token + # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves + R_PKG_CACHE_DIR: /mnt/cache/pkgcache + R_LIBS_USER: /mnt/cache/R-pkgs + # volumes: + # - amd64-binaries-r-dep-cache-alpine-322:/mnt/cache + commands: + - git clone -q https://pat-s:$$GIT_RO_TOKEN@git.devxy.io/devxy/build-binaries.git . + - rm -rf /mnt/cache/packages/* + - R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git")' + - R -q -e 'pak::pak("pat-s/cranlike@s3")' + - R -q -e 'pak::pak("pat-s/desc@description-from-remote")' + - R -q -e 'packageVersion("bincraftR")' + # - cd /mnt/cache/packages + - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); future::plan("multisession", workers = 4); library(progressr); handlers(global = TRUE); handlers("progress"); upload_package_index(codename = "alpine322", s3_endpoint = "https://hel1.your-objectstorage.com", s3_region = "hel1", s3_bucket = "devxy-r-package-binaries-hel1", s3_access_key_id = Sys.getenv("HETZNER_S3_ACCESS_KEY_K3S"), s3_secret_access_key = Sys.getenv("HETZNER_S3_SECRET_KEY_K3S"))' + backend_options: + kubernetes: + resources: + requests: + memory: 3000Mi + cpu: 2000m + limits: + memory: 3000Mi + cpu: 8000m + nodeSelector: + kubernetes.io/arch: 'amd64' + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' + + - name: Purge CDN cache + image: curlimages/curl + environment: + BUNNYNET_API_KEY: + from_secret: BUNNYNET_API_KEY + commands: | + curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES&async=false + curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.gz&async=false + curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.rds&async=false + curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPACKAGES.db&async=false + curl -sL -X POST -H "AccessKey: $BUNNYNET_API_KEY" https://api.bunny.net/purge?url=https%3A%2F%2Fcran.devxy.io%2Famd64%2Falpine322%2Flatest%2Fsrc%2Fcontrib%2FPMeta%2FParchive.rds&async=false + sleep 20 # need to wait a bit to successfully deliver API calls + + - name: 'ntfy' + image: binwiederhier/ntfy:v2.11.0 + when: + - status: [failure] + environment: + NTFY_TOKEN: + from_secret: ntfy_token + commands: | + ntfy publish --tags=no_entry_sign --token $NTFY_TOKEN -t "Run '${CI_REPO}' failed" --click="${CI_PIPELINE_URL}" --actions="view, Open build, ${CI_PIPELINE_URL}, clear=true;" ntfy.devxy.io/ci-builds Workflow: ${CI_WORKFLOW_NAME} Commit message: ${CI_COMMIT_MESSAGE} Commit Author: ${CI_COMMIT_AUTHOR} diff --git a/docker/Containerfile-alpine-322 b/docker/Containerfile-alpine-322 new file mode 100644 index 0000000..18d5d55 --- /dev/null +++ b/docker/Containerfile-alpine-322 @@ -0,0 +1,69 @@ +FROM alpine:3.22 AS jags + +RUN apk add -q --no-cache curl tar make lapack lapack-dev blas blas-dev gcc gfortran g++ git cmake tk-dev xz + +# These usually are not packaged in OS repositories and need to be installed from source +# - JAGS (needed for runjags which itself is again a dependency of many packages) +RUN curl -s -L -O https://sourceforge.net/projects/mcmc-jags/files/JAGS/4.x/Source/JAGS-4.3.2.tar.gz && \ + tar -xzf JAGS-4.3.2.tar.gz && cd JAGS-4.3.2 && ./configure --enable-silent-rules && make 2>&1 + +FROM alpine:3.22 AS final + +ARG R_VERSION=4.5.0 +ENV PATH=/opt/R/${R_VERSION}/bin:$PATH +ENV TZ="Europe/Berlin" +ENV R_LIBS_USER=/mnt/cache/R-pkgs +ENV R_PKG_CACHE_DIR=/mnt/cache/pkgcache + +ARG ARCH=arm64 +ENV NCPUS=2 + +ARG GITHUB_PAT + +COPY --link --from=jags /JAGS-4.3.2 /JAGS-4.3.2 + +# Background: JAGS needs lapack & lapack-dev but these conflict with openblas-dev which is required by R-dev +# Solution: first compile JAGS, then remove lapack & lapack-dev again +# # xvfb: for graphical pkgs requiring tcl, like gWidgets2tcltk +RUN apk add -q --no-cache curl tar make blas blas-dev gcc gfortran g++ htop git cmake tk-dev xvfb xvfb-run xorg-server xauth xinit xz libdeflate libdeflate-dev + +### CUSTOM DEPENDENCIES + +# linux-headers: ps +# libxml2-dev: xml2 +# libgit2-dev: gert +# libpq-dev: RPostgres +# libfftw3-dev: ravetools -> actually auto-detect but somehow doesn't work in container? +RUN apk add -q --no-cache R-dev curl ca-certificates bash g++ tzdata openjdk17 tar sed patch openblas-dev pkgconfig linux-headers ccache libxml2-dev libgit2-dev libpq-dev pkgconfig fftw-dev automake autoconf libtool elfutils-dev +# - Chromium should be auto-installed once pkgdepends merges the latest system-requirements rule changes +RUN apk add -q --no-cache chromium +# FIXME: https://github.com/dmlc/xgboost/issues/8595 +RUN apk add --no-cache --update --repository=https://dl-cdn.alpinelinux.org/alpine/v3.16/main/ libexecinfo-dev + +# https://github.com/RcppCore/Rcpp/issues/448#issuecomment-220148774 +# LDFLAGS+=-fPIC -> arrow (https://github.com/r-hub/r-minimal/blob/main/examples/arrow/Dockerfile) +# dbarts: CFLAGS += -flax-vector-conversions (https://github.com/vdorie/dbarts/issues/66) +# CXXFLAGS: -> Rcpp +# -DDMLC_USE_FOPEN64=0 -> xgboost (https://github.com/dmlc/xgboost/issues/8595) +# can't set ccache for gfortran because of https://gitlab.com/roigrp/solver/highs/-/issues/4 +RUN mkdir -p /root/.R && bash -c 'echo -e "CXX_STD = CXX14\nCXXFLAGS=-D__MUSL__ -DDMLC_USE_FOPEN64=0\nCPPFLAGS=-D__MUSL__ -DDMLC_USE_FOPEN64=0\nCFLAGS += -flax-vector-conversions\nLDFLAGS+=-fPIC\nCC=ccache gcc\nCPP=gcc -E\nCXXCPP=gcc -e\nCXX=ccache g++\nCXX11=ccache g++\nCXX14=ccache g++\nCXX17=ccache g++\nF77=gfortran\nFC=gfortran" > /root/.R/Makevars' + +# R repos +RUN bash -c 'echo -e "options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.globals.onReference = NULL, repos = structure(c(CRAN = \"https://cloud.r-project.org\", INLA = \"https://inla.r-inla-download.org/R/stable\")))" > /root/.Rprofile' +RUN bash -c 'echo -e "PKG_SYSREQS=TRUE\nPKG_SYSREQS_VERBOSE=TRUE" > ~/.Renviron' + +### INSTALL R +RUN curl -O "https://devxy-r-builds.fsn1.your-objectstorage.com/alpine322/r-${R_VERSION}_1_$(arch).apk" +RUN apk add --allow-untrusted "r-${R_VERSION}_1_$(arch).apk" && rm "r-${R_VERSION}_1_$(arch).apk" + +RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))' + +# these dirs will later be used by the CI with existing mounts +RUN mkdir -p /mnt/cache/packages /mnt/cache/pkgcache /mnt/cache/R-pkgs + +# encoding -> https://github.com/r-hub/r-minimal/blob/fa9c117313e05860a0d20b113e5f794c51be8b8c/Dockerfile#L84 +RUN sed -i 's/,//g' /opt/R/${R_VERSION}/lib/R/library/utils/iconvlist + +RUN R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraftr.git", dependencies = TRUE)' + +CMD ["/bin/bash"]