From 125caa8a3e8093768bd1a9e6bb95cbe846fd6387 Mon Sep 17 00:00:00 2001 From: pat-s Date: Wed, 4 Sep 2024 22:38:55 +0200 Subject: [PATCH] make alpine dockerfile arch-agnostic --- .woodpecker/build.yaml | 43 ++++++---------------- .woodpecker/images.yaml | 2 +- .woodpecker/process_cran_updates.yaml | 4 +-- .woodpecker/update-package-index.yaml | 4 +-- Justfile | 4 +-- curl-benchmark.sh | 51 +++++++++++++++++++-------- docker/Dockerfile-alpine-320 | 9 ++--- docker/Dockerfile-redhat-8 | 2 +- docker/Dockerfile-redhat-9 | 2 +- docker/Dockerfile-ubuntu-2204 | 2 +- docker/Dockerfile-ubuntu-2404 | 2 +- 11 files changed, 64 insertions(+), 61 deletions(-) diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index a4988e0..cd14abd 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -7,18 +7,26 @@ matrix: include: # - PLATFORM: redhat-9 # BLOCK: 20099:21169 + # ARCH: arm64 # - PLATFORM: redhat-8 # BLOCK: 20099:21169 + # ARCH: arm64 # - PLATFORM: ubuntu-2404 # BLOCK: 20099:21169 + # ARCH: arm64 # - PLATFORM: ubuntu-2204 # BLOCK: 20099:21169 + # ARCH: arm64 + - PLATFORM: alpine-320 + BLOCK: 5881:length(pkgs) + ARCH: arm64 - PLATFORM: alpine-320 BLOCK: 4680:length(pkgs) + ARCH: amd64 steps: - name: 'Build binaries (${PLATFORM}: ${BLOCK})' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] - image: docker.io/devxygmbh/arm-binaries-r-${PLATFORM} + image: docker.io/devxygmbh/${ARCH}-binaries-r-${PLATFORM} pull: true # to ensure the PACKAGES upload step is being executed failure: ignore @@ -29,7 +37,7 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache volumes: - - arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache + - ${ARCH}-binaries-r-dep-cache-${PLATFORM}:/mnt/cache commands: # download latest package version (avoid having to rebuild image every time) # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 @@ -65,38 +73,9 @@ steps: # memory: 4Gi # cpu: 2000m nodeSelector: - kubernetes.io/arch: 'arm64' + kubernetes.io/arch: "${ARCH}" tolerations: - key: "CI" operator: "Equal" value: "true" effect: "NoSchedule" - # - name: Upload PACKAGES files - # secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token] - # image: docker.io/pats22/arm-binaries-r-ubi9 - # pull: true - # environment: - # GIT_USER: pat-s - # # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves - # R_PKG_CACHE_DIR: /mnt/cache/pkgcache - # R_LIBS_USER: /mnt/cache/R-pkgs - # volumes: - # - arm-binaries-r-dep-cache-rhel9:/mnt/cache - # commands: - # # download latest package version (avoid having to rebuild image every time) - # # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 - # - R -q -e 'pak::local_install(); packageVersion("bincraftR")' - # - R -q -e 'pak::pak(c("dyfanjones/s3fs", "pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))' - # - cd /mnt/cache/packages - # - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); upload_package_index()' - # backend_options: - # kubernetes: - # resources: - # requests: - # memory: 1000Mi - # cpu: 0m - # limits: - # memory: 1300Mi - # cpu: 1000m - # nodeSelector: - # kubernetes.io/arch: 'arm64' diff --git a/.woodpecker/images.yaml b/.woodpecker/images.yaml index 3aacb55..1a0766d 100644 --- a/.woodpecker/images.yaml +++ b/.woodpecker/images.yaml @@ -22,7 +22,7 @@ steps: image: woodpeckerci/plugin-docker-buildx:4.2.0 secrets: [RED_HAT_DEV_PW] settings: - platforms: linux/arm64 + platforms: linux/${ARCH} Dockerfile: docker/Dockerfile-${PLATFORM} experimental: true repo: devxygmbh/${ARCH}-binaries-r-${PLATFORM} diff --git a/.woodpecker/process_cran_updates.yaml b/.woodpecker/process_cran_updates.yaml index 036d082..30846c9 100644 --- a/.woodpecker/process_cran_updates.yaml +++ b/.woodpecker/process_cran_updates.yaml @@ -14,7 +14,7 @@ matrix: steps: - name: 'Processing Updates: (${PLATFORM})' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] - image: docker.io/devxygmbh/arm-binaries-r-${PLATFORM} + image: docker.io/devxygmbh/arm64-binaries-r-${PLATFORM} pull: true # to ensure the PACKAGES upload step is being executed failure: ignore @@ -25,7 +25,7 @@ steps: R_LIBS_USER: /mnt/cache/R-pkgs CCACHE_DIR: /mnt/cache/ccache volumes: - - arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache + - arm64-binaries-r-dep-cache-${PLATFORM}:/mnt/cache commands: # download latest package version (avoid having to rebuild image every time) - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git . diff --git a/.woodpecker/update-package-index.yaml b/.woodpecker/update-package-index.yaml index 7b2d0d9..1a426b7 100644 --- a/.woodpecker/update-package-index.yaml +++ b/.woodpecker/update-package-index.yaml @@ -14,7 +14,7 @@ matrix: steps: - name: Upload PACKAGES files secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] - image: docker.io/devxygmbh/arm-binaries-r-${PLATFORM} + image: docker.io/devxygmbh/arm64-binaries-r-${PLATFORM} pull: true environment: GIT_USER: pat-s @@ -22,7 +22,7 @@ steps: R_PKG_CACHE_DIR: /mnt/cache/pkgcache R_LIBS_USER: /mnt/cache/R-pkgs volumes: - - arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache + - arm64-binaries-r-dep-cache-${PLATFORM}:/mnt/cache commands: # download latest package version (avoid having to rebuild image every time) # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 diff --git a/Justfile b/Justfile index 0068214..fe15cd1 100644 --- a/Justfile +++ b/Justfile @@ -1,6 +1,6 @@ image OS: if echo "{{OS}}" | grep -qi "redhat"; then \ - docker buildx build --progress plain --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm-binaries-r-{{OS}}:latest --push .; \ + docker buildx build --progress plain --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm64-binaries-r-{{OS}}:latest --push .; \ else \ - docker buildx build --build-arg GITHUB_PAT="$GITHUB_PAT" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm-binaries-r-{{OS}}:latest --push .; \ + docker buildx build --build-arg GITHUB_PAT="$GITHUB_PAT" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm64-binaries-r-{{OS}}:latest --push .; \ fi \ No newline at end of file diff --git a/curl-benchmark.sh b/curl-benchmark.sh index 0c1ad80..4834fde 100644 --- a/curl-benchmark.sh +++ b/curl-benchmark.sh @@ -11,27 +11,50 @@ echo -e " time_total: %{time_total}s\n " > curl-format.txt -curl -w "@curl-format.txt" -o /dev/null -s "https://packagemanager.posit.co/cran/latest/src/contrib/arrow_16.1.0.tar.gz" +# DE +curl -w "@curl-format.txt" -o /dev/null -s "https://packagemanager.posit.co/cran/__linux__/rhel9/latest/brew_1.0-10.tar.gz" -time_namelookup: 0.006233s -time_connect: 0.125975s -time_appconnect: 0.372086s -time_pretransfer: 0.373104s +time_namelookup: 0.031104s +time_connect: 0.140160s +time_appconnect: 0.393117s +time_pretransfer: 0.393304s time_redirect: 0.000000s -time_starttransfer: 0.499798s +time_starttransfer: 0.502755s ---------- -time_total: 0.500019s⏎ +time_total: 0.502838s -curl -w "@curl-format.txt" -o NUL -s "https://cran.devxy.io/arm64/rhel9/latest/src/contrib/arrow_16.1.0.tar.gz" +curl -w "@curl-format.txt" -o /dev/null -s -O "https://cran.devxy.io/arm64/rhel9/latest/src/contrib/brew_1.0-10.tar.gz" -time_namelookup: 0.003505s -time_connect: 0.018930s -time_appconnect: 0.041507s -time_pretransfer: 0.041587s +time_namelookup: 0.015006s +time_connect: 0.021476s +time_appconnect: 0.081913s +time_pretransfer: 0.082039s time_redirect: 0.000000s -time_starttransfer: 0.056802s +time_starttransfer: 0.089900s ---------- -time_total: 0.057002s⏎ +time_total: 0.096288s + +### ZH + +# Posit PM +time_namelookup: 0.009141s +time_connect: 0.130860s +time_appconnect: 0.382310s +time_pretransfer: 0.382722s +time_redirect: 0.000000s +time_starttransfer: 0.505626s +---------- +time_total: 0.505885s⏎ + +# devxy +time_namelookup: 0.003021s +time_connect: 0.009213s +time_appconnect: 0.018338s +time_pretransfer: 0.018439s +time_redirect: 0.000000s +time_starttransfer: 0.025136s +---------- +time_total: 0.025260s⏎ rm curl-format.txt \ No newline at end of file diff --git a/docker/Dockerfile-alpine-320 b/docker/Dockerfile-alpine-320 index 5a27e6a..24bbeba 100644 --- a/docker/Dockerfile-alpine-320 +++ b/docker/Dockerfile-alpine-320 @@ -24,14 +24,15 @@ RUN apk add -q --no-cache R-dev curl ca-certificates bash g++ tzdata openjdk17 t RUN mkdir ~/.R && echo -e "CXXFLAGS+=-D__MUSL__\nCPPFLAGS+= -D__MUSL__\nLDFLAGS+=-fPIC" >> ~/.R/Makevars ### INSTALL R -RUN curl -O https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/alpine320/r-4.4.1_1_aarch64.apk -RUN apk add --allow-untrusted r-4.4.1_1_aarch64.apk && rm r-4.4.1_1_aarch64.apk +RUN curl -O https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/alpine320/r-4.4.1_1_$(arch).apk +RUN apk add --allow-untrusted r-4.4.1_1_$(arch).apk && rm r-4.4.1_1_$(arch).apk RUN R -q -e 'install.packages("remotes", repo = "cloud.r-project.org"); remotes::install_github("pat-s/pak@alpine")' COPY . . # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628 -RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/alpine320/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()' -RUN rm -rf arm-binaries-r +RUN export ARCH=$(echo $(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')) && \ + R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = sprintf("https://cran.devxy.io/%s/alpine320/latest", Sys.getenv("ARCH")), CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()' +RUN r-pkg-binaries CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/Dockerfile-redhat-8 b/docker/Dockerfile-redhat-8 index 9f2c3a7..c4ed7c9 100644 --- a/docker/Dockerfile-redhat-8 +++ b/docker/Dockerfile-redhat-8 @@ -22,6 +22,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/ COPY . . # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/rhel8/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()' -RUN rm -rf arm-binaries-r +RUN r-pkg-binaries CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/Dockerfile-redhat-9 b/docker/Dockerfile-redhat-9 index a4a74db..9857278 100644 --- a/docker/Dockerfile-redhat-9 +++ b/docker/Dockerfile-redhat-9 @@ -22,6 +22,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/ COPY . . # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/rhel9/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()' -RUN rm -rf arm-binaries-r +RUN r-pkg-binaries CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/Dockerfile-ubuntu-2204 b/docker/Dockerfile-ubuntu-2204 index 2339d95..b151106 100644 --- a/docker/Dockerfile-ubuntu-2204 +++ b/docker/Dockerfile-ubuntu-2204 @@ -19,6 +19,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/ COPY . . # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/jammy/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()' -RUN rm -rf arm64-binaries-r +RUN r-pkg-binaries CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/Dockerfile-ubuntu-2404 b/docker/Dockerfile-ubuntu-2404 index 0b6b80a..a8e726f 100644 --- a/docker/Dockerfile-ubuntu-2404 +++ b/docker/Dockerfile-ubuntu-2404 @@ -19,6 +19,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/ COPY . . # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/noble/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()' -RUN rm -rf arm64-binaries-r +RUN r-pkg-binaries CMD ["/bin/bash"] \ No newline at end of file