make alpine dockerfile arch-agnostic

2024-09-04 22:38:55 +02:00 · 2024-09-04 22:38:55 +02:00 · 125caa8a3e
commit 125caa8a3e
parent 5eced4ed2d
11 changed files with 64 additions and 61 deletions
--- a/.woodpecker/build.yaml
+++ b/.woodpecker/build.yaml
@ -7,18 +7,26 @@ matrix:
  include:
    # - PLATFORM: redhat-9
    #   BLOCK: 20099:21169
+    #   ARCH: arm64
    # - PLATFORM: redhat-8
    #   BLOCK: 20099:21169
+    #   ARCH: arm64
    # - PLATFORM: ubuntu-2404
    #   BLOCK: 20099:21169
+    #   ARCH: arm64
    # - PLATFORM: ubuntu-2204
    #   BLOCK: 20099:21169
+    #   ARCH: arm64
+    - PLATFORM: alpine-320
+      BLOCK: 5881:length(pkgs)
+      ARCH: arm64
    - PLATFORM: alpine-320
      BLOCK: 4680:length(pkgs)
+      ARCH: amd64
 steps:
  - name: 'Build binaries (${PLATFORM}: ${BLOCK})'
    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
-    image: docker.io/devxygmbh/arm-binaries-r-${PLATFORM}
+    image: docker.io/devxygmbh/${ARCH}-binaries-r-${PLATFORM}
    pull: true
    # to ensure the PACKAGES upload step is being executed
    failure: ignore
@ -29,7 +37,7 @@ steps:
      R_LIBS_USER: /mnt/cache/R-pkgs
      CCACHE_DIR: /mnt/cache/ccache
    volumes:
-      - arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
+      - ${ARCH}-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
    commands:
      # download latest package version (avoid having to rebuild image every time)
      # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
@ -65,38 +73,9 @@ steps:
        #     memory: 4Gi
        #     cpu: 2000m
        nodeSelector:
-          kubernetes.io/arch: 'arm64'
+          kubernetes.io/arch: "${ARCH}"
        tolerations:
          - key: "CI"
            operator: "Equal"
            value: "true"
            effect: "NoSchedule"
-  # - name: Upload PACKAGES files
-  #   secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
-  #   image: docker.io/pats22/arm-binaries-r-ubi9
-  #   pull: true
-  #   environment:
-  #     GIT_USER: pat-s
-  #     # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
-  #     R_PKG_CACHE_DIR: /mnt/cache/pkgcache
-  #     R_LIBS_USER: /mnt/cache/R-pkgs
-  #   volumes:
-  #     - arm-binaries-r-dep-cache-rhel9:/mnt/cache
-  #   commands:
-  #     # download latest package version (avoid having to rebuild image every time)
-  #     # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
-  #     - R -q -e 'pak::local_install(); packageVersion("bincraftR")'
-  #     - R -q -e 'pak::pak(c("dyfanjones/s3fs", "pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
-  #     - cd /mnt/cache/packages
-  #     - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); upload_package_index()'
-  #   backend_options:
-  #     kubernetes:
-  #       resources:
-  #         requests:
-  #           memory: 1000Mi
-  #           cpu: 0m
-  #         limits:
-  #           memory: 1300Mi
-  #           cpu: 1000m
-  #       nodeSelector:
-  #         kubernetes.io/arch: 'arm64'
--- a/.woodpecker/images.yaml
+++ b/.woodpecker/images.yaml
@ -22,7 +22,7 @@ steps:
    image: woodpeckerci/plugin-docker-buildx:4.2.0
    secrets: [RED_HAT_DEV_PW]
    settings:
-      platforms: linux/arm64
+      platforms: linux/${ARCH}
      Dockerfile: docker/Dockerfile-${PLATFORM}
      experimental: true
      repo: devxygmbh/${ARCH}-binaries-r-${PLATFORM}
--- a/.woodpecker/process_cran_updates.yaml
+++ b/.woodpecker/process_cran_updates.yaml
@ -14,7 +14,7 @@ matrix:
 steps:
  - name: 'Processing Updates: (${PLATFORM})'
    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
-    image: docker.io/devxygmbh/arm-binaries-r-${PLATFORM}
+    image: docker.io/devxygmbh/arm64-binaries-r-${PLATFORM}
    pull: true
    # to ensure the PACKAGES upload step is being executed
    failure: ignore
@ -25,7 +25,7 @@ steps:
      R_LIBS_USER: /mnt/cache/R-pkgs
      CCACHE_DIR: /mnt/cache/ccache
    volumes:
-      - arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
+      - arm64-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
    commands:
      # download latest package version (avoid having to rebuild image every time)
      - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git .
--- a/.woodpecker/update-package-index.yaml
+++ b/.woodpecker/update-package-index.yaml
@ -14,7 +14,7 @@ matrix:
 steps:
  - name: Upload PACKAGES files
    secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
-    image: docker.io/devxygmbh/arm-binaries-r-${PLATFORM}
+    image: docker.io/devxygmbh/arm64-binaries-r-${PLATFORM}
    pull: true
    environment:
      GIT_USER: pat-s
@ -22,7 +22,7 @@ steps:
      R_PKG_CACHE_DIR: /mnt/cache/pkgcache
      R_LIBS_USER: /mnt/cache/R-pkgs
    volumes:
-      - arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
+      - arm64-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
    commands:
      # download latest package version (avoid having to rebuild image every time)
      # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
--- a/4
+++ b/4
@ -1,6 +1,6 @@
 image OS:
    if echo "{{OS}}" | grep -qi "redhat"; then \
-        docker buildx build --progress plain --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm-binaries-r-{{OS}}:latest --push .; \
+        docker buildx build --progress plain --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm64-binaries-r-{{OS}}:latest --push .; \
    else \
-        docker buildx build --build-arg GITHUB_PAT="$GITHUB_PAT" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm-binaries-r-{{OS}}:latest --push .; \
+        docker buildx build --build-arg GITHUB_PAT="$GITHUB_PAT" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm64-binaries-r-{{OS}}:latest --push .; \
    fi
--- a/curl-benchmark.sh
+++ b/curl-benchmark.sh
@ -11,27 +11,50 @@ echo -e "
          time_total:  %{time_total}s\n
 " > curl-format.txt

-curl -w "@curl-format.txt" -o /dev/null -s "https://packagemanager.posit.co/cran/latest/src/contrib/arrow_16.1.0.tar.gz"
+# DE
+curl -w "@curl-format.txt" -o /dev/null -s "https://packagemanager.posit.co/cran/__linux__/rhel9/latest/brew_1.0-10.tar.gz"

-time_namelookup:  0.006233s        
-time_connect:  0.125975s     
-time_appconnect:  0.372086s    
-time_pretransfer:  0.373104s       
+time_namelookup:  0.031104s        
+time_connect:  0.140160s     
+time_appconnect:  0.393117s    
+time_pretransfer:  0.393304s       
 time_redirect:  0.000000s  
-time_starttransfer:  0.499798s                     
+time_starttransfer:  0.502755s                     
 ----------          
-time_total:  0.500019s⏎  
+time_total:  0.502838s                    

-curl -w "@curl-format.txt" -o NUL -s "https://cran.devxy.io/arm64/rhel9/latest/src/contrib/arrow_16.1.0.tar.gz"
+curl -w "@curl-format.txt" -o /dev/null -s -O "https://cran.devxy.io/arm64/rhel9/latest/src/contrib/brew_1.0-10.tar.gz"

-time_namelookup:  0.003505s        
-time_connect:  0.018930s     
-time_appconnect:  0.041507s    
-time_pretransfer:  0.041587s       
+time_namelookup:  0.015006s        
+time_connect:  0.021476s     
+time_appconnect:  0.081913s    
+time_pretransfer:  0.082039s       
 time_redirect:  0.000000s  
-time_starttransfer:  0.056802s                     
+time_starttransfer:  0.089900s                     
 ----------          
-time_total:  0.057002s⏎ 
+time_total:  0.096288s
+
+### ZH
+
+# Posit PM
+time_namelookup:  0.009141s        
+time_connect:  0.130860s     
+time_appconnect:  0.382310s    
+time_pretransfer:  0.382722s       
+time_redirect:  0.000000s  
+time_starttransfer:  0.505626s                     
+----------          
+time_total:  0.505885s⏎ 
+
+# devxy
+time_namelookup:  0.003021s        
+time_connect:  0.009213s     
+time_appconnect:  0.018338s    
+time_pretransfer:  0.018439s       
+time_redirect:  0.000000s  
+time_starttransfer:  0.025136s                     
+----------          
+time_total:  0.025260s⏎  


 rm curl-format.txt
--- a/docker/Dockerfile-alpine-320
+++ b/docker/Dockerfile-alpine-320
@ -24,14 +24,15 @@ RUN apk add -q --no-cache R-dev curl ca-certificates bash g++ tzdata openjdk17 t
 RUN mkdir ~/.R && echo -e "CXXFLAGS+=-D__MUSL__\nCPPFLAGS+= -D__MUSL__\nLDFLAGS+=-fPIC" >> ~/.R/Makevars

 ### INSTALL R
-RUN curl -O https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/alpine320/r-4.4.1_1_aarch64.apk
-RUN apk add --allow-untrusted r-4.4.1_1_aarch64.apk && rm r-4.4.1_1_aarch64.apk
+RUN curl -O https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/alpine320/r-4.4.1_1_$(arch).apk
+RUN apk add --allow-untrusted r-4.4.1_1_$(arch).apk && rm r-4.4.1_1_$(arch).apk

 RUN R -q -e 'install.packages("remotes", repo = "cloud.r-project.org"); remotes::install_github("pat-s/pak@alpine")'

 COPY . .
 # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628
-RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/alpine320/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
-RUN rm -rf arm-binaries-r
+RUN export ARCH=$(echo $(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')) && \
+ R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = sprintf("https://cran.devxy.io/%s/alpine320/latest", Sys.getenv("ARCH")), CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
+RUN r-pkg-binaries

 CMD ["/bin/bash"]
--- a/docker/Dockerfile-redhat-8
+++ b/docker/Dockerfile-redhat-8
@ -22,6 +22,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/
 COPY . .
 # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628
 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/rhel8/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
-RUN rm -rf arm-binaries-r
+RUN r-pkg-binaries

 CMD ["/bin/bash"]
--- a/docker/Dockerfile-redhat-9
+++ b/docker/Dockerfile-redhat-9
@ -22,6 +22,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/
 COPY . .
 # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628
 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/rhel9/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
-RUN rm -rf arm-binaries-r
+RUN r-pkg-binaries

 CMD ["/bin/bash"]
--- a/docker/Dockerfile-ubuntu-2204
+++ b/docker/Dockerfile-ubuntu-2204
@ -19,6 +19,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/
 COPY . .
 # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628
 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/jammy/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
-RUN rm -rf arm64-binaries-r
+RUN r-pkg-binaries

 CMD ["/bin/bash"]
--- a/docker/Dockerfile-ubuntu-2404
+++ b/docker/Dockerfile-ubuntu-2404
@ -19,6 +19,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/
 COPY . .
 # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628
 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/noble/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
-RUN rm -rf arm64-binaries-r
+RUN r-pkg-binaries

 CMD ["/bin/bash"]