make alpine dockerfile arch-agnostic

.woodpecker/build.yaml

@@ -7,18 +7,26 @@ matrix:
   include:
     # - PLATFORM: redhat-9
     #   BLOCK: 20099:21169
+    #   ARCH: arm64
     # - PLATFORM: redhat-8
     #   BLOCK: 20099:21169
+    #   ARCH: arm64
     # - PLATFORM: ubuntu-2404
     #   BLOCK: 20099:21169
+    #   ARCH: arm64
     # - PLATFORM: ubuntu-2204
     #   BLOCK: 20099:21169
+    #   ARCH: arm64
+    - PLATFORM: alpine-320
+      BLOCK: 5881:length(pkgs)
+      ARCH: arm64
     - PLATFORM: alpine-320
       BLOCK: 4680:length(pkgs)
+      ARCH: amd64
 steps:
   - name: 'Build binaries (${PLATFORM}: ${BLOCK})'
     secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
-    image: docker.io/devxygmbh/arm-binaries-r-${PLATFORM}
+    image: docker.io/devxygmbh/${ARCH}-binaries-r-${PLATFORM}
     pull: true
     # to ensure the PACKAGES upload step is being executed
     failure: ignore
@@ -29,7 +37,7 @@ steps:
       R_LIBS_USER: /mnt/cache/R-pkgs
       CCACHE_DIR: /mnt/cache/ccache
     volumes:
-      - arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
+      - ${ARCH}-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
     commands:
       # download latest package version (avoid having to rebuild image every time)
       # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
@@ -65,38 +73,9 @@ steps:
         #     memory: 4Gi
         #     cpu: 2000m
         nodeSelector:
-          kubernetes.io/arch: 'arm64'
+          kubernetes.io/arch: "${ARCH}"
         tolerations:
           - key: "CI"
             operator: "Equal"
             value: "true"
             effect: "NoSchedule"
-  # - name: Upload PACKAGES files
-  #   secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token]
-  #   image: docker.io/pats22/arm-binaries-r-ubi9
-  #   pull: true
-  #   environment:
-  #     GIT_USER: pat-s
-  #     # set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
-  #     R_PKG_CACHE_DIR: /mnt/cache/pkgcache
-  #     R_LIBS_USER: /mnt/cache/R-pkgs
-  #   volumes:
-  #     - arm-binaries-r-dep-cache-rhel9:/mnt/cache
-  #   commands:
-  #     # download latest package version (avoid having to rebuild image every time)
-  #     # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169
-  #     - R -q -e 'pak::local_install(); packageVersion("bincraftR")'
-  #     - R -q -e 'pak::pak(c("dyfanjones/s3fs", "pat-s/desc@description-from-remote", "pat-s/cranlike@s3"))'
-  #     - cd /mnt/cache/packages
-  #     - R -q -e 'options(crayon.enabled = TRUE); library(bincraftR); upload_package_index()'
-  #   backend_options:
-  #     kubernetes:
-  #       resources:
-  #         requests:
-  #           memory: 1000Mi
-  #           cpu: 0m
-  #         limits:
-  #           memory: 1300Mi
-  #           cpu: 1000m
-  #       nodeSelector:
-  #         kubernetes.io/arch: 'arm64'

.woodpecker/images.yaml

@@ -22,7 +22,7 @@ steps:
     image: woodpeckerci/plugin-docker-buildx:4.2.0
     secrets: [RED_HAT_DEV_PW]
     settings:
-      platforms: linux/arm64
+      platforms: linux/${ARCH}
       Dockerfile: docker/Dockerfile-${PLATFORM}
       experimental: true
       repo: devxygmbh/${ARCH}-binaries-r-${PLATFORM}

.woodpecker/process_cran_updates.yaml

@@ -14,7 +14,7 @@ matrix:
 steps:
   - name: 'Processing Updates: (${PLATFORM})'
     secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
-    image: docker.io/devxygmbh/arm-binaries-r-${PLATFORM}
+    image: docker.io/devxygmbh/arm64-binaries-r-${PLATFORM}
     pull: true
     # to ensure the PACKAGES upload step is being executed
     failure: ignore
@@ -25,7 +25,7 @@ steps:
       R_LIBS_USER: /mnt/cache/R-pkgs
       CCACHE_DIR: /mnt/cache/ccache
     volumes:
-      - arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
+      - arm64-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
     commands:
       # download latest package version (avoid having to rebuild image every time)
       - git clone https://pat-s:$$git_ro_token@git.devxy.io/devxy/arm64-r-binaries.git .

.woodpecker/update-package-index.yaml

@@ -14,7 +14,7 @@ matrix:
 steps:
   - name: Upload PACKAGES files
     secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
-    image: docker.io/devxygmbh/arm-binaries-r-${PLATFORM}
+    image: docker.io/devxygmbh/arm64-binaries-r-${PLATFORM}
     pull: true
     environment:
       GIT_USER: pat-s
@@ -22,7 +22,7 @@ steps:
       R_PKG_CACHE_DIR: /mnt/cache/pkgcache
       R_LIBS_USER: /mnt/cache/R-pkgs
     volumes:
-      - arm-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
+      - arm64-binaries-r-dep-cache-${PLATFORM}:/mnt/cache
     commands:
       # download latest package version (avoid having to rebuild image every time)
       # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169

Justfile

@@ -1,6 +1,6 @@
 image OS:
     if echo "{{OS}}" | grep -qi "redhat"; then \
-        docker buildx build --progress plain --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm-binaries-r-{{OS}}:latest --push .; \
+        docker buildx build --progress plain --build-arg GITHUB_PAT="$GITHUB_PAT" --build-arg RED_HAT_DEV_PW="$RED_HAT_DEV_PW" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm64-binaries-r-{{OS}}:latest --push .; \
     else \
-        docker buildx build --build-arg GITHUB_PAT="$GITHUB_PAT" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm-binaries-r-{{OS}}:latest --push .; \
+        docker buildx build --build-arg GITHUB_PAT="$GITHUB_PAT" -f docker/Dockerfile-{{OS}} -t devxygmbh/arm64-binaries-r-{{OS}}:latest --push .; \
     fi

curl-benchmark.sh

@@ -11,27 +11,50 @@ echo -e "
           time_total:  %{time_total}s\n
 " > curl-format.txt
 
-curl -w "@curl-format.txt" -o /dev/null -s "https://packagemanager.posit.co/cran/latest/src/contrib/arrow_16.1.0.tar.gz"
+# DE
+curl -w "@curl-format.txt" -o /dev/null -s "https://packagemanager.posit.co/cran/__linux__/rhel9/latest/brew_1.0-10.tar.gz"
 
-time_namelookup:  0.006233s        
+time_namelookup:  0.031104s        
-time_connect:  0.125975s     
+time_connect:  0.140160s     
-time_appconnect:  0.372086s    
+time_appconnect:  0.393117s    
-time_pretransfer:  0.373104s       
+time_pretransfer:  0.393304s       
 time_redirect:  0.000000s  
-time_starttransfer:  0.499798s                     
+time_starttransfer:  0.502755s                     
 ----------          
-time_total:  0.500019s⏎  
+time_total:  0.502838s                    
 
-curl -w "@curl-format.txt" -o NUL -s "https://cran.devxy.io/arm64/rhel9/latest/src/contrib/arrow_16.1.0.tar.gz"
+curl -w "@curl-format.txt" -o /dev/null -s -O "https://cran.devxy.io/arm64/rhel9/latest/src/contrib/brew_1.0-10.tar.gz"
 
-time_namelookup:  0.003505s        
+time_namelookup:  0.015006s        
-time_connect:  0.018930s     
+time_connect:  0.021476s     
-time_appconnect:  0.041507s    
+time_appconnect:  0.081913s    
-time_pretransfer:  0.041587s       
+time_pretransfer:  0.082039s       
 time_redirect:  0.000000s  
-time_starttransfer:  0.056802s                     
+time_starttransfer:  0.089900s                     
 ----------          
-time_total:  0.057002s⏎ 
+time_total:  0.096288s
+
+### ZH
+
+# Posit PM
+time_namelookup:  0.009141s        
+time_connect:  0.130860s     
+time_appconnect:  0.382310s    
+time_pretransfer:  0.382722s       
+time_redirect:  0.000000s  
+time_starttransfer:  0.505626s                     
+----------          
+time_total:  0.505885s⏎ 
+
+# devxy
+time_namelookup:  0.003021s        
+time_connect:  0.009213s     
+time_appconnect:  0.018338s    
+time_pretransfer:  0.018439s       
+time_redirect:  0.000000s  
+time_starttransfer:  0.025136s                     
+----------          
+time_total:  0.025260s⏎  
 
 
 rm curl-format.txt

docker/Dockerfile-alpine-320

@@ -24,14 +24,15 @@ RUN apk add -q --no-cache R-dev curl ca-certificates bash g++ tzdata openjdk17 t
 RUN mkdir ~/.R && echo -e "CXXFLAGS+=-D__MUSL__\nCPPFLAGS+= -D__MUSL__\nLDFLAGS+=-fPIC" >> ~/.R/Makevars
 
 ### INSTALL R
-RUN curl -O https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/alpine320/r-4.4.1_1_aarch64.apk
+RUN curl -O https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/alpine320/r-4.4.1_1_$(arch).apk
-RUN apk add --allow-untrusted r-4.4.1_1_aarch64.apk && rm r-4.4.1_1_aarch64.apk
+RUN apk add --allow-untrusted r-4.4.1_1_$(arch).apk && rm r-4.4.1_1_$(arch).apk
 
 RUN R -q -e 'install.packages("remotes", repo = "cloud.r-project.org"); remotes::install_github("pat-s/pak@alpine")'
 
 COPY . .
 # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628
-RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/alpine320/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
+RUN export ARCH=$(echo $(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')) && \
-RUN rm -rf arm-binaries-r
+ R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = sprintf("https://cran.devxy.io/%s/alpine320/latest", Sys.getenv("ARCH")), CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
+RUN r-pkg-binaries
 
 CMD ["/bin/bash"]

docker/Dockerfile-redhat-8

@@ -22,6 +22,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/
 COPY . .
 # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628
 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/rhel8/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
-RUN rm -rf arm-binaries-r
+RUN r-pkg-binaries
 
 CMD ["/bin/bash"]

docker/Dockerfile-redhat-9

@@ -22,6 +22,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/
 COPY . .
 # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628
 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/rhel9/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
-RUN rm -rf arm-binaries-r
+RUN r-pkg-binaries
 
 CMD ["/bin/bash"]

docker/Dockerfile-ubuntu-2204

@@ -19,6 +19,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/
 COPY . .
 # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628
 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/jammy/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
-RUN rm -rf arm64-binaries-r
+RUN r-pkg-binaries
 
 CMD ["/bin/bash"]

docker/Dockerfile-ubuntu-2404

@@ -19,6 +19,6 @@ RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/
 COPY . .
 # FIXME: switch to pak after https://github.com/r-lib/pak/issues/628
 RUN R -q -e 'Sys.setenv(PKG_SYSREQS = TRUE, PKG_SYSREQS_VERBOSE = TRUE); options(Ncpus = 4, repos = structure(c(devxy = "https://cran.devxy.io/arm64/noble/latest", CRAN = "https://cloud.r-project.org"))); install.packages("remotes"); remotes::install_deps()'
-RUN rm -rf arm64-binaries-r
+RUN r-pkg-binaries
 
 CMD ["/bin/bash"]