pat-s
e47949f546
Some checks failed
ci/crow/manual/build-all-versions-install-deps-amd64 Pipeline was successful
ci/crow/manual/build-all-versions-amd64/1 Pipeline failed
ci/crow/manual/build-all-versions-amd64/2 Pipeline failed
ci/crow/manual/build-all-versions-amd64/4 Pipeline failed
ci/crow/manual/build-all-versions-amd64/3 Pipeline failed
force=TRUE bypasses the error-DB filter and overwrites build metadata, causing failed packages to be retried indefinitely. Since the precompute step already filters to missing versions, force is unnecessary.
112 lines
6.6 KiB
YAML
112 lines
6.6 KiB
YAML
### Required extra vars
|
|
# ARCH (amd64)
|
|
# OS (alpine)
|
|
# OS_VERSION (3.22)
|
|
# R_VERSION (4.5)
|
|
# echo "c('RInno','KeyboardSimulator','R2PPT','RWinEdt','blatr','excel.link','spectrino','taskscheduleR','MDSGUI','BiplotGUI','R2wd','rFUSION','MediaNews','doBy','IDPmisc','frailtypack','afex','FrF2','DoE.base','agricolae','doFuture','fscaret','PHYLOGR','seewave','pls','relaimpo','geepack','gggenes','NPCirc','repmis','PNDSIBGE','lidR','poismf','neonstore','MachineShop','mvst','MacBehaviour','mcmcderive','RGIFT','KnowBR','netmeta','spdep','sf','Rfast','compareGroups','ff','GsymPoint','RcppDynProg','comtradr','RcppDynProg','FD','PearsonDS','DCluster','gRc','mixlm','geospt','fdth','ffmanova','fiery','ffscrapr','cold','fiery','RcmdrPlugin.DoE','RcmdrPlugin.NMBU')" | base64
|
|
# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.22 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --branch=main --var SKIP_PKGS=YygnUklubm8nLCdLZXlib2FyZFNpbXVsYXRvcicsJ1IyUFBUJywnUldpbkVkdCcsJ2JsYXRyJywnZXhjZWwubGluaycsJ3NwZWN0cmlubycsJ3Rhc2tzY2hlZHVsZVInLCdNRFNHVUknLCdCaXBsb3RHVUknLCdSMndkJywnckZVU0lPTicsJ01lZGlhTmV3cycsJ2RvQnknLCdJRFBtaXNjJywnZnJhaWx0eXBhY2snLCdhZmV4JywnRnJGMicsJ0RvRS5iYXNlJywnYWdyaWNvbGFlJywnZG9GdXR1cmUnLCdmc2NhcmV0JywnUEhZTE9HUicsJ3NlZXdhdmUnLCdwbHMnLCdyZWxhaW1wbycsJ2dlZXBhY2snLCdnZ2dlbmVzJywnTlBDaXJjJywncmVwbWlzJywnUE5EU0lCR0UnLCdsaWRSJywncG9pc21mJywnbmVvbnN0b3JlJywnTWFjaGluZVNob3AnLCdtdnN0JywnTWFjQmVoYXZpb3VyJywnbWNtY2Rlcml2ZScsJ1JHSUZUJywnS25vd0JSJywnbmV0bWV0YScsJ3NwZGVwJywnc2YnLCdSZmFzdCcsJ2NvbXBhcmVHcm91cHMnLCdmZicsJ0dzeW1Qb2ludCcsJ1JjcHBEeW5Qcm9nJywnY29tdHJhZHInLCdSY3BwRHluUHJvZycsJ0ZEJywnUGVhcnNvbkRTJywnRENsdXN0ZXInLCdnUmMnLCdtaXhsbScsJ2dlb3NwdCcsJ2ZkdGgnLCdmZm1hbm92YScsJ2ZpZXJ5JywnZmZzY3JhcHInLCdjb2xkJywnZmllcnknLCdSY21kclBsdWdpbi5Eb0UnLCdSY21kclBsdWdpbi5OTUJVJykK --log-level=info 7
|
|
# crow pipeline create --var ARCH=arm64 --var OS=alpine --var OS_VERSION=3.23 --var R_VERSION=4.5 --var task=build-all-arm64 --var K8S_INSTANCE_TYPE=cax41 --branch=main --log-level=info 5
|
|
# crow pipeline create --var ARCH=arm64 --var OS=redhat --var OS_VERSION=10 --var R_VERSION=4.5.3 --var task=build-all-arm64 --branch=main --log-level=info 5
|
|
when:
|
|
- event: manual
|
|
evaluate: 'task == "build-all-arm64"'
|
|
|
|
skip_clone: true
|
|
|
|
labels:
|
|
agent: gaia
|
|
|
|
matrix:
|
|
include:
|
|
# - SPLIT_INTO: 5
|
|
# SPLIT_INDEX: 1
|
|
# BACKEND: kubernetes
|
|
# - SPLIT_INTO: 5
|
|
# SPLIT_INDEX: 2
|
|
# BACKEND: kubernetes
|
|
# - SPLIT_INTO: 5
|
|
# SPLIT_INDEX: 3
|
|
# BACKEND: kubernetes
|
|
# - SPLIT_INTO: 5
|
|
# SPLIT_INDEX: 4
|
|
# BACKEND: kubernetes
|
|
# - SPLIT_INTO: 5
|
|
# SPLIT_INDEX: 5
|
|
# BACKEND: kubernetes
|
|
- SPLIT_INTO: 4
|
|
SPLIT_INDEX: 1
|
|
BACKEND: docker
|
|
- SPLIT_INTO: 4
|
|
SPLIT_INDEX: 2
|
|
BACKEND: docker
|
|
- SPLIT_INTO: 4
|
|
SPLIT_INDEX: 3
|
|
BACKEND: docker
|
|
- SPLIT_INTO: 4
|
|
SPLIT_INDEX: 4
|
|
BACKEND: docker
|
|
# - SPLIT_INTO: 5
|
|
# SPLIT_INDEX: 5
|
|
# BACKEND: docker
|
|
|
|
depends_on:
|
|
- build-all-versions-install-deps-arm64
|
|
|
|
steps:
|
|
- name: 'Build binaries'
|
|
image: reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}-${R_VERSION}
|
|
pull: true
|
|
environment:
|
|
RED_HAT_DEV_PW:
|
|
from_secret: RED_HAT_DEV_PW
|
|
B2_S3_ACCESS_KEY:
|
|
from_secret: B2_S3_ACCESS_KEY
|
|
B2_S3_SECRET_KEY:
|
|
from_secret: B2_S3_SECRET_KEY
|
|
PGPASS:
|
|
from_secret: PGPASS
|
|
REPO_RO_TOKEN:
|
|
from_secret: REPO_RO_TOKEN
|
|
GITHUB_PAT:
|
|
from_secret: GITHUB_PAT
|
|
# normal env vars
|
|
GIT_USER: pat-s
|
|
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
|
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
|
CCACHE_DIR: /mnt/cache/ccache
|
|
NCPUS: 2
|
|
volumes:
|
|
- ${ARCH}-binaries-r-dep-cache-${OS}-${OS_VERSION//./}:/mnt/cache
|
|
commands:
|
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
|
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
|
- XVFB=$(command -v xwfb-run 2>/dev/null || command -v xvfb-run); XVFB_ARGS=""; if command -v xwfb-run >/dev/null 2>&1; then dnf install -y -q weston 2>/dev/null; XVFB_ARGS="-c weston"; fi
|
|
- $XVFB $XVFB_ARGS -n $SPLIT_INDEX -- R -q -e "sink(stdout(), type = 'message'); options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = readRDS('/mnt/cache/packages/pkgs_to_build.rds'); sprintf('Total# of remaining package versions:%s', nrow(pkgs)); chunks = split(pkgs, cut(seq_len(nrow(pkgs)), $SPLIT_INTO, labels = FALSE)); chunk = chunks[[$SPLIT_INDEX]]; sprintf('# of package versions for this job:%s', nrow(chunk)); exclude = c('RInno', 'RcmdrPlugin.DoE', 'RcmdrPlugin.NMBU', 'RcmdrPlugin.ROC', 'RcmdrPlugin.RiskDemo', 'RcmdrPlugin.TeachStat', 'RcmdrPlugin.TeachingDemos', 'RcmdrPlugin.UCA', 'RcmdrPlugin.WorldFlora', 'RcmdrPlugin.aRnova', 'RcmdrPlugin.depthTools', 'RcmdrPlugin.orloca', 'RcmdrPlugin.sos', 'RcmdrPlugin.survival', 'RcmdrPlugin.temis'); chunk = chunk[!chunk\$Package %in% exclude, ]; library(future); plan('sequential'); n = nrow(chunk); mapply(function(pkg, ver, i) { cat(sprintf('[%d/%d] %s_%s\n', i, n, pkg, ver)); bincraft::build_binary_package(pkg, tag = ver, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE) }, chunk\$Package, chunk\$Version, seq_len(n))" 2>&1
|
|
# archive missed packages
|
|
- R -q -e "bincraft::process_unarchived_pkgs(paste(Sys.getenv('OS'), Sys.getenv('OS_VERSION')), Sys.getenv('ARCH'), workers = $NCPUS)"
|
|
backend_options:
|
|
docker:
|
|
resources:
|
|
requests:
|
|
memory: 5Gi
|
|
cpu: 1000m
|
|
limits:
|
|
memory: 20Gi
|
|
cpu: 2000m
|
|
kubernetes:
|
|
resources:
|
|
requests:
|
|
memory: 5Gi
|
|
cpu: 1000m
|
|
limits:
|
|
memory: 20Gi
|
|
cpu: 2000m
|
|
nodeSelector:
|
|
kubernetes.io/arch: ${ARCH}
|
|
node.kubernetes.io/instance-type: ${K8S_INSTANCE_TYPE}
|
|
tolerations:
|
|
- key: 'CI'
|
|
operator: 'Equal'
|
|
value: 'true'
|
|
effect: 'NoSchedule'
|