devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 10 Pull requests Releases Activity

feat: add script to infer which pkgs were built last

Browse source
This commit is contained in:
Patrick Schratz 2024-11-04 14:42:47 +01:00
commit 93f872a8eb
Signed by: pat-s
GPG key ID: 3C6318841EF78925
7 changed files with 177 additions and 168 deletions
Download patch file Download diff file Expand all files Collapse all files

83
.woodpecker/build-alpine-320-amd64.yaml
View file

@ -3,33 +3,35 @@ when:
evaluate: 'build == "all"'
- event: manual
evaluate: 'build == "alpine-320-amd64"'
- event: manual
evaluate: 'build == "all-amd64"'
skip_clone: true
steps:
- name: 'Warming up'
image: busybox:latest
failure: ignore
commands:
- sleep 300
- echo "Warmed Up"
backend_options:
kubernetes:
resources:
requests:
memory: 29Gi
cpu: 4000m
limits:
memory: 29Gi
cpu: 4000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: cpx51
tolerations:
- key: 'CI'
operator: 'Equal'
value: 'true'
effect: 'NoSchedule'
# - name: 'Warming up'
# image: busybox:latest
# failure: ignore
# commands:
# - sleep 300
# - echo "Warmed Up"
# backend_options:
# kubernetes:
# resources:
# requests:
# memory: 14Gi
# cpu: 2000m
# limits:
# memory: 14Gi
# cpu: 2000m
# nodeSelector:
# kubernetes.io/arch: amd64
# node.kubernetes.io/instance-type: ccx23
# tolerations:
# - key: 'CI'
# operator: 'Equal'
# value: 'true'
# effect: 'NoSchedule'
- name: 'Build binaries'
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
@ -47,9 +49,9 @@ steps:
BLOCK: 1581:2993
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
INSTANCE_TYPE: cpx51
STRATEGY: multisession
MEMORY: 14Gi
INSTANCE_TYPE: ccx23
STRATEGY: sequential
volumes:
- amd64-binaries-r-dep-cache-alpine-320:/mnt/cache
commands:
@ -102,35 +104,18 @@ steps:
# - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = c( "RcmdrPlugin.orloca", "RcmdrPlugin.PcaRobust", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.ROC", "RcmdrPlugin.sos" , "RcmdrPlugin.survival", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.TeachStat", "RcmdrPlugin.temis", "RcmdrPlugin.UCA", "RcmdrPlugin.WorldFlora", "streamDAG", "stan4bart", "snplinkage", "saeHB.panel.beta", "rTwig", "rswipl", "RPointCloud", "rPDBapi", "rolog", "rmcfs", "rjags", "ResultModelManager", "RcppCWB", "RcmdrPlugin.WorldFlora", "RcmdrPlugin.UCA", "RcmdrPlugin.temis", "RcmdrPlugin.TeachStat", "RcmdrPlugin.TeachingDemos", "RcmdrPlugin.survival", "RcmdrPlugin.sos", "RcmdrPlugin.ROC", "RcmdrPlugin.RMTCJags", "RcmdrPlugin.RiskDemo", "RcmdrPlugin.PcaRobust", "RclusTool", "Rbeast", "rayrender", "quanteda.textmodels", "polmineR", "plotBart", "NlinTS", "ndtv", "overtureR", "mathml", "magickGUI", "lgpr", "KappaGUI", "historicalborrow", "GSODR", "GRIDCOPULA", "googleAnalyticsR", "fluidsynth", "EventDetectGUI", "evalITR", "EBcoBART", "deeptrafo", "dbarts", "CRE", "countSTAR", "bartMan", "bartCause", "altmeta", "EthSEQ", "dartR.spatial", "dartR.sim", "gwid", "Mega2R", "priorCON", "prior3D", "MCARtest", "designmatch", "highs", "ROI.plugin.highs", "GPCMlasso"); library(bincraftR); options(progressr.enable = TRUE); foo = lapply(pkgs, function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "amd64"))'
backend_options:
kubernetes:
### PROD-parallel
resources:
requests:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
limits:
memory: 29Gi
cpu: 4000m
### PROD-sequential
# resources:
# requests:
# memory: 29Gi
# cpu: 2000m
# limits:
# memory: 29Gi
# cpu: 4000m
# DEBUG
# resources:
# requests:
# memory: 4Gi
# cpu: 2000m
# limits:
# memory: 4Gi
# cpu: 2000m
memory: 14Gi
cpu: 2000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: cpx51
node.kubernetes.io/instance-type: ccx23
tolerations:
- key: 'CI'
operator: 'Equal'
value: 'true'
effect: 'NoSchedule'
effect: 'NoSchedule'

68
.woodpecker/build-redhat-8-amd64.yaml
View file

@ -3,33 +3,35 @@ when:
evaluate: 'build == "all"'
- event: manual
evaluate: 'build == "redhat-8-amd64"'
- event: manual
evaluate: 'build == "all-amd64"'
skip_clone: true
steps:
- name: 'Warming up'
image: busybox:latest
failure: ignore
commands:
- sleep 300
- echo "Warmed Up"
backend_options:
kubernetes:
resources:
requests:
memory: 29Gi
cpu: 4000m
limits:
memory: 29Gi
cpu: 4000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: cpx51
tolerations:
- key: 'CI'
operator: 'Equal'
value: 'true'
effect: 'NoSchedule'
# - name: 'Warming up'
# image: busybox:latest
# failure: ignore
# commands:
# - sleep 300
# - echo "Warmed Up"
# backend_options:
# kubernetes:
# resources:
# requests:
# memory: 14Gi
# cpu: 2000m
# limits:
# memory: 14Gi
# cpu: 2000m
# nodeSelector:
# kubernetes.io/arch: amd64
# node.kubernetes.io/instance-type: ccx23
# tolerations:
# - key: 'CI'
# operator: 'Equal'
# value: 'true'
# effect: 'NoSchedule'
- name: 'Build binaries'
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
@ -47,9 +49,9 @@ steps:
BLOCK: 228:1833
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
INSTANCE_TYPE: cpx51
STRATEGY: multisession
MEMORY: 14Gi
INSTANCE_TYPE: ccx23
STRATEGY: sequential
volumes:
- amd64-binaries-r-dep-cache-redhat-8:/mnt/cache
commands:
@ -105,18 +107,18 @@ steps:
### PROD-parallel
resources:
requests:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
limits:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
### PROD-sequential
# resources:
# requests:
# memory: 29Gi
# memory: 14Gi
# cpu: 2000m
# limits:
# memory: 29Gi
# memory: 14Gi
# cpu: 4000m
# DEBUG
# resources:
@ -128,7 +130,7 @@ steps:
# cpu: 2000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: cpx51
node.kubernetes.io/instance-type: ccx23
tolerations:
- key: 'CI'
operator: 'Equal'

32
.woodpecker/build-redhat-9-amd64.yaml
View file

@ -3,6 +3,8 @@ when:
evaluate: 'build == "all"'
- event: manual
evaluate: 'build == "redhat-9-amd64"'
- event: manual
evaluate: 'build == "all-amd64"'
skip_clone: true
@ -17,14 +19,14 @@ steps:
kubernetes:
resources:
requests:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
limits:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: cpx51
node.kubernetes.io/instance-type: ccx23
tolerations:
- key: 'CI'
operator: 'Equal'
@ -47,9 +49,9 @@ steps:
BLOCK: 715:2428
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
INSTANCE_TYPE: cpx51
STRATEGY: multisession
MEMORY: 14Gi
INSTANCE_TYPE: ccx23
STRATEGY: sequential
volumes:
- amd64-binaries-r-dep-cache-redhat-9:/mnt/cache
commands:
@ -105,18 +107,18 @@ steps:
### PROD-parallel
resources:
requests:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
limits:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
### PROD-sequential
# resources:
# requests:
# memory: 29Gi
# memory: 14Gi
# cpu: 2000m
# limits:
# memory: 29Gi
# memory: 14Gi
# cpu: 4000m
# DEBUG
# resources:
@ -128,7 +130,7 @@ steps:
# cpu: 2000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: cpx51
node.kubernetes.io/instance-type: ccx23
tolerations:
- key: 'CI'
operator: 'Equal'

68
.woodpecker/build-ubuntu-2204-amd64.yaml
View file

@ -3,33 +3,35 @@ when:
# evaluate: 'build == "all"'
- event: manual
evaluate: 'build == "ubuntu-2204-amd64"'
- event: manual
evaluate: 'build == "all-amd64"'
skip_clone: true
steps:
- name: 'Warming up'
image: busybox:latest
failure: ignore
commands:
- sleep 300
- echo "Warmed Up"
backend_options:
kubernetes:
resources:
requests:
memory: 29Gi
cpu: 4000m
limits:
memory: 29Gi
cpu: 4000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: cpx51
tolerations:
- key: 'CI'
operator: 'Equal'
value: 'true'
effect: 'NoSchedule'
# - name: 'Warming up'
# image: busybox:latest
# failure: ignore
# commands:
# - sleep 300
# - echo "Warmed Up"
# backend_options:
# kubernetes:
# resources:
# requests:
# memory: 14Gi
# cpu: 2000m
# limits:
# memory: 14Gi
# cpu: 2000m
# nodeSelector:
# kubernetes.io/arch: amd64
# node.kubernetes.io/instance-type: ccx23
# tolerations:
# - key: 'CI'
# operator: 'Equal'
# value: 'true'
# effect: 'NoSchedule'
- name: 'Build binaries'
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
@ -47,9 +49,9 @@ steps:
BLOCK: 1913:4755
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
INSTANCE_TYPE: cpx51
STRATEGY: multisession
MEMORY: 14Gi
INSTANCE_TYPE: ccx23
STRATEGY: sequential
volumes:
- amd64-binaries-r-dep-cache-ubuntu-2204:/mnt/cache
commands:
@ -105,18 +107,18 @@ steps:
### PROD-parallel
resources:
requests:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
limits:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
### PROD-sequential
# resources:
# requests:
# memory: 29Gi
# memory: 14Gi
# cpu: 2000m
# limits:
# memory: 29Gi
# memory: 14Gi
# cpu: 4000m
# DEBUG
# resources:
@ -128,7 +130,7 @@ steps:
# cpu: 2000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: cpx51
node.kubernetes.io/instance-type: ccx23
tolerations:
- key: 'CI'
operator: 'Equal'

68
.woodpecker/build-ubuntu-2404-amd64.yaml
View file

@ -3,33 +3,35 @@ when:
# evaluate: 'build == "all"'
- event: manual
evaluate: 'build == "ubuntu-2404-amd64"'
- event: manual
evaluate: 'build == "all-amd64"'
skip_clone: true
steps:
- name: 'Warming up'
image: busybox:latest
failure: ignore
commands:
- sleep 300
- echo "Warmed Up"
backend_options:
kubernetes:
resources:
requests:
memory: 29Gi
cpu: 4000m
limits:
memory: 29Gi
cpu: 4000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: cpx51
tolerations:
- key: 'CI'
operator: 'Equal'
value: 'true'
effect: 'NoSchedule'
# - name: 'Warming up'
# image: busybox:latest
# failure: ignore
# commands:
# - sleep 300
# - echo "Warmed Up"
# backend_options:
# kubernetes:
# resources:
# requests:
# memory: 14Gi
# cpu: 2000m
# limits:
# memory: 14Gi
# cpu: 2000m
# nodeSelector:
# kubernetes.io/arch: amd64
# node.kubernetes.io/instance-type: ccx23
# tolerations:
# - key: 'CI'
# operator: 'Equal'
# value: 'true'
# effect: 'NoSchedule'
- name: 'Build binaries'
secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT]
@ -47,9 +49,9 @@ steps:
BLOCK: 1813:4298
ARCH: amd64
NCPUS: 2
MEMORY: 29Gi
INSTANCE_TYPE: cpx51
STRATEGY: multisession
MEMORY: 14Gi
INSTANCE_TYPE: ccx23
STRATEGY: sequential
volumes:
- amd64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache
commands:
@ -105,18 +107,18 @@ steps:
### PROD-parallel
resources:
requests:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
limits:
memory: 29Gi
cpu: 4000m
memory: 14Gi
cpu: 2000m
### PROD-sequential
# resources:
# requests:
# memory: 29Gi
# memory: 14Gi
# cpu: 2000m
# limits:
# memory: 29Gi
# memory: 14Gi
# cpu: 4000m
# DEBUG
# resources:
@ -128,7 +130,7 @@ steps:
# cpu: 2000m
nodeSelector:
kubernetes.io/arch: amd64
node.kubernetes.io/instance-type: cpx51
node.kubernetes.io/instance-type: ccx23
tolerations:
- key: 'CI'
operator: 'Equal'

10
.woodpecker/process-updates-redhat-8-arm64.yaml
View file

@ -19,10 +19,10 @@ steps:
kubernetes:
resources:
requests:
memory: 14Gi
memory: 13Gi
cpu: 4000m
limits:
memory: 14Gi
memory: 13Gi
cpu: 4000m
nodeSelector:
kubernetes.io/arch: arm64
@ -46,7 +46,7 @@ steps:
PLATFORM: redhat-8
ARCH: arm64
NCPUS: 2
MEMORY: 14Gi
MEMORY: 13Gi
STRATEGY: sequential
INSTANCE_TYPE: cax31
INTERVAL: lubridate::interval(lubridate::today() - 3, lubridate::today() - 3)
@ -69,10 +69,10 @@ steps:
### PROD
resources:
requests:
memory: 14Gi
memory: 13Gi
cpu: 4000m
limits:
memory: 14Gi
memory: 13Gi
cpu: 4000m
# DEBUG
# resources:

16
local/last-processed-by-platform.R Normal file
View file

@ -0,0 +1,16 @@
devtools::load_all()
library(dplyr)
### Show last processed package by each platform/arch
query_metadata_table() |>
group_by(platform, arch) |>
arrange(desc(timestamp)) |>
select(name, timestamp) |>
filter(row_number()==1)
### Query index of package in the CRAN DB (which is sorted alphabetically)
pkgs <- tools::CRAN_package_db()$Package
pkgs <- setdiff(pkgs, c("biplotbootGUI", "Rsymphony", "PortfolioAnalytics", "ROI.plugin.symphony", "adea", "RInno", "skedastic", "RPushbullet", "SimNPH", "SimDesign","simCAT", "SIAmodules", "shinytest2", "shinyIRT", "ShinyItemAnalysis", "shiny.benchmark", "scDIFtest", "RSP", "ROracle", "ROI.plugin.qpoases", "ROI.plugin.cplex", "RmecabKo", "rmarchingcubes", "RcppMeCab", "Rcplex", "ravetools", "Rbeast", "Rblpapi", "RcppEnsmallen", "qtkit", "priorCON", "prior3D", "PerFit", "outlierensembles", "nomnoml", "mxfda", "MultBiplotR", "mstDIF", "mdsr", "makepipe", "lordif", "KSgeneral", "kequate", "jstager", "jrt", "irtreliability", "irtQ", "irtpwr", "irtawsi", "GRShiny", "gpuR", "googletraffic", "giacR", "gamstransfer", "flps", "flow", "filters", "faoutlier", "equateIRT", "EnrichIntersect", "EFA.dimensions", "DFIT", "D3mirt", "ctgdist", "CoTiMA", "ConvertPar", "clrng", "bscui", "BifactorIndicesCalculator", "autoFC", "airt"))
which(pkgs == "urca")
pkgs[19121:19122]