From 7797efc8c533c73fa39c3521e3e962ff2ccd9b80 Mon Sep 17 00:00:00 2001 From: pat-s Date: Tue, 5 Nov 2024 17:20:44 +0100 Subject: [PATCH] fix: fix tag parsing for 'latest' --- .woodpecker/build-alpine-320-amd64.yaml | 46 +++++++++---------- .woodpecker/build-redhat-8-amd64.yaml | 46 +++++++++---------- .woodpecker/build-redhat-9-amd64.yaml | 56 ++++++++++++------------ .woodpecker/build-ubuntu-2204-amd64.yaml | 48 ++++++++++---------- .woodpecker/build-ubuntu-2404-amd64.yaml | 48 ++++++++++---------- R/build_binaries.R | 23 +++++----- 6 files changed, 134 insertions(+), 133 deletions(-) diff --git a/.woodpecker/build-alpine-320-amd64.yaml b/.woodpecker/build-alpine-320-amd64.yaml index 3adae99..2196c09 100644 --- a/.woodpecker/build-alpine-320-amd64.yaml +++ b/.woodpecker/build-alpine-320-amd64.yaml @@ -9,29 +9,29 @@ when: skip_clone: true steps: - # - name: 'Warming up' - # image: busybox:latest - # failure: ignore - # commands: - # - sleep 300 - # - echo "Warmed Up" - # backend_options: - # kubernetes: - # resources: - # requests: - # memory: 14Gi - # cpu: 2000m - # limits: - # memory: 14Gi - # cpu: 2000m - # nodeSelector: - # kubernetes.io/arch: amd64 - # node.kubernetes.io/instance-type: ccx23 - # tolerations: - # - key: 'CI' - # operator: 'Equal' - # value: 'true' - # effect: 'NoSchedule' + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 14Gi + cpu: 2000m + limits: + memory: 14Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: ccx23 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' - name: 'Build binaries' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] diff --git a/.woodpecker/build-redhat-8-amd64.yaml b/.woodpecker/build-redhat-8-amd64.yaml index 79ce68b..275b8c8 100644 --- a/.woodpecker/build-redhat-8-amd64.yaml +++ b/.woodpecker/build-redhat-8-amd64.yaml @@ -9,29 +9,29 @@ when: skip_clone: true steps: - # - name: 'Warming up' - # image: busybox:latest - # failure: ignore - # commands: - # - sleep 300 - # - echo "Warmed Up" - # backend_options: - # kubernetes: - # resources: - # requests: - # memory: 14Gi - # cpu: 2000m - # limits: - # memory: 14Gi - # cpu: 2000m - # nodeSelector: - # kubernetes.io/arch: amd64 - # node.kubernetes.io/instance-type: ccx23 - # tolerations: - # - key: 'CI' - # operator: 'Equal' - # value: 'true' - # effect: 'NoSchedule' + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 14Gi + cpu: 2000m + limits: + memory: 14Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: ccx23 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' - name: 'Build binaries' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] diff --git a/.woodpecker/build-redhat-9-amd64.yaml b/.woodpecker/build-redhat-9-amd64.yaml index b7e8cc0..f93cbbc 100644 --- a/.woodpecker/build-redhat-9-amd64.yaml +++ b/.woodpecker/build-redhat-9-amd64.yaml @@ -9,29 +9,29 @@ when: skip_clone: true steps: - - name: 'Warming up' - image: busybox:latest - failure: ignore - commands: - - sleep 300 - - echo "Warmed Up" - backend_options: - kubernetes: - resources: - requests: - memory: 14Gi - cpu: 2000m - limits: - memory: 14Gi - cpu: 2000m - nodeSelector: - kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: ccx23 - tolerations: - - key: 'CI' - operator: 'Equal' - value: 'true' - effect: 'NoSchedule' + # - name: 'Warming up' + # image: busybox:latest + # failure: ignore + # commands: + # - sleep 300 + # - echo "Warmed Up" + # backend_options: + # kubernetes: + # resources: + # requests: + # memory: 14Gi + # cpu: 2000m + # limits: + # memory: 14Gi + # cpu: 2000m + # nodeSelector: + # kubernetes.io/arch: amd64 + # node.kubernetes.io/instance-type: ccx23 + # tolerations: + # - key: 'CI' + # operator: 'Equal' + # value: 'true' + # effect: 'NoSchedule' - name: 'Build binaries' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] @@ -50,10 +50,10 @@ steps: ARCH: amd64 NCPUS: 2 MEMORY: 14Gi - INSTANCE_TYPE: ccx23 + INSTANCE_TYPE: AX42 STRATEGY: sequential - volumes: - - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache + # volumes: + # - amd64-binaries-r-dep-cache-redhat-9:/mnt/cache commands: # download latest package version (avoid having to rebuild image every time) # rscript_startup arg: required to pass down global option to future workers https://github.com/HenrikBengtsson/future/issues/134#issuecomment-2245666169 @@ -107,7 +107,7 @@ steps: ### PROD-parallel resources: requests: - memory: 14Gi + memory: 5Gi cpu: 2000m limits: memory: 14Gi @@ -130,7 +130,7 @@ steps: # cpu: 2000m nodeSelector: kubernetes.io/arch: amd64 - node.kubernetes.io/instance-type: ccx23 + node.kubernetes.io/instance-type: AX42 tolerations: - key: 'CI' operator: 'Equal' diff --git a/.woodpecker/build-ubuntu-2204-amd64.yaml b/.woodpecker/build-ubuntu-2204-amd64.yaml index 1422d22..44b6257 100644 --- a/.woodpecker/build-ubuntu-2204-amd64.yaml +++ b/.woodpecker/build-ubuntu-2204-amd64.yaml @@ -9,33 +9,33 @@ when: skip_clone: true steps: - # - name: 'Warming up' - # image: busybox:latest - # failure: ignore - # commands: - # - sleep 300 - # - echo "Warmed Up" - # backend_options: - # kubernetes: - # resources: - # requests: - # memory: 14Gi - # cpu: 2000m - # limits: - # memory: 14Gi - # cpu: 2000m - # nodeSelector: - # kubernetes.io/arch: amd64 - # node.kubernetes.io/instance-type: ccx23 - # tolerations: - # - key: 'CI' - # operator: 'Equal' - # value: 'true' - # effect: 'NoSchedule' + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 14Gi + cpu: 2000m + limits: + memory: 14Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: ccx23 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' - name: 'Build binaries' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] - image: docker.io/devxygmbh/amd64-binaries-r-amd64 + image: docker.io/devxygmbh/amd64-binaries-r-ubuntu-2204 pull: true # to ensure the PACKAGES upload step is being executed failure: ignore diff --git a/.woodpecker/build-ubuntu-2404-amd64.yaml b/.woodpecker/build-ubuntu-2404-amd64.yaml index 7e06acf..1fcfb5c 100644 --- a/.woodpecker/build-ubuntu-2404-amd64.yaml +++ b/.woodpecker/build-ubuntu-2404-amd64.yaml @@ -9,29 +9,29 @@ when: skip_clone: true steps: - # - name: 'Warming up' - # image: busybox:latest - # failure: ignore - # commands: - # - sleep 300 - # - echo "Warmed Up" - # backend_options: - # kubernetes: - # resources: - # requests: - # memory: 14Gi - # cpu: 2000m - # limits: - # memory: 14Gi - # cpu: 2000m - # nodeSelector: - # kubernetes.io/arch: amd64 - # node.kubernetes.io/instance-type: ccx23 - # tolerations: - # - key: 'CI' - # operator: 'Equal' - # value: 'true' - # effect: 'NoSchedule' + - name: 'Warming up' + image: busybox:latest + failure: ignore + commands: + - sleep 300 + - echo "Warmed Up" + backend_options: + kubernetes: + resources: + requests: + memory: 14Gi + cpu: 2000m + limits: + memory: 14Gi + cpu: 2000m + nodeSelector: + kubernetes.io/arch: amd64 + node.kubernetes.io/instance-type: ccx23 + tolerations: + - key: 'CI' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' - name: 'Build binaries' secrets: [red_hat_dev_pw, aws_access_key_id, aws_secret_access_key, pgpass, git_ro_token, GITHUB_PAT] @@ -70,7 +70,7 @@ steps: # - Rsymphony/ROI.plugin.symphony/adea/PortfolioAnalytics -> can't install/link external SYMPHONY lib (only on RHEL, as for others syslib exists) # - Rglpk # biplotbootGUI: somehow ubuntu still hangs even when using xvfb-run - - xvfb-run R -q -e 'options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c("biplotbootGUI", "Rsymphony", "PortfolioAnalytics", "ROI.plugin.symphony", "adea", "RInno", "skedastic", "RPushbullet", "SimNPH", "SimDesign","simCAT", "SIAmodules", "shinytest2", "shinyIRT", "ShinyItemAnalysis", "shiny.benchmark", "scDIFtest", "RSP", "ROracle", "ROI.plugin.qpoases", "ROI.plugin.cplex", "RmecabKo", "rmarchingcubes", "RcppMeCab", "Rcplex", "ravetools", "Rbeast", "Rblpapi", "RcppEnsmallen", "qtkit", "priorCON", "prior3D", "PerFit", "outlierensembles", "nomnoml", "mxfda", "MultBiplotR", "mstDIF", "mdsr", "makepipe", "lordif", "KSgeneral", "kequate", "jstager", "jrt", "irtreliability", "irtQ", "irtpwr", "irtawsi", "GRShiny", "gpuR", "googletraffic", "giacR", "gamstransfer", "aflps", "flow", "filters", "faoutlier", "equateIRT", "EnrichIntersect", "EFA.dimensions", "DFIT", "D3mirt", "ctgdist", "CoTiMA", "ConvertPar", "clrng", "bscui", "BifactorIndicesCalculator", "autoFC", "airt")); library(bincraftR); future::plan("$STRATEGY", workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers("cli"); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = "amd64"))' + - xvfb-run R -q -e "options(crayon.enabled = TRUE, Ncpus = $NCPUS, future.globals.onReference = NULL); pkgs = tools::CRAN_package_db()[[1]]; pkgs = setdiff(pkgs, c('biplotbootGUI', 'Rsymphony', 'PortfolioAnalytics', 'ROI.plugin.symphony', 'adea', 'RInno', 'skedastic', 'RPushbullet', 'SimNPH', 'SimDesign','simCAT', 'SIAmodules', 'shinytest2', 'shinyIRT', 'ShinyItemAnalysis', 'shiny.benchmark', 'scDIFtest', 'RSP', 'ROracle', 'ROI.plugin.qpoases', 'ROI.plugin.cplex', 'RmecabKo', 'rmarchingcubes', 'RcppMeCab', 'Rcplex', 'ravetools', 'Rbeast', 'Rblpapi', 'RcppEnsmallen', 'qtkit', 'priorCON', 'prior3D', 'PerFit', 'outlierensembles', 'nomnoml', 'mxfda', 'MultBiplotR', 'mstDIF', 'mdsr', 'makepipe', 'lordif', 'KSgeneral', 'kequate', 'jstager', 'jrt', 'irtreliability', 'irtQ', 'irtpwr', 'irtawsi', 'GRShiny', 'gpuR', 'googletraffic', 'giacR', 'gamstransfer', 'aflps', 'flow', 'filters', 'faoutlier', 'equateIRT', 'EnrichIntersect', 'EFA.dimensions', 'DFIT', 'D3mirt', 'ctgdist', 'CoTiMA', 'ConvertPar', 'clrng', 'bscui', 'BifactorIndicesCalculator', 'autoFC', 'airt')); library(bincraftR); future::plan('$STRATEGY', workers = $NCPUS, rscript_startup = quote(options(crayon.enabled = TRUE))); options(progressr.enable = TRUE); progressr::handlers('cli'); progressr::handlers(global = TRUE); foo = lapply(pkgs[$BLOCK], function(x) build_binary_package(x, build_for_minor=FALSE, debug = FALSE, force = TRUE, platform = 'ubuntu-2404'))" ### Manual pkg builds # webshot2, tabledown, TestAnaAPP, trekcolors, Spectran, SimNPH, RPushbullet, SimDesign, simCAT, SIAmodules, shinytest2, ShinyItemAnalysis, shinyIRT, shiny.benchmark, scDIFtest, RSP, qtkit, PROsetta, prettifyAddins, PerFit, outlierensembles, nomnoml, mxfda, MultBiplotR, mstDIF, mdsr, makepipe, lordif, kequate, jstager, jrt, irtreliability, irtQ, irtpwr, irtGUI, irtawsi, GRShiny, GPCMlasso, googletraffic, giacR, flps, flow, faoutlier, equateIRT, EnrichIntersect, EFA.dimensions, DFIT, D3mirt, ctgdist, CoTiMA, ConvertPar, bscui, BifactorIndicesCalculator, autoFC, airt: wait for https://github.com/rstudio/r-system-requirements/pull/178 # RInno: Windows-only diff --git a/R/build_binaries.R b/R/build_binaries.R index ee7696e..c1bf3f4 100644 --- a/R/build_binaries.R +++ b/R/build_binaries.R @@ -72,22 +72,23 @@ build_binary_package <- function(package_name, tag = NULL, codename = NULL, gert::git_config_global_set("advice.detachedHead", "false") - if (tag == "latest" || is.null(tag)) { + if (is.null(tag) || tag == "latest") { gert::git_clone(sprintf("https://github.com/cran/%s", package_name), path = sprintf("%s/%s", tempdir(), "tmp1"), verbose = FALSE ) - # Retrieve all tags - all_tags <- gert::git_tag_list(repo = sprintf("%s/%s", tempdir(), "tmp1")) - # filter out tags that start with R- (= non-valid ones) - all_tags <- all_tags[!grepl("R-", all_tags$name), ] - - unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE) - tag <- all_tags$name # gert cannot sort by date (which is a problem for properly sorting tags like 1.0-10 and others) - if (tag == "latest") { - system("git tag --sort=-creatordate | head -1") - } + if (!is.null(tag) && tag == "latest") { + tag = system("git tag --sort=-creatordate | head -1", intern = TRUE) + } else { + # Retrieve all tags + all_tags <- gert::git_tag_list(repo = sprintf("%s/%s", tempdir(), "tmp1")) + # filter out tags that start with R- (= non-valid ones) + all_tags <- all_tags[!grepl("R-", all_tags$name), ] + + unlink(sprintf("%s/%s", tempdir(), "tmp1"), force = TRUE, recursive = TRUE) + tag <- all_tags$name + } package_name <- rep(package_name, length(tag)) }