pat-s
13d51ef4dd
process_cran_updates uploads fresh PACKAGES* to S3, but the CDN was only purged by the daily update-package-index-* chain. Between the two cron runs, edges kept serving the stale index, so a binary that had just landed in S3 was effectively invisible until the next index-update purge. Adds the same purge step that update-package-index-* already uses, parameterized per platform/arch.
73 lines
4.1 KiB
YAML
73 lines
4.1 KiB
YAML
when:
|
|
- event: [cron]
|
|
cron: process-cran-updates-ubuntu-2404-arm64
|
|
- event: manual
|
|
evaluate: 'process_cran_updates == "all"'
|
|
- event: manual
|
|
evaluate: 'process_cran_updates == "ubuntu-2404-arm64"'
|
|
|
|
skip_clone: true
|
|
|
|
labels:
|
|
agent: gaia
|
|
|
|
steps:
|
|
- name: 'Processing Updates'
|
|
image: reg.devxy.io/rpkgs/build-env-ubuntu:noble
|
|
pull: true
|
|
environment:
|
|
RED_HAT_DEV_PW:
|
|
from_secret: RED_HAT_DEV_PW
|
|
B2_S3_ACCESS_KEY:
|
|
from_secret: B2_S3_ACCESS_KEY
|
|
B2_S3_SECRET_KEY:
|
|
from_secret: B2_S3_SECRET_KEY
|
|
PGPASS:
|
|
from_secret: PGPASS
|
|
REPO_RO_TOKEN:
|
|
from_secret: REPO_RO_TOKEN
|
|
GITHUB_PAT:
|
|
from_secret: GITHUB_PAT
|
|
# normal env vars
|
|
GIT_USER: pat-s
|
|
NTFY_SERVER:
|
|
from_secret: NTFY_SERVER
|
|
NTFY_TOPIC:
|
|
from_secret: NTFY_TOPIC
|
|
NTFY_AUTH: TRUE
|
|
NTFY_PASSWORD:
|
|
from_secret: ntfy_token
|
|
# set the location of the 'pkgcache' cache dir which persists the R package dependencies needed to install the packages themselves
|
|
R_PKG_CACHE_DIR: /mnt/cache/pkgcache
|
|
R_LIBS_USER: /mnt/cache/R-pkgs
|
|
R_VERSION: 4.4.3
|
|
CCACHE_DIR: /mnt/cache/ccache
|
|
PLATFORM: ubuntu-2404
|
|
ARCH: arm64
|
|
NCPUS: 2
|
|
INTERVAL: lubridate::interval(lubridate::today() - 6, lubridate::today() - 3)
|
|
# volumes:
|
|
# - arm64-binaries-r-dep-cache-ubuntu-2404:/mnt/cache
|
|
commands:
|
|
- git clone -q https://pat-s:$$REPO_RO_TOKEN@git.devxy.io/devxy/build-cran-binaries.git .
|
|
- rm -rf /mnt/cache/R-pkgs/00LOCK-r-pkg-binaries /mnt/cache/R-pkgs/00LOCK-bincraft /mnt/cache/R-pkgs/00LOCK-pak /mnt/cache/pkgcache/R/pkgcache /mnt/cache/pkgcache/R/pkgcache /mnt/cache/R-pkgs/pkgcache /mnt/cache/R-pkgs/00LOCK-pak/mnt/cache/R-pkgs/00LOCK-RPostgres /mnt/cache/R-pkgs/bincraft
|
|
- /opt/R/$R_VERSION/bin/R -q -e 'pak::pak("git::https://codefloe.com/rpkgs/bincraft.git")'
|
|
- /opt/R/$R_VERSION/bin/R -q -e 'packageVersion("bincraft")'
|
|
- mkdir -p /mnt/cache/pkgcache /mnt/cache/R-pkgs /mnt/cache/ccache /mnt/cache/packages
|
|
# options(future.globals.onReference = NULL): for some reason s3fs::file_delete() throws 'Error: Detected a non-exportable reference ('externalptr') in one of the globals ('FUN' of class 'function') used in the future expression' otherwise
|
|
- xvfb-run /opt/R/$R_VERSION/bin/R -q -e "options(crayon.enabled = TRUE, Ncpus = 4, future.globals.onReference = 'error', repos = structure(c(getOption('repos'),INLA='https://inla.r-inla-download.org/R/stable'))); progressr::handlers('cli'); progressr::handlers(global = TRUE); options(future.globals.onReference = NULL); bincraft::process_cran_updates(interval = $INTERVAL, platform = 'ubuntu-2404', process_updated = TRUE, process_new = TRUE, process_removed = TRUE, s3_endpoint = 'https://s3.eu-central-003.backblazeb2.com', s3_region = 'eu-central-003', s3_bucket = 'devxy-rpkgs-binaries', s3_access_key_id = Sys.getenv('B2_S3_ACCESS_KEY'), s3_secret_access_key = Sys.getenv('B2_S3_SECRET_KEY'), metadata_db_host = 'r-binaries.devxy.io', metadata_db_name = 'build_metadata', metadata_db_table = 'single_builds', metadata_db_user = 'rpkgs', metadata_db_password = Sys.getenv('PGPASS'), metadata_db_sslmode = 'require', metadata_db_port = 15432, archive = TRUE, upload = TRUE, store_build_metadata = TRUE)"
|
|
- /opt/R/$R_VERSION/bin/R -q -e 'library(bincraft); upload_package_index(codename = "noble", s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", s3_region = "eu-central-003", s3_bucket = "devxy-rpkgs-binaries", s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"))'
|
|
|
|
- name: Purge CDN cache
|
|
image: reg.devxy.io/docker.io/library/alpine:3.23
|
|
environment:
|
|
BUNNYNET_API_KEY:
|
|
from_secret: BUNNYNET_API_KEY
|
|
SUBDOMAIN1: 'cran.devxy.io'
|
|
SUBDOMAIN2: 'cran.allianceswisspass.devxy.io'
|
|
SUBDOMAIN3: 'cran.rpkgs.com'
|
|
OS_ID: noble
|
|
ARCH: arm64
|
|
commands:
|
|
- apk add --no-cache -q bash curl
|
|
- bash scripts/purge_cdn_cache.sh "$BUNNYNET_API_KEY" "$ARCH" "$OS_ID" "$SUBDOMAIN1" "$SUBDOMAIN2" "$SUBDOMAIN3"
|