From a9ad907f9c71051fe5a44a6f9efd15fabaccfad7 Mon Sep 17 00:00:00 2001 From: pat-s Date: Sun, 14 Jun 2026 13:02:48 +0200 Subject: [PATCH] chore(local): add S3 migration and CRAN-source helper scripts - migrate-s3-hetzner-to-backblaze.sh: rclone-based bucket migration helper. - find-R-api-packages.sh: scan CRAN package sources for a C-API usage pattern. - query-pkgs-without-old-versions.R, test-package-loading.R: ad-hoc helpers. --- local/find-R-api-packages.sh | 34 +++ local/migrate-s3-hetzner-to-backblaze.sh | 90 ++++++ local/query-pkgs-without-old-versions.R | 39 +++ local/test-package-loading.R | 331 +++++++++++++++++++++++ 4 files changed, 494 insertions(+) create mode 100644 local/find-R-api-packages.sh create mode 100644 local/migrate-s3-hetzner-to-backblaze.sh create mode 100644 local/query-pkgs-without-old-versions.R create mode 100644 local/test-package-loading.R diff --git a/local/find-R-api-packages.sh b/local/find-R-api-packages.sh new file mode 100644 index 0000000..1f36961 --- /dev/null +++ b/local/find-R-api-packages.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Directory to clone repos into +WORKDIR="cran_repos" +mkdir -p "$WORKDIR" +cd "$WORKDIR" + +# GitHub API paginates results, so we loop through pages +PAGE=1 +PER_PAGE=100 +MATCHES=() + +while :; do + # Fetch a page of repos + REPOS=$(curl -s "https://api.github.com/orgs/cran/repos?per_page=$PER_PAGE&page=$PAGE" | jq -r '.[].clone_url') + [ -z "$REPOS" ] && break + + for REPO_URL in $REPOS; do + REPO_NAME=$(basename "$REPO_URL" .git) + # Skip if already cloned + [ -d "$REPO_NAME" ] && continue + git clone --depth 1 "$REPO_URL" "$REPO_NAME" >/dev/null 2>&1 + if [ -d "$REPO_NAME/src" ]; then + # Search for Rinternals.h in src/ + if grep -r -q 'R_VERSION < R_Version(' "$REPO_NAME/src"; then + echo "$REPO_NAME" + fi + fi + # Clean up to save space + rm -rf "$REPO_NAME" + done + + PAGE=$((PAGE + 1)) +done diff --git a/local/migrate-s3-hetzner-to-backblaze.sh b/local/migrate-s3-hetzner-to-backblaze.sh new file mode 100644 index 0000000..8cbd829 --- /dev/null +++ b/local/migrate-s3-hetzner-to-backblaze.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Migrate S3 buckets from Hetzner Object Storage to Backblaze B2 via rclone. +# +# Prerequisites: +# 1. Install rclone: https://rclone.org/install/ +# 2. Configure two rclone remotes: +# rclone config create hetzner s3 \ +# provider=Other \ +# env_auth=false \ +# access_key_id=YOUR_HETZNER_KEY \ +# secret_access_key=YOUR_HETZNER_SECRET \ +# endpoint=fsn1.your-objectstorage.com # adjust region +# +# rclone config create backblaze s3 \ +# provider=Other \ +# env_auth=false \ +# access_key_id=YOUR_B2_KEY \ +# secret_access_key=YOUR_B2_APP_KEY \ +# endpoint=s3.us-west-004.backblazeb2.com # adjust region +# +# Usage: +# ./migrate-s3-hetzner-to-backblaze.sh [src:dst] ... +# ./migrate-s3-hetzner-to-backblaze.sh hetzner-bucket:backblaze-bucket + +HETZNER_REMOTE="${HETZNER_REMOTE:-hetzner}" +BACKBLAZE_REMOTE="${BACKBLAZE_REMOTE:-backblaze}" +RCLONE_FLAGS="${RCLONE_FLAGS:---transfers=64 --checkers=64 --fast-list}" + +if [[ $# -eq 0 ]]; then + echo "Usage: $0 [src-bucket:dst-bucket...]" + echo "" + echo " Each argument is a source:destination bucket pair separated by a colon." + echo "" + echo "Environment variables:" + echo " HETZNER_REMOTE rclone remote name for Hetzner (default: hetzner)" + echo " BACKBLAZE_REMOTE rclone remote name for Backblaze (default: backblaze)" + echo " RCLONE_FLAGS extra rclone flags (default: --transfers=16 --checkers=16 --fast-list)" + echo " DRY_RUN=1 show what would be copied without copying" + exit 1 +fi + +for cmd in rclone; do + if ! command -v "$cmd" &>/dev/null; then + echo "Error: $cmd is not installed." >&2 + exit 1 + fi +done + +# Verify remotes exist +for remote in "$HETZNER_REMOTE" "$BACKBLAZE_REMOTE"; do + if ! rclone listremotes | grep -q "^${remote}:$"; then + echo "Error: rclone remote '${remote}' not found. Run 'rclone config' to set it up." >&2 + exit 1 + fi +done + +DRY_RUN_FLAG="" +if [[ "${DRY_RUN:-0}" == "1" ]]; then + DRY_RUN_FLAG="--dry-run" + echo "=== DRY RUN MODE ===" +fi + +for pair in "$@"; do + if [[ "$pair" != *:* ]]; then + echo "Error: '$pair' is not a valid src:dst pair. Use format 'hetzner-bucket:backblaze-bucket'." >&2 + exit 1 + fi + + src_bucket="${pair%%:*}" + dst_bucket="${pair#*:}" + src="${HETZNER_REMOTE}:${src_bucket}" + dst="${BACKBLAZE_REMOTE}:${dst_bucket}" + + echo "" + echo "--- Migrating: ${src} -> ${dst} ---" + + # shellcheck disable=SC2086 + rclone sync \ + ${RCLONE_FLAGS} \ + ${DRY_RUN_FLAG} \ + --progress \ + "$src" "$dst" + + echo "--- Done: ${src_bucket} -> ${dst_bucket} ---" +done + +echo "" +echo "Migration complete." diff --git a/local/query-pkgs-without-old-versions.R b/local/query-pkgs-without-old-versions.R new file mode 100644 index 0000000..29ba891 --- /dev/null +++ b/local/query-pkgs-without-old-versions.R @@ -0,0 +1,39 @@ +library(s3fs) + +# List all files under contrib// +all_files <- s3fs::s3_dir_ls( + "s3://devxy-r-package-binaries-hel1/arm64/alpine322/latest/src/contrib/", + recurse = TRUE, + type = "file" +) + +pattern <- ".*/src/contrib/([^/_]+)_.*" +matches <- regmatches(all_files, regexec(pattern, all_files)) +pkg_names <- unique( + vapply( + matches, + function(x) if (length(x) > 1) x[2] else NA_character_, + character(1) + ) +) +pkg_names <- pkg_names[!is.na(pkg_names)] + +# For each package, check if Archive// contains any files +no_archive_files <- character(0) +for (pkg in pkg_names) { + archive_dir1 <- sprintf( + "s3://devxy-r-package-binaries-hel1/arm64/alpine322/latest/src/contrib/Archive/%s", + pkg + ) + archive_files <- unique(c( + tryCatch( + s3fs::s3_dir_ls(archive_dir1, recurse = TRUE), + error = function(e) character(0) + ) + )) + if (length(archive_files) == 0) { + no_archive_files <- c(no_archive_files, pkg) + } +} + +print(no_archive_files) diff --git a/local/test-package-loading.R b/local/test-package-loading.R new file mode 100644 index 0000000..26a2e82 --- /dev/null +++ b/local/test-package-loading.R @@ -0,0 +1,331 @@ +install.packages( + "pak", + repos = sprintf( + "https://r-lib.github.io/p/pak/stable/%s/%s/%s", + .Platform$pkgType, + R.Version()$os, + R.Version()$arch + ) +) + +Sys.setenv(PKG_SYSREQS = TRUE) +all_pkgs <- rownames(available.packages()) + +to_skip = c("ABRSQOL", "ACA", "ACE.CoCo") +all_pkgs = setdiff(all_pkgs, to_skip) + +for (i in all_pkgs) { + message(sprintf("\nInstalling %s", i)) + pak::pkg_install(i) + library(i, character.only = TRUE) +} + + +# Example data +all_pkgs <- rownames(available.packages()) +to_skip <- c( + "ABRSQOL", + "ACA", + "ACE.CoCo", + "ACEsimFit", + "ACNE", + "absorber", + "adapt4pv", + "adaptMCMC", + "addhaz", + "ADDT", + "ahaz", + "arm", + "arules", + "arulesCBA", + "aster2", + "BayesFactor", + "bc3net", + "bgsmtr", + "biglasso", + "BinNonNor", + "BinNor", + "bioassayR", + "birankr", + "BiRewire", + "bolasso", + "Boptbd", + "Brobdingnag", + "BSW", + "BTLLasso", + "bvartools", + "cAIC4", + "Category", + "celda", + "centiserve", + "cjoint", + "clinical", + "clipper", + "CodataGS", + "conos", + "CopulaInference", + "covEB", + "cplm", + "CRTgeeDR", + "cthreshER", + "ctmcmove", + "curephEM", + "CVST", + "dcGSA", + "dclone", + "dcsvm", + "DelayedArray", + "dglars", + "dhglm", + "disordR", + "distrom", + "dmm", + "DNABarcodes", + "DoubleCone", + "DRR", + "DTRlearn2", + "DWDLargeR", + "eds", + "EMCluster", + "EMMREML", + "evalITR", + "EventPointer", + "evola", + "excursions", + "expm", + "fanc", + "FAS", + "fastadi", + "fastPLS", + "fastRG", + "fdaPDE", + "flare", + "FoReco", + "frailtyHL", + "freebird", + "FSTpackage", + "gamlr", + "gamlss.lasso", + "gamm4", + "gbmt", + "gdim", + "gdistance", + "GeDS", + "geeM", + "genlasso", + "GenOrd", + "GenoScan", + "geomorph", + "geostatsp", + "GhostKnockoff", + "GIGSEA", + "GlarmaVarSel", + "glmm", + "glmmrBase", + "glmmrOptim", + "glmnet", + "glober", + "GPvam", + "graphpcor", + "gremlin", + "growthrate", + "grpCox", + "GSD", + "HelpersMG", + "hglm", + "hglm.data", + "hibayes", + "hierSDR", + "HMTL", + "hsem", + "ibmdbR", + "inca", + "INLAspacetime", + "INLAtools", + "invertiforms", + "irlba", + "islasso", + "ISLET", + "isotonic.pen", + "jordan", + "kinship2", + "KnockoffScreen", + "lcpm", + "leidenAlg", + "lfe", + "lingmatch", + "LKT", + "lme4", + "lme4breeding", + "lme4GS", + "logcondiscr", + "LPmerge", + "LRMF3", + "MAP", + "marcox", + "markovchain", + "MatrixExtra", + "matter", + "MBC", + "mcen", + "mclogit", + "MCMCglmm", + "mdhglm", + "MDPtoolbox", + "mediation", + "mefa4", + "metafor", + "mgwrsar", + "mi", + "midasml", + "mind", + "monocle", + "msda", + "MuData", + "MultiGlarmaVarSel", + "MultiOrd", + "MultiVarSel", + "mvglmmRank", + "N2R", + "nadiv", + "NBtsVarSel", + "NegBinBetaBinreg", + "NetworkRiskMeasures", + "neuroim2", + "NOISeq", + "numbat", + "optbdmaeAT", + "optimbase", + "OptimModel", + "optrcdmaeAT", + "OrdNor", + "pagoda2", + "PCovR", + "pedgene", + "pedigree", + "pedigreemm", + "pense", + "PERMANOVA", + "phateR", + "PhylogeneticEM", + "pleio", + "POINT", + "PoisBinNonNor", + "PoisBinOrd", + "PoisBinOrdNonNor", + "PoisBinOrdNor", + "PoisNonNor", + "PoisNor", + "PRISMA", + "ProbitSpatial", + "prodest", + "psqn", + "qlcMatrix", + "qpcR", + "QRM", + "quadrupen", + "QZ", + "ramps", + "randnet", + "randPedPCA", + "rBMF", + "RCBR", + "RealVAMS", + "REBayes", + "recommenderlab", + "Rediscover", + "reglogit", + "RESET", + "RGE", + "RGENERATEPREC", + "riemtan", + "RNewsflow", + "robustlmm", + "rsparse", + "rSPDE", + "rwc", + "S4Arrays", + "saeMSPE", + "sbw", + "scITD", + "scoup", + "sdwd", + "SEAGLE", + "sensory", + "serrsBayes", + "sglasso", + "sharpPen", + "SiPSiC", + "SKAT", + "snpReady", + "snpStats", + "softImpute", + "sommer", + "soptdmaeA", + "SOR", + "SparseArray", + "SparseChol", + "sparseLRMatrix", + "sparsenet", + "sparsenetgls", + "sparsestep", + "spatialprobit", + "spatialreg", + "spatstat.sparse", + "speedglm", + "sRDA", + "sSDR", + "ssfa", + "stcos", + "StratifiedSampling", + "sureLDA", + "survey", + "surveyvoi", + "svydiags", + "systemfit", + "TargetScore", + "text2map", + "textir", + "textmineR", + "textTinyR", + "tmvtnorm", + "TPEA", + "triversity", + "tsapp", + "tvReg", + "uwot", + "vagam", + "VAM", + "WaveSampling", + "WGScan", + "wordspace", + "workflowsets", + "ACSSpack", + "ADDT", + "AER" +) + +# Find the position of the last package in to_skip within all_pkgs +last_skip <- tail(to_skip, 1) + +# Find its position in all_pkgs (returns NA if not found) +start_pos <- match(last_skip, all_pkgs) + +# If not found, start from the beginning; else, start after last_skip +if (is.na(start_pos)) { + to_process <- all_pkgs +} else { + to_process <- all_pkgs[(start_pos + 1):length(all_pkgs)] +} + +if (length(to_process) == 0) { + message("All packages have been processed.") +} else { + for (i in to_process) { + message(sprintf("\nInstalling %s", i)) + pak::pkg_install(i) + library(i, character.only = TRUE) + } + # Update to_skip to include all up to the last processed + to_skip <- all_pkgs[1:(last_skip_pos + length(to_process))] +}