chore(local): add S3 migration and CRAN-source helper scripts
- migrate-s3-hetzner-to-backblaze.sh: rclone-based bucket migration helper. - find-R-api-packages.sh: scan CRAN package sources for a C-API usage pattern. - query-pkgs-without-old-versions.R, test-package-loading.R: ad-hoc helpers.
This commit is contained in:
parent
3ae4672d66
commit
a9ad907f9c
GPG key ID:
3C6318841EF78925
4 changed files with 494 additions and 0 deletions
34
local/find-R-api-packages.sh
Normal file
34
local/find-R-api-packages.sh
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Directory to clone repos into
|
||||
WORKDIR="cran_repos"
|
||||
mkdir -p "$WORKDIR"
|
||||
cd "$WORKDIR"
|
||||
|
||||
# GitHub API paginates results, so we loop through pages
|
||||
PAGE=1
|
||||
PER_PAGE=100
|
||||
MATCHES=()
|
||||
|
||||
while :; do
|
||||
# Fetch a page of repos
|
||||
REPOS=$(curl -s "https://api.github.com/orgs/cran/repos?per_page=$PER_PAGE&page=$PAGE" | jq -r '.[].clone_url')
|
||||
[ -z "$REPOS" ] && break
|
||||
|
||||
for REPO_URL in $REPOS; do
|
||||
REPO_NAME=$(basename "$REPO_URL" .git)
|
||||
# Skip if already cloned
|
||||
[ -d "$REPO_NAME" ] && continue
|
||||
git clone --depth 1 "$REPO_URL" "$REPO_NAME" >/dev/null 2>&1
|
||||
if [ -d "$REPO_NAME/src" ]; then
|
||||
# Search for Rinternals.h in src/
|
||||
if grep -r -q 'R_VERSION < R_Version(' "$REPO_NAME/src"; then
|
||||
echo "$REPO_NAME"
|
||||
fi
|
||||
fi
|
||||
# Clean up to save space
|
||||
rm -rf "$REPO_NAME"
|
||||
done
|
||||
|
||||
PAGE=$((PAGE + 1))
|
||||
done
|
||||
90
local/migrate-s3-hetzner-to-backblaze.sh
Normal file
90
local/migrate-s3-hetzner-to-backblaze.sh
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Migrate S3 buckets from Hetzner Object Storage to Backblaze B2 via rclone.
|
||||
#
|
||||
# Prerequisites:
|
||||
# 1. Install rclone: https://rclone.org/install/
|
||||
# 2. Configure two rclone remotes:
|
||||
# rclone config create hetzner s3 \
|
||||
# provider=Other \
|
||||
# env_auth=false \
|
||||
# access_key_id=YOUR_HETZNER_KEY \
|
||||
# secret_access_key=YOUR_HETZNER_SECRET \
|
||||
# endpoint=fsn1.your-objectstorage.com # adjust region
|
||||
#
|
||||
# rclone config create backblaze s3 \
|
||||
# provider=Other \
|
||||
# env_auth=false \
|
||||
# access_key_id=YOUR_B2_KEY \
|
||||
# secret_access_key=YOUR_B2_APP_KEY \
|
||||
# endpoint=s3.us-west-004.backblazeb2.com # adjust region
|
||||
#
|
||||
# Usage:
|
||||
# ./migrate-s3-hetzner-to-backblaze.sh <src:dst> [src:dst] ...
|
||||
# ./migrate-s3-hetzner-to-backblaze.sh hetzner-bucket:backblaze-bucket
|
||||
|
||||
HETZNER_REMOTE="${HETZNER_REMOTE:-hetzner}"
|
||||
BACKBLAZE_REMOTE="${BACKBLAZE_REMOTE:-backblaze}"
|
||||
RCLONE_FLAGS="${RCLONE_FLAGS:---transfers=64 --checkers=64 --fast-list}"
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
echo "Usage: $0 <src-bucket:dst-bucket> [src-bucket:dst-bucket...]"
|
||||
echo ""
|
||||
echo " Each argument is a source:destination bucket pair separated by a colon."
|
||||
echo ""
|
||||
echo "Environment variables:"
|
||||
echo " HETZNER_REMOTE rclone remote name for Hetzner (default: hetzner)"
|
||||
echo " BACKBLAZE_REMOTE rclone remote name for Backblaze (default: backblaze)"
|
||||
echo " RCLONE_FLAGS extra rclone flags (default: --transfers=16 --checkers=16 --fast-list)"
|
||||
echo " DRY_RUN=1 show what would be copied without copying"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for cmd in rclone; do
|
||||
if ! command -v "$cmd" &>/dev/null; then
|
||||
echo "Error: $cmd is not installed." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Verify remotes exist
|
||||
for remote in "$HETZNER_REMOTE" "$BACKBLAZE_REMOTE"; do
|
||||
if ! rclone listremotes | grep -q "^${remote}:$"; then
|
||||
echo "Error: rclone remote '${remote}' not found. Run 'rclone config' to set it up." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
DRY_RUN_FLAG=""
|
||||
if [[ "${DRY_RUN:-0}" == "1" ]]; then
|
||||
DRY_RUN_FLAG="--dry-run"
|
||||
echo "=== DRY RUN MODE ==="
|
||||
fi
|
||||
|
||||
for pair in "$@"; do
|
||||
if [[ "$pair" != *:* ]]; then
|
||||
echo "Error: '$pair' is not a valid src:dst pair. Use format 'hetzner-bucket:backblaze-bucket'." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
src_bucket="${pair%%:*}"
|
||||
dst_bucket="${pair#*:}"
|
||||
src="${HETZNER_REMOTE}:${src_bucket}"
|
||||
dst="${BACKBLAZE_REMOTE}:${dst_bucket}"
|
||||
|
||||
echo ""
|
||||
echo "--- Migrating: ${src} -> ${dst} ---"
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
rclone sync \
|
||||
${RCLONE_FLAGS} \
|
||||
${DRY_RUN_FLAG} \
|
||||
--progress \
|
||||
"$src" "$dst"
|
||||
|
||||
echo "--- Done: ${src_bucket} -> ${dst_bucket} ---"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Migration complete."
|
||||
39
local/query-pkgs-without-old-versions.R
Normal file
39
local/query-pkgs-without-old-versions.R
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
library(s3fs)
|
||||
|
||||
# List all files under contrib/<pkg>/
|
||||
all_files <- s3fs::s3_dir_ls(
|
||||
"s3://devxy-r-package-binaries-hel1/arm64/alpine322/latest/src/contrib/",
|
||||
recurse = TRUE,
|
||||
type = "file"
|
||||
)
|
||||
|
||||
pattern <- ".*/src/contrib/([^/_]+)_.*"
|
||||
matches <- regmatches(all_files, regexec(pattern, all_files))
|
||||
pkg_names <- unique(
|
||||
vapply(
|
||||
matches,
|
||||
function(x) if (length(x) > 1) x[2] else NA_character_,
|
||||
character(1)
|
||||
)
|
||||
)
|
||||
pkg_names <- pkg_names[!is.na(pkg_names)]
|
||||
|
||||
# For each package, check if Archive/<pkg>/ contains any files
|
||||
no_archive_files <- character(0)
|
||||
for (pkg in pkg_names) {
|
||||
archive_dir1 <- sprintf(
|
||||
"s3://devxy-r-package-binaries-hel1/arm64/alpine322/latest/src/contrib/Archive/%s",
|
||||
pkg
|
||||
)
|
||||
archive_files <- unique(c(
|
||||
tryCatch(
|
||||
s3fs::s3_dir_ls(archive_dir1, recurse = TRUE),
|
||||
error = function(e) character(0)
|
||||
)
|
||||
))
|
||||
if (length(archive_files) == 0) {
|
||||
no_archive_files <- c(no_archive_files, pkg)
|
||||
}
|
||||
}
|
||||
|
||||
print(no_archive_files)
|
||||
331
local/test-package-loading.R
Normal file
331
local/test-package-loading.R
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
install.packages(
|
||||
"pak",
|
||||
repos = sprintf(
|
||||
"https://r-lib.github.io/p/pak/stable/%s/%s/%s",
|
||||
.Platform$pkgType,
|
||||
R.Version()$os,
|
||||
R.Version()$arch
|
||||
)
|
||||
)
|
||||
|
||||
Sys.setenv(PKG_SYSREQS = TRUE)
|
||||
all_pkgs <- rownames(available.packages())
|
||||
|
||||
to_skip = c("ABRSQOL", "ACA", "ACE.CoCo")
|
||||
all_pkgs = setdiff(all_pkgs, to_skip)
|
||||
|
||||
for (i in all_pkgs) {
|
||||
message(sprintf("\nInstalling %s", i))
|
||||
pak::pkg_install(i)
|
||||
library(i, character.only = TRUE)
|
||||
}
|
||||
|
||||
|
||||
# Example data
|
||||
all_pkgs <- rownames(available.packages())
|
||||
to_skip <- c(
|
||||
"ABRSQOL",
|
||||
"ACA",
|
||||
"ACE.CoCo",
|
||||
"ACEsimFit",
|
||||
"ACNE",
|
||||
"absorber",
|
||||
"adapt4pv",
|
||||
"adaptMCMC",
|
||||
"addhaz",
|
||||
"ADDT",
|
||||
"ahaz",
|
||||
"arm",
|
||||
"arules",
|
||||
"arulesCBA",
|
||||
"aster2",
|
||||
"BayesFactor",
|
||||
"bc3net",
|
||||
"bgsmtr",
|
||||
"biglasso",
|
||||
"BinNonNor",
|
||||
"BinNor",
|
||||
"bioassayR",
|
||||
"birankr",
|
||||
"BiRewire",
|
||||
"bolasso",
|
||||
"Boptbd",
|
||||
"Brobdingnag",
|
||||
"BSW",
|
||||
"BTLLasso",
|
||||
"bvartools",
|
||||
"cAIC4",
|
||||
"Category",
|
||||
"celda",
|
||||
"centiserve",
|
||||
"cjoint",
|
||||
"clinical",
|
||||
"clipper",
|
||||
"CodataGS",
|
||||
"conos",
|
||||
"CopulaInference",
|
||||
"covEB",
|
||||
"cplm",
|
||||
"CRTgeeDR",
|
||||
"cthreshER",
|
||||
"ctmcmove",
|
||||
"curephEM",
|
||||
"CVST",
|
||||
"dcGSA",
|
||||
"dclone",
|
||||
"dcsvm",
|
||||
"DelayedArray",
|
||||
"dglars",
|
||||
"dhglm",
|
||||
"disordR",
|
||||
"distrom",
|
||||
"dmm",
|
||||
"DNABarcodes",
|
||||
"DoubleCone",
|
||||
"DRR",
|
||||
"DTRlearn2",
|
||||
"DWDLargeR",
|
||||
"eds",
|
||||
"EMCluster",
|
||||
"EMMREML",
|
||||
"evalITR",
|
||||
"EventPointer",
|
||||
"evola",
|
||||
"excursions",
|
||||
"expm",
|
||||
"fanc",
|
||||
"FAS",
|
||||
"fastadi",
|
||||
"fastPLS",
|
||||
"fastRG",
|
||||
"fdaPDE",
|
||||
"flare",
|
||||
"FoReco",
|
||||
"frailtyHL",
|
||||
"freebird",
|
||||
"FSTpackage",
|
||||
"gamlr",
|
||||
"gamlss.lasso",
|
||||
"gamm4",
|
||||
"gbmt",
|
||||
"gdim",
|
||||
"gdistance",
|
||||
"GeDS",
|
||||
"geeM",
|
||||
"genlasso",
|
||||
"GenOrd",
|
||||
"GenoScan",
|
||||
"geomorph",
|
||||
"geostatsp",
|
||||
"GhostKnockoff",
|
||||
"GIGSEA",
|
||||
"GlarmaVarSel",
|
||||
"glmm",
|
||||
"glmmrBase",
|
||||
"glmmrOptim",
|
||||
"glmnet",
|
||||
"glober",
|
||||
"GPvam",
|
||||
"graphpcor",
|
||||
"gremlin",
|
||||
"growthrate",
|
||||
"grpCox",
|
||||
"GSD",
|
||||
"HelpersMG",
|
||||
"hglm",
|
||||
"hglm.data",
|
||||
"hibayes",
|
||||
"hierSDR",
|
||||
"HMTL",
|
||||
"hsem",
|
||||
"ibmdbR",
|
||||
"inca",
|
||||
"INLAspacetime",
|
||||
"INLAtools",
|
||||
"invertiforms",
|
||||
"irlba",
|
||||
"islasso",
|
||||
"ISLET",
|
||||
"isotonic.pen",
|
||||
"jordan",
|
||||
"kinship2",
|
||||
"KnockoffScreen",
|
||||
"lcpm",
|
||||
"leidenAlg",
|
||||
"lfe",
|
||||
"lingmatch",
|
||||
"LKT",
|
||||
"lme4",
|
||||
"lme4breeding",
|
||||
"lme4GS",
|
||||
"logcondiscr",
|
||||
"LPmerge",
|
||||
"LRMF3",
|
||||
"MAP",
|
||||
"marcox",
|
||||
"markovchain",
|
||||
"MatrixExtra",
|
||||
"matter",
|
||||
"MBC",
|
||||
"mcen",
|
||||
"mclogit",
|
||||
"MCMCglmm",
|
||||
"mdhglm",
|
||||
"MDPtoolbox",
|
||||
"mediation",
|
||||
"mefa4",
|
||||
"metafor",
|
||||
"mgwrsar",
|
||||
"mi",
|
||||
"midasml",
|
||||
"mind",
|
||||
"monocle",
|
||||
"msda",
|
||||
"MuData",
|
||||
"MultiGlarmaVarSel",
|
||||
"MultiOrd",
|
||||
"MultiVarSel",
|
||||
"mvglmmRank",
|
||||
"N2R",
|
||||
"nadiv",
|
||||
"NBtsVarSel",
|
||||
"NegBinBetaBinreg",
|
||||
"NetworkRiskMeasures",
|
||||
"neuroim2",
|
||||
"NOISeq",
|
||||
"numbat",
|
||||
"optbdmaeAT",
|
||||
"optimbase",
|
||||
"OptimModel",
|
||||
"optrcdmaeAT",
|
||||
"OrdNor",
|
||||
"pagoda2",
|
||||
"PCovR",
|
||||
"pedgene",
|
||||
"pedigree",
|
||||
"pedigreemm",
|
||||
"pense",
|
||||
"PERMANOVA",
|
||||
"phateR",
|
||||
"PhylogeneticEM",
|
||||
"pleio",
|
||||
"POINT",
|
||||
"PoisBinNonNor",
|
||||
"PoisBinOrd",
|
||||
"PoisBinOrdNonNor",
|
||||
"PoisBinOrdNor",
|
||||
"PoisNonNor",
|
||||
"PoisNor",
|
||||
"PRISMA",
|
||||
"ProbitSpatial",
|
||||
"prodest",
|
||||
"psqn",
|
||||
"qlcMatrix",
|
||||
"qpcR",
|
||||
"QRM",
|
||||
"quadrupen",
|
||||
"QZ",
|
||||
"ramps",
|
||||
"randnet",
|
||||
"randPedPCA",
|
||||
"rBMF",
|
||||
"RCBR",
|
||||
"RealVAMS",
|
||||
"REBayes",
|
||||
"recommenderlab",
|
||||
"Rediscover",
|
||||
"reglogit",
|
||||
"RESET",
|
||||
"RGE",
|
||||
"RGENERATEPREC",
|
||||
"riemtan",
|
||||
"RNewsflow",
|
||||
"robustlmm",
|
||||
"rsparse",
|
||||
"rSPDE",
|
||||
"rwc",
|
||||
"S4Arrays",
|
||||
"saeMSPE",
|
||||
"sbw",
|
||||
"scITD",
|
||||
"scoup",
|
||||
"sdwd",
|
||||
"SEAGLE",
|
||||
"sensory",
|
||||
"serrsBayes",
|
||||
"sglasso",
|
||||
"sharpPen",
|
||||
"SiPSiC",
|
||||
"SKAT",
|
||||
"snpReady",
|
||||
"snpStats",
|
||||
"softImpute",
|
||||
"sommer",
|
||||
"soptdmaeA",
|
||||
"SOR",
|
||||
"SparseArray",
|
||||
"SparseChol",
|
||||
"sparseLRMatrix",
|
||||
"sparsenet",
|
||||
"sparsenetgls",
|
||||
"sparsestep",
|
||||
"spatialprobit",
|
||||
"spatialreg",
|
||||
"spatstat.sparse",
|
||||
"speedglm",
|
||||
"sRDA",
|
||||
"sSDR",
|
||||
"ssfa",
|
||||
"stcos",
|
||||
"StratifiedSampling",
|
||||
"sureLDA",
|
||||
"survey",
|
||||
"surveyvoi",
|
||||
"svydiags",
|
||||
"systemfit",
|
||||
"TargetScore",
|
||||
"text2map",
|
||||
"textir",
|
||||
"textmineR",
|
||||
"textTinyR",
|
||||
"tmvtnorm",
|
||||
"TPEA",
|
||||
"triversity",
|
||||
"tsapp",
|
||||
"tvReg",
|
||||
"uwot",
|
||||
"vagam",
|
||||
"VAM",
|
||||
"WaveSampling",
|
||||
"WGScan",
|
||||
"wordspace",
|
||||
"workflowsets",
|
||||
"ACSSpack",
|
||||
"ADDT",
|
||||
"AER"
|
||||
)
|
||||
|
||||
# Find the position of the last package in to_skip within all_pkgs
|
||||
last_skip <- tail(to_skip, 1)
|
||||
|
||||
# Find its position in all_pkgs (returns NA if not found)
|
||||
start_pos <- match(last_skip, all_pkgs)
|
||||
|
||||
# If not found, start from the beginning; else, start after last_skip
|
||||
if (is.na(start_pos)) {
|
||||
to_process <- all_pkgs
|
||||
} else {
|
||||
to_process <- all_pkgs[(start_pos + 1):length(all_pkgs)]
|
||||
}
|
||||
|
||||
if (length(to_process) == 0) {
|
||||
message("All packages have been processed.")
|
||||
} else {
|
||||
for (i in to_process) {
|
||||
message(sprintf("\nInstalling %s", i))
|
||||
pak::pkg_install(i)
|
||||
library(i, character.only = TRUE)
|
||||
}
|
||||
# Update to_skip to include all up to the last processed
|
||||
to_skip <- all_pkgs[1:(last_skip_pos + length(to_process))]
|
||||
}
|
||||
Loading…
Reference in a new issue