chore(local): add S3 migration and CRAN-source helper scripts
- migrate-s3-hetzner-to-backblaze.sh: rclone-based bucket migration helper. - find-R-api-packages.sh: scan CRAN package sources for a C-API usage pattern. - query-pkgs-without-old-versions.R, test-package-loading.R: ad-hoc helpers.
This commit is contained in:
parent
3ae4672d66
commit
a9ad907f9c
1 changed files with 494 additions and 0 deletions
90
local/migrate-s3-hetzner-to-backblaze.sh
Normal file
90
local/migrate-s3-hetzner-to-backblaze.sh
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Migrate S3 buckets from Hetzner Object Storage to Backblaze B2 via rclone.
|
||||
#
|
||||
# Prerequisites:
|
||||
# 1. Install rclone: https://rclone.org/install/
|
||||
# 2. Configure two rclone remotes:
|
||||
# rclone config create hetzner s3 \
|
||||
# provider=Other \
|
||||
# env_auth=false \
|
||||
# access_key_id=YOUR_HETZNER_KEY \
|
||||
# secret_access_key=YOUR_HETZNER_SECRET \
|
||||
# endpoint=fsn1.your-objectstorage.com # adjust region
|
||||
#
|
||||
# rclone config create backblaze s3 \
|
||||
# provider=Other \
|
||||
# env_auth=false \
|
||||
# access_key_id=YOUR_B2_KEY \
|
||||
# secret_access_key=YOUR_B2_APP_KEY \
|
||||
# endpoint=s3.us-west-004.backblazeb2.com # adjust region
|
||||
#
|
||||
# Usage:
|
||||
# ./migrate-s3-hetzner-to-backblaze.sh <src:dst> [src:dst] ...
|
||||
# ./migrate-s3-hetzner-to-backblaze.sh hetzner-bucket:backblaze-bucket
|
||||
|
||||
HETZNER_REMOTE="${HETZNER_REMOTE:-hetzner}"
|
||||
BACKBLAZE_REMOTE="${BACKBLAZE_REMOTE:-backblaze}"
|
||||
RCLONE_FLAGS="${RCLONE_FLAGS:---transfers=64 --checkers=64 --fast-list}"
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
echo "Usage: $0 <src-bucket:dst-bucket> [src-bucket:dst-bucket...]"
|
||||
echo ""
|
||||
echo " Each argument is a source:destination bucket pair separated by a colon."
|
||||
echo ""
|
||||
echo "Environment variables:"
|
||||
echo " HETZNER_REMOTE rclone remote name for Hetzner (default: hetzner)"
|
||||
echo " BACKBLAZE_REMOTE rclone remote name for Backblaze (default: backblaze)"
|
||||
echo " RCLONE_FLAGS extra rclone flags (default: --transfers=16 --checkers=16 --fast-list)"
|
||||
echo " DRY_RUN=1 show what would be copied without copying"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for cmd in rclone; do
|
||||
if ! command -v "$cmd" &>/dev/null; then
|
||||
echo "Error: $cmd is not installed." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Verify remotes exist
|
||||
for remote in "$HETZNER_REMOTE" "$BACKBLAZE_REMOTE"; do
|
||||
if ! rclone listremotes | grep -q "^${remote}:$"; then
|
||||
echo "Error: rclone remote '${remote}' not found. Run 'rclone config' to set it up." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
DRY_RUN_FLAG=""
|
||||
if [[ "${DRY_RUN:-0}" == "1" ]]; then
|
||||
DRY_RUN_FLAG="--dry-run"
|
||||
echo "=== DRY RUN MODE ==="
|
||||
fi
|
||||
|
||||
for pair in "$@"; do
|
||||
if [[ "$pair" != *:* ]]; then
|
||||
echo "Error: '$pair' is not a valid src:dst pair. Use format 'hetzner-bucket:backblaze-bucket'." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
src_bucket="${pair%%:*}"
|
||||
dst_bucket="${pair#*:}"
|
||||
src="${HETZNER_REMOTE}:${src_bucket}"
|
||||
dst="${BACKBLAZE_REMOTE}:${dst_bucket}"
|
||||
|
||||
echo ""
|
||||
echo "--- Migrating: ${src} -> ${dst} ---"
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
rclone sync \
|
||||
${RCLONE_FLAGS} \
|
||||
${DRY_RUN_FLAG} \
|
||||
--progress \
|
||||
"$src" "$dst"
|
||||
|
||||
echo "--- Done: ${src_bucket} -> ${dst_bucket} ---"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Migration complete."
|
||||
Loading…
Reference in a new issue