pat-s
a9ad907f9c
- migrate-s3-hetzner-to-backblaze.sh: rclone-based bucket migration helper. - find-R-api-packages.sh: scan CRAN package sources for a C-API usage pattern. - query-pkgs-without-old-versions.R, test-package-loading.R: ad-hoc helpers.
90 lines
2.7 KiB
Shell
90 lines
2.7 KiB
Shell
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Migrate S3 buckets from Hetzner Object Storage to Backblaze B2 via rclone.
|
|
#
|
|
# Prerequisites:
|
|
# 1. Install rclone: https://rclone.org/install/
|
|
# 2. Configure two rclone remotes:
|
|
# rclone config create hetzner s3 \
|
|
# provider=Other \
|
|
# env_auth=false \
|
|
# access_key_id=YOUR_HETZNER_KEY \
|
|
# secret_access_key=YOUR_HETZNER_SECRET \
|
|
# endpoint=fsn1.your-objectstorage.com # adjust region
|
|
#
|
|
# rclone config create backblaze s3 \
|
|
# provider=Other \
|
|
# env_auth=false \
|
|
# access_key_id=YOUR_B2_KEY \
|
|
# secret_access_key=YOUR_B2_APP_KEY \
|
|
# endpoint=s3.us-west-004.backblazeb2.com # adjust region
|
|
#
|
|
# Usage:
|
|
# ./migrate-s3-hetzner-to-backblaze.sh <src:dst> [src:dst] ...
|
|
# ./migrate-s3-hetzner-to-backblaze.sh hetzner-bucket:backblaze-bucket
|
|
|
|
HETZNER_REMOTE="${HETZNER_REMOTE:-hetzner}"
|
|
BACKBLAZE_REMOTE="${BACKBLAZE_REMOTE:-backblaze}"
|
|
RCLONE_FLAGS="${RCLONE_FLAGS:---transfers=64 --checkers=64 --fast-list}"
|
|
|
|
if [[ $# -eq 0 ]]; then
|
|
echo "Usage: $0 <src-bucket:dst-bucket> [src-bucket:dst-bucket...]"
|
|
echo ""
|
|
echo " Each argument is a source:destination bucket pair separated by a colon."
|
|
echo ""
|
|
echo "Environment variables:"
|
|
echo " HETZNER_REMOTE rclone remote name for Hetzner (default: hetzner)"
|
|
echo " BACKBLAZE_REMOTE rclone remote name for Backblaze (default: backblaze)"
|
|
echo " RCLONE_FLAGS extra rclone flags (default: --transfers=16 --checkers=16 --fast-list)"
|
|
echo " DRY_RUN=1 show what would be copied without copying"
|
|
exit 1
|
|
fi
|
|
|
|
for cmd in rclone; do
|
|
if ! command -v "$cmd" &>/dev/null; then
|
|
echo "Error: $cmd is not installed." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# Verify remotes exist
|
|
for remote in "$HETZNER_REMOTE" "$BACKBLAZE_REMOTE"; do
|
|
if ! rclone listremotes | grep -q "^${remote}:$"; then
|
|
echo "Error: rclone remote '${remote}' not found. Run 'rclone config' to set it up." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
DRY_RUN_FLAG=""
|
|
if [[ "${DRY_RUN:-0}" == "1" ]]; then
|
|
DRY_RUN_FLAG="--dry-run"
|
|
echo "=== DRY RUN MODE ==="
|
|
fi
|
|
|
|
for pair in "$@"; do
|
|
if [[ "$pair" != *:* ]]; then
|
|
echo "Error: '$pair' is not a valid src:dst pair. Use format 'hetzner-bucket:backblaze-bucket'." >&2
|
|
exit 1
|
|
fi
|
|
|
|
src_bucket="${pair%%:*}"
|
|
dst_bucket="${pair#*:}"
|
|
src="${HETZNER_REMOTE}:${src_bucket}"
|
|
dst="${BACKBLAZE_REMOTE}:${dst_bucket}"
|
|
|
|
echo ""
|
|
echo "--- Migrating: ${src} -> ${dst} ---"
|
|
|
|
# shellcheck disable=SC2086
|
|
rclone sync \
|
|
${RCLONE_FLAGS} \
|
|
${DRY_RUN_FLAG} \
|
|
--progress \
|
|
"$src" "$dst"
|
|
|
|
echo "--- Done: ${src_bucket} -> ${dst_bucket} ---"
|
|
done
|
|
|
|
echo ""
|
|
echo "Migration complete."
|