525 lines
18 KiB
Python
525 lines
18 KiB
Python
import asyncio
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import tarfile
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Dict, Any, List, Optional
|
|
from backend.config import load_settings, get_setting, BACKUP_INDEX_FILE, DEFAULT_LOCAL_BACKUP, DATA_DIR
|
|
from backend.runner import runner
|
|
from backend.remote import run_remote_quick, build_ssh_prefix, build_scp_prefix
|
|
from backend.system import format_bytes
|
|
|
|
def get_backup_dir() -> Path:
|
|
s = load_settings()
|
|
configured = s.get("backup_dir", "").strip()
|
|
if configured:
|
|
bdir = Path(configured)
|
|
else:
|
|
bdir = Path(DEFAULT_LOCAL_BACKUP)
|
|
|
|
try:
|
|
bdir.mkdir(parents=True, exist_ok=True)
|
|
return bdir
|
|
except Exception as e:
|
|
print(f"Warning: could not create backup directory {bdir}: {e}")
|
|
fallback = Path("/backups") if os.path.exists("/backups") else (DATA_DIR / "backups")
|
|
fallback.mkdir(parents=True, exist_ok=True)
|
|
return fallback
|
|
|
|
def load_backup_index() -> List[Dict[str, Any]]:
|
|
if not BACKUP_INDEX_FILE.exists():
|
|
return []
|
|
try:
|
|
with open(BACKUP_INDEX_FILE, "r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
except Exception:
|
|
return []
|
|
|
|
def save_backup_index(index: List[Dict[str, Any]]) -> None:
|
|
try:
|
|
BACKUP_INDEX_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(BACKUP_INDEX_FILE, "w", encoding="utf-8") as f:
|
|
json.dump(index, f, indent=2)
|
|
except Exception as e:
|
|
print(f"Error saving backup index: {e}")
|
|
|
|
def compute_sha256(filepath: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with open(filepath, "rb") as f:
|
|
while chunk := f.read(65536):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
async def run_backup_task(
|
|
scope: str = "pangolin",
|
|
description: str = "",
|
|
auto_created: bool = False
|
|
) -> Dict[str, Any]:
|
|
settings = load_settings()
|
|
vps_host = settings.get("vps_host", "").strip()
|
|
p_dir = settings.get("vps_pangolin_dir", "/opt/pangolin").strip()
|
|
backup_dir = get_backup_dir()
|
|
|
|
if not vps_host:
|
|
return {"success": False, "error": "No VPS host configured. Please configure your VPS in Settings first."}
|
|
|
|
timestamp = time.strftime("%Y%m%d_%H%M%S")
|
|
backup_id = f"bk_{timestamp}_{int(time.time()) % 10000:04d}"
|
|
archive_name = f"pangolin-backup_{scope}_{timestamp}.tar.gz"
|
|
local_archive_path = backup_dir / archive_name
|
|
remote_stage = f"/tmp/pangolin_snap_{backup_id}"
|
|
remote_tar = f"/tmp/{archive_name}"
|
|
|
|
# Build multi-step backup workflow script executed on Unraid server
|
|
# 1. SSH to VPS: stage files, hot backup SQLite, tar on VPS
|
|
# 2. SCP/rsync from VPS to Unraid server
|
|
# 3. Clean up remote VPS /tmp
|
|
# 4. Verify local archive on Unraid server
|
|
|
|
ssh_prefix_str = " ".join(build_ssh_prefix(settings))
|
|
scp_prefix_str = " ".join(build_scp_prefix(settings))
|
|
vps_user = settings.get("vps_user", "root")
|
|
vps_port = str(settings.get("vps_port", 22))
|
|
|
|
remote_commands = [
|
|
f'echo "=== STEP 1: Creating Hot-Snapshot on Remote VPS ({vps_host}) ==="',
|
|
f'mkdir -p "{remote_stage}/pangolin"',
|
|
f'''
|
|
eval P_DIR="{p_dir}"
|
|
P_DIR="${{P_DIR/#~/$HOME}}"
|
|
|
|
# Auto-discovery if configured directory does not exist or is empty
|
|
if [ ! -d "$P_DIR" ] || [ -z "$(ls -A "$P_DIR" 2>/dev/null)" ]; then
|
|
echo "Notice: Configured path $P_DIR is empty or does not exist."
|
|
echo "Searching VPS for Pangolin Docker compose stack..."
|
|
DETECTED=$(docker inspect --format '{{{{index .Config.Labels "com.docker.compose.project.working_dir"}}}}' $(docker ps -aq --filter "name=pangolin" 2>/dev/null) 2>/dev/null | head -n 1)
|
|
if [ -z "$DETECTED" ] || [ ! -d "$DETECTED" ]; then
|
|
for candidate in "$HOME/config" "/root/config" "$HOME" "/root" "$HOME/pangolin" "/root/pangolin" "/opt/pangolin"; do
|
|
if [ -d "$candidate" ] && [ -n "$(ls -A "$candidate" 2>/dev/null)" ]; then
|
|
DETECTED="$candidate"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
if [ -n "$DETECTED" ] && [ -d "$DETECTED" ]; then
|
|
echo "--> Auto-discovered active Pangolin directory at: $DETECTED"
|
|
P_DIR="$DETECTED"
|
|
fi
|
|
fi
|
|
|
|
# Standard HHF Pangolin stack items
|
|
BACKUP_ITEMS=("docker-compose.yml" "compose.yaml" "docker-compose.yaml" ".env" "config" "crowdsec" "traefik")
|
|
|
|
echo "--- Staging Pangolin files from $P_DIR ---"
|
|
if [ -d "$P_DIR" ]; then
|
|
if [ -f "$P_DIR/docker-compose.yml" ] || [ -d "$P_DIR/config" ]; then
|
|
echo "Detected Pangolin stack root at $P_DIR"
|
|
for item in "${{BACKUP_ITEMS[@]}}"; do
|
|
if [ -e "$P_DIR/$item" ]; then
|
|
echo "Staging $item..."
|
|
mkdir -p "{remote_stage}/pangolin/$(dirname "$item")"
|
|
cp -a "$P_DIR/$item" "{remote_stage}/pangolin/$item" 2>/dev/null || true
|
|
fi
|
|
done
|
|
else
|
|
echo "Staging contents of $P_DIR into {remote_stage}/pangolin/config/..."
|
|
mkdir -p "{remote_stage}/pangolin/config"
|
|
rsync -av --exclude="*.db" --exclude="*.db-wal" --exclude="*.db-shm" "$P_DIR/" "{remote_stage}/pangolin/config/"
|
|
|
|
# Check parent folder for companion compose and env files
|
|
PARENT_DIR="$(dirname "$P_DIR")"
|
|
for extra in docker-compose.yml compose.yaml docker-compose.yaml .env; do
|
|
if [ -f "$PARENT_DIR/$extra" ]; then
|
|
echo "--- Staging companion $extra from $PARENT_DIR ---"
|
|
cp "$PARENT_DIR/$extra" "{remote_stage}/pangolin/" 2>/dev/null || true
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
|
|
echo "--- Performing zero-lock hot-backup of SQLite databases on VPS ---"
|
|
if [ -d "$P_DIR/config" ]; then
|
|
TARGET_BASE="{remote_stage}/pangolin/config"
|
|
SEARCH_BASE="$P_DIR/config"
|
|
else
|
|
TARGET_BASE="{remote_stage}/pangolin"
|
|
SEARCH_BASE="$P_DIR"
|
|
fi
|
|
|
|
for db in $(find "$SEARCH_BASE" -maxdepth 4 -name "*.db" -o -name "*.sqlite" 2>/dev/null); do
|
|
rel=$(realpath --relative-to="$SEARCH_BASE" "$db")
|
|
mkdir -p "$TARGET_BASE/$(dirname "$rel")"
|
|
echo "Hot-backing up SQLite database: $rel..."
|
|
sqlite3 "$db" ".backup '$TARGET_BASE/$rel'" 2>/dev/null || cp -v "$db" "$TARGET_BASE/$rel"
|
|
done
|
|
# Clean dangling lock/WAL files in staging archive
|
|
find "" -name "*.db-wal" -delete -o -name "*.db-shm" -delete 2>/dev/null || true
|
|
|
|
# Backup any named Docker volumes attached to pangolin containers
|
|
for vol in $(docker inspect --format '{{{{range .Mounts}}}}{{{{if eq .Type "volume"}}}}{{{{.Source}}}}{{{{"\\n"}}}}{{{{end}}}}{{{{end}}}}' $(docker ps -aq --filter "name=pangolin" 2>/dev/null) 2>/dev/null); do
|
|
if [ -d "$vol" ] && [ -n "$(ls -A "$vol" 2>/dev/null)" ]; then
|
|
vname=$(basename "$vol")
|
|
echo "--- Staging Docker Volume: $vname from $vol ---"
|
|
mkdir -p "{remote_stage}/pangolin/volumes/$vname"
|
|
cp -a "$vol/." "{remote_stage}/pangolin/volumes/$vname/" 2>/dev/null || true
|
|
fi
|
|
done
|
|
'''
|
|
]
|
|
|
|
if scope in ["vps_config", "full"]:
|
|
remote_commands.append(f"""
|
|
echo "--- Staging VPS Host Configurations (/etc/ufw, network, cron) ---"
|
|
mkdir -p "{remote_stage}/vps_config/etc"
|
|
for t in /etc/ufw /etc/network /etc/netplan /etc/hosts /etc/hostname /etc/resolv.conf /etc/systemd/system/pangolin*.service /etc/cron* /var/spool/cron; do
|
|
[ -e "$t" ] && cp -r --parents "$t" "{remote_stage}/vps_config/" 2>/dev/null || true
|
|
done
|
|
""")
|
|
|
|
# Metadata & Tar on VPS
|
|
remote_commands.append(f"""
|
|
cat << 'MANIFEST_EOF' > "{remote_stage}/backup_manifest.json"
|
|
{{
|
|
"id": "{backup_id}",
|
|
"scope": "{scope}",
|
|
"source_vps": "{vps_host}",
|
|
"timestamp": "{timestamp}",
|
|
"created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"description": "{description}",
|
|
"auto_created": {str(auto_created).lower()}
|
|
}}
|
|
MANIFEST_EOF
|
|
|
|
echo "--- Compressing snapshot on VPS into {remote_tar} ---"
|
|
tar -czf "{remote_tar}" -C "{remote_stage}" .
|
|
rm -rf "{remote_stage}"
|
|
echo "Remote snapshot prepared successfully."
|
|
""")
|
|
|
|
remote_script_one_liner = "\n".join(remote_commands)
|
|
|
|
# Full bash execution pipeline on Unraid server
|
|
unraid_script = f"""
|
|
set -e
|
|
echo "================================================================"
|
|
echo " Starting Pangolin VPS Backup -> Pull to Unraid Server"
|
|
echo " Remote VPS : {vps_user}@{vps_host}:{vps_port}"
|
|
echo " Local Store: {local_archive_path}"
|
|
echo "================================================================"
|
|
|
|
# Step 1: Execute remote snapshot creation
|
|
{ssh_prefix_str} << 'REMOTE_EXEC_EOF'
|
|
{remote_script_one_liner}
|
|
REMOTE_EXEC_EOF
|
|
|
|
echo ""
|
|
echo "=== STEP 2: Transferring backup archive to Unraid Server ==="
|
|
mkdir -p "{backup_dir}"
|
|
{scp_prefix_str} "{vps_user}@{vps_host}:{remote_tar}" "{local_archive_path}"
|
|
|
|
echo ""
|
|
echo "=== STEP 3: Cleaning up temporary staging on Remote VPS ==="
|
|
{ssh_prefix_str} "rm -f '{remote_tar}'"
|
|
|
|
echo ""
|
|
echo "=== STEP 4: Verifying archive integrity on Unraid Server ==="
|
|
tar -tzf "{local_archive_path}" > /dev/null && echo "Archive integrity check: PASSED"
|
|
echo -n "Unraid Local SHA256: "
|
|
sha256sum "{local_archive_path}"
|
|
|
|
echo ""
|
|
echo "=== BACKUP PULL COMPLETED SUCCESSFULLY ==="
|
|
"""
|
|
|
|
res = await runner.run_command_job(
|
|
name=f"Backup VPS -> Unraid ({scope.upper()})",
|
|
cmd=unraid_script,
|
|
job_type="backup",
|
|
description=description or f"Pangolin VPS backup to Unraid ({scope})"
|
|
)
|
|
|
|
if res.get("success"):
|
|
asyncio.create_task(_monitor_and_index_backup(
|
|
res["job_id"], backup_id, local_archive_path, scope, description, auto_created
|
|
))
|
|
|
|
return res
|
|
|
|
async def _monitor_and_index_backup(
|
|
job_id: str,
|
|
backup_id: str,
|
|
archive_path: Path,
|
|
scope: str,
|
|
description: str,
|
|
auto_created: bool
|
|
):
|
|
while True:
|
|
await asyncio.sleep(1)
|
|
job = runner.get_job(job_id)
|
|
if not job or job.get("status") in ["completed", "failed"]:
|
|
break
|
|
|
|
if job and job.get("status") == "completed" and archive_path.exists():
|
|
size_bytes = archive_path.stat().st_size
|
|
sha256 = compute_sha256(archive_path)
|
|
|
|
files_count = 0
|
|
try:
|
|
with tarfile.open(archive_path, "r:gz") as tar:
|
|
files_count = len(tar.getmembers())
|
|
except Exception:
|
|
pass
|
|
|
|
entry = {
|
|
"id": backup_id,
|
|
"filename": archive_path.name,
|
|
"path": str(archive_path),
|
|
"scope": scope,
|
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"size_bytes": size_bytes,
|
|
"formatted_size": format_bytes(size_bytes),
|
|
"sha256": sha256,
|
|
"files_count": files_count,
|
|
"description": description,
|
|
"auto_created": auto_created,
|
|
"verified": True
|
|
}
|
|
|
|
index = load_backup_index()
|
|
index.insert(0, entry)
|
|
save_backup_index(index)
|
|
|
|
# Auto-prune if retention count exceeded
|
|
retention = get_setting("retention_count", 10)
|
|
prune_backups(retention)
|
|
|
|
def list_backups() -> List[Dict[str, Any]]:
|
|
index = load_backup_index()
|
|
backup_dir = get_backup_dir()
|
|
|
|
valid = []
|
|
for item in index:
|
|
fp = Path(item.get("path", ""))
|
|
if fp.exists():
|
|
valid.append(item)
|
|
|
|
# Detect any unindexed .tar.gz in local backup_dir
|
|
known_files = {item["filename"] for item in valid}
|
|
for f in backup_dir.glob("pangolin-backup_*.tar.gz"):
|
|
if f.name not in known_files:
|
|
try:
|
|
sz = f.stat().st_size
|
|
mtime = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(f.stat().st_mtime))
|
|
h = compute_sha256(f)
|
|
valid.append({
|
|
"id": f"bk_imported_{f.stem}",
|
|
"filename": f.name,
|
|
"path": str(f),
|
|
"scope": "pangolin" if "pangolin" in f.name else "full",
|
|
"created_at": mtime,
|
|
"size_bytes": sz,
|
|
"formatted_size": format_bytes(sz),
|
|
"sha256": h,
|
|
"files_count": 0,
|
|
"description": "Detected in Unraid storage",
|
|
"auto_created": False,
|
|
"verified": True
|
|
})
|
|
except Exception:
|
|
pass
|
|
|
|
valid.sort(key=lambda x: x.get("created_at", ""), reverse=True)
|
|
if len(valid) != len(index):
|
|
save_backup_index(valid)
|
|
return valid
|
|
|
|
def inspect_backup(backup_id: str) -> Dict[str, Any]:
|
|
index = load_backup_index()
|
|
target = next((b for b in index if b["id"] == backup_id), None)
|
|
if not target:
|
|
return {"success": False, "error": "Backup not found"}
|
|
|
|
filepath = Path(target["path"])
|
|
if not filepath.exists():
|
|
return {"success": False, "error": "Backup archive file not found on Unraid disk"}
|
|
|
|
members = []
|
|
try:
|
|
with tarfile.open(filepath, "r:gz") as tar:
|
|
for m in tar.getmembers():
|
|
members.append({
|
|
"name": m.name,
|
|
"size": m.size,
|
|
"formatted_size": format_bytes(m.size),
|
|
"is_dir": m.isdir(),
|
|
"mtime": time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(m.mtime))
|
|
})
|
|
return {
|
|
"success": True,
|
|
"backup": target,
|
|
"total_files": len(members),
|
|
"members": members
|
|
}
|
|
except Exception as e:
|
|
return {"success": False, "error": f"Failed to inspect archive: {str(e)}"}
|
|
|
|
def verify_backup(backup_id: str) -> Dict[str, Any]:
|
|
index = load_backup_index()
|
|
target = next((b for b in index if b["id"] == backup_id), None)
|
|
if not target:
|
|
return {"success": False, "error": "Backup not found"}
|
|
|
|
filepath = Path(target["path"])
|
|
if not filepath.exists():
|
|
return {"success": False, "error": "Backup file not found on Unraid disk"}
|
|
|
|
current_hash = compute_sha256(filepath)
|
|
hash_match = (current_hash == target.get("sha256"))
|
|
|
|
tar_valid = False
|
|
try:
|
|
with tarfile.open(filepath, "r:gz") as tar:
|
|
tar.getmembers()
|
|
tar_valid = True
|
|
except Exception:
|
|
tar_valid = False
|
|
|
|
is_verified = hash_match and tar_valid
|
|
target["verified"] = is_verified
|
|
save_backup_index(index)
|
|
|
|
return {
|
|
"success": True,
|
|
"verified": is_verified,
|
|
"sha256_match": hash_match,
|
|
"tar_integrity": tar_valid,
|
|
"expected_sha256": target.get("sha256"),
|
|
"computed_sha256": current_hash
|
|
}
|
|
|
|
async def restore_backup(backup_id: str) -> Dict[str, Any]:
|
|
index = load_backup_index()
|
|
target = next((b for b in index if b["id"] == backup_id), None)
|
|
if not target:
|
|
return {"success": False, "error": "Backup not found"}
|
|
|
|
archive_path = Path(target["path"])
|
|
if not archive_path.exists():
|
|
return {"success": False, "error": "Archive file does not exist on Unraid disk"}
|
|
|
|
settings = load_settings()
|
|
vps_host = settings.get("vps_host", "").strip()
|
|
p_dir = settings.get("vps_pangolin_dir", "/opt/pangolin").strip()
|
|
if not vps_host:
|
|
return {"success": False, "error": "No VPS host configured."}
|
|
|
|
ssh_prefix_str = " ".join(build_ssh_prefix(settings))
|
|
scp_prefix_str = " ".join(build_scp_prefix(settings))
|
|
vps_user = settings.get("vps_user", "root")
|
|
|
|
remote_tar = f"/tmp/restore_{target['filename']}"
|
|
remote_restore_stage = f"/tmp/restore_stage_{int(time.time())}"
|
|
|
|
# Script: Upload archive to VPS -> extract and restore -> restart containers
|
|
restore_script = f"""
|
|
set -e
|
|
echo "================================================================"
|
|
echo " RESTORING BACKUP FROM UNRAID TO REMOTE VPS ({vps_host})"
|
|
echo " Archive: {target['filename']}"
|
|
echo "================================================================"
|
|
|
|
echo ""
|
|
echo "=== STEP 1: Uploading backup archive from Unraid to VPS ==="
|
|
{scp_prefix_str} "{archive_path}" "{vps_user}@{vps_host}:{remote_tar}"
|
|
|
|
echo ""
|
|
echo "=== STEP 2: Applying restore on remote VPS ==="
|
|
{ssh_prefix_str} << 'REMOTE_RESTORE_EOF'
|
|
set -e
|
|
mkdir -p "{remote_restore_stage}"
|
|
tar -xzf "{remote_tar}" -C "{remote_restore_stage}"
|
|
|
|
# Stop containers on VPS
|
|
if [ -d "{p_dir}" ]; then
|
|
echo "Stopping existing containers on VPS..."
|
|
cd "{p_dir}" && docker compose down 2>/dev/null || true
|
|
fi
|
|
|
|
# Restore Pangolin files
|
|
if [ -d "{remote_restore_stage}/pangolin" ]; then
|
|
echo "Restoring Pangolin directory files on VPS..."
|
|
mkdir -p "{p_dir}"
|
|
rsync -av "{remote_restore_stage}/pangolin/" "{p_dir}/"
|
|
fi
|
|
|
|
# Restore VPS Config if present
|
|
if [ -d "{remote_restore_stage}/vps_config" ]; then
|
|
echo "Restoring VPS system configurations..."
|
|
rsync -av "{remote_restore_stage}/vps_config/" "/"
|
|
fi
|
|
|
|
# Restart containers on VPS
|
|
if [ -f "{p_dir}/docker-compose.yml" ] || [ -f "{p_dir}/compose.yaml" ]; then
|
|
echo "Starting Pangolin containers on VPS..."
|
|
cd "{p_dir}" && docker compose up -d
|
|
docker compose ps
|
|
fi
|
|
|
|
# Cleanup VPS tmp
|
|
rm -rf "{remote_restore_stage}" "{remote_tar}"
|
|
echo "Remote VPS restore completed successfully!"
|
|
REMOTE_RESTORE_EOF
|
|
|
|
echo ""
|
|
echo "=== RESTORE OPERATION COMPLETED SUCCESSFULLY ==="
|
|
"""
|
|
|
|
res = await runner.run_command_job(
|
|
name=f"Restore to VPS ({target['filename']})",
|
|
cmd=restore_script,
|
|
job_type="restore",
|
|
description=f"Restore backup {backup_id} from Unraid to remote VPS ({vps_host})"
|
|
)
|
|
return res
|
|
|
|
def delete_backup(backup_id: str) -> Dict[str, Any]:
|
|
index = load_backup_index()
|
|
target = next((b for b in index if b["id"] == backup_id), None)
|
|
if not target:
|
|
return {"success": False, "error": "Backup not found"}
|
|
|
|
try:
|
|
p = Path(target["path"])
|
|
if p.exists():
|
|
p.unlink()
|
|
except Exception as e:
|
|
print(f"Error removing local file {target['path']}: {e}")
|
|
|
|
new_index = [b for b in index if b["id"] != backup_id]
|
|
save_backup_index(new_index)
|
|
return {"success": True, "message": "Backup deleted from Unraid server"}
|
|
|
|
def prune_backups(keep_count: int = 10) -> int:
|
|
index = load_backup_index()
|
|
if len(index) <= keep_count:
|
|
return 0
|
|
|
|
to_prune = index[keep_count:]
|
|
kept = index[:keep_count]
|
|
|
|
pruned = 0
|
|
for item in to_prune:
|
|
try:
|
|
p = Path(item["path"])
|
|
if p.exists():
|
|
p.unlink()
|
|
pruned += 1
|
|
except Exception:
|
|
pass
|
|
|
|
save_backup_index(kept)
|
|
return pruned
|