Initial commit: Pangolin Sentinel Docker & Unraid Hub

backend/backup.py

@@ -0,0 +1,454 @@
+import asyncio
+import hashlib
+import json
+import os
+import shutil
+import tarfile
+import time
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+from backend.config import load_settings, get_setting, BACKUP_INDEX_FILE, DEFAULT_LOCAL_BACKUP, DATA_DIR
+from backend.runner import runner
+from backend.remote import run_remote_quick, build_ssh_prefix, build_scp_prefix
+from backend.system import format_bytes
+
+def get_backup_dir() -> Path:
+    s = load_settings()
+    configured = s.get("backup_dir", "").strip()
+    if configured:
+        bdir = Path(configured)
+    else:
+        bdir = Path(DEFAULT_LOCAL_BACKUP)
+
+    try:
+        bdir.mkdir(parents=True, exist_ok=True)
+        return bdir
+    except Exception as e:
+        print(f"Warning: could not create backup directory {bdir}: {e}")
+        fallback = Path("/backups") if os.path.exists("/backups") else (DATA_DIR / "backups")
+        fallback.mkdir(parents=True, exist_ok=True)
+        return fallback
+
+def load_backup_index() -> List[Dict[str, Any]]:
+    if not BACKUP_INDEX_FILE.exists():
+        return []
+    try:
+        with open(BACKUP_INDEX_FILE, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception:
+        return []
+
+def save_backup_index(index: List[Dict[str, Any]]) -> None:
+    try:
+        BACKUP_INDEX_FILE.parent.mkdir(parents=True, exist_ok=True)
+        with open(BACKUP_INDEX_FILE, "w", encoding="utf-8") as f:
+            json.dump(index, f, indent=2)
+    except Exception as e:
+        print(f"Error saving backup index: {e}")
+
+def compute_sha256(filepath: Path) -> str:
+    h = hashlib.sha256()
+    with open(filepath, "rb") as f:
+        while chunk := f.read(65536):
+            h.update(chunk)
+    return h.hexdigest()
+
+async def run_backup_task(
+    scope: str = "pangolin",
+    description: str = "",
+    auto_created: bool = False
+) -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    p_dir = settings.get("vps_pangolin_dir", "/opt/pangolin").strip()
+    backup_dir = get_backup_dir()
+
+    if not vps_host:
+        return {"success": False, "error": "No VPS host configured. Please configure your VPS in Settings first."}
+
+    timestamp = time.strftime("%Y%m%d_%H%M%S")
+    backup_id = f"bk_{timestamp}_{int(time.time()) % 10000:04d}"
+    archive_name = f"pangolin-backup_{scope}_{timestamp}.tar.gz"
+    local_archive_path = backup_dir / archive_name
+    remote_stage = f"/tmp/pangolin_snap_{backup_id}"
+    remote_tar = f"/tmp/{archive_name}"
+
+    # Build multi-step backup workflow script executed on Unraid server
+    # 1. SSH to VPS: stage files, hot backup SQLite, tar on VPS
+    # 2. SCP/rsync from VPS to Unraid server
+    # 3. Clean up remote VPS /tmp
+    # 4. Verify local archive on Unraid server
+
+    ssh_prefix_str = " ".join(build_ssh_prefix(settings))
+    scp_prefix_str = " ".join(build_scp_prefix(settings))
+    vps_user = settings.get("vps_user", "root")
+    vps_port = str(settings.get("vps_port", 22))
+
+    remote_commands = [
+        f'echo "=== STEP 1: Creating Hot-Snapshot on Remote VPS ({vps_host}) ==="',
+        f'mkdir -p "{remote_stage}/pangolin"',
+        f'echo "--- Staging Pangolin files from {p_dir} ---"',
+        f'if [ -d "{p_dir}" ]; then rsync -av --exclude="*.db" --exclude="*.db-wal" --exclude="*.db-shm" "{p_dir}/" "{remote_stage}/pangolin/"; fi',
+        f'echo "--- Performing zero-lock hot-backup of SQLite databases on VPS ---"',
+        f"""
+for db in $(find "{p_dir}" -name "*.db" -o -name "*.sqlite" 2>/dev/null); do
+    rel=$(realpath --relative-to="{p_dir}" "$db")
+    mkdir -p "{remote_stage}/pangolin/$(dirname "$rel")"
+    echo "Hot-backing up $rel..."
+    sqlite3 "$db" ".backup '{remote_stage}/pangolin/$rel'" 2>/dev/null || cp -v "$db" "{remote_stage}/pangolin/$rel"
+done
+"""
+    ]
+
+    if scope in ["vps_config", "full"]:
+        remote_commands.append(f"""
+echo "--- Staging VPS Host Configurations (/etc/ufw, network, cron) ---"
+mkdir -p "{remote_stage}/vps_config/etc"
+for t in /etc/ufw /etc/network /etc/netplan /etc/hosts /etc/hostname /etc/resolv.conf /etc/systemd/system/pangolin*.service /etc/cron* /var/spool/cron; do
+    [ -e "$t" ] && cp -r --parents "$t" "{remote_stage}/vps_config/" 2>/dev/null || true
+done
+""")
+
+    # Metadata & Tar on VPS
+    remote_commands.append(f"""
+cat << 'MANIFEST_EOF' > "{remote_stage}/backup_manifest.json"
+{{
+  "id": "{backup_id}",
+  "scope": "{scope}",
+  "source_vps": "{vps_host}",
+  "timestamp": "{timestamp}",
+  "created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+  "description": "{description}",
+  "auto_created": {str(auto_created).lower()}
+}}
+MANIFEST_EOF
+
+echo "--- Compressing snapshot on VPS into {remote_tar} ---"
+tar -czf "{remote_tar}" -C "{remote_stage}" .
+rm -rf "{remote_stage}"
+echo "Remote snapshot prepared successfully."
+""")
+
+    remote_script_one_liner = "\n".join(remote_commands)
+
+    # Full bash execution pipeline on Unraid server
+    unraid_script = f"""
+set -e
+echo "================================================================"
+echo " Starting Pangolin VPS Backup -> Pull to Unraid Server"
+echo " Remote VPS : {vps_user}@{vps_host}:{vps_port}"
+echo " Local Store: {local_archive_path}"
+echo "================================================================"
+
+# Step 1: Execute remote snapshot creation
+{ssh_prefix_str} << 'REMOTE_EXEC_EOF'
+{remote_script_one_liner}
+REMOTE_EXEC_EOF
+
+echo ""
+echo "=== STEP 2: Transferring backup archive to Unraid Server ==="
+mkdir -p "{backup_dir}"
+{scp_prefix_str} "{vps_user}@{vps_host}:{remote_tar}" "{local_archive_path}"
+
+echo ""
+echo "=== STEP 3: Cleaning up temporary staging on Remote VPS ==="
+{ssh_prefix_str} "rm -f '{remote_tar}'"
+
+echo ""
+echo "=== STEP 4: Verifying archive integrity on Unraid Server ==="
+tar -tzf "{local_archive_path}" > /dev/null && echo "Archive integrity check: PASSED"
+echo -n "Unraid Local SHA256: "
+sha256sum "{local_archive_path}"
+
+echo ""
+echo "=== BACKUP PULL COMPLETED SUCCESSFULLY ==="
+"""
+
+    res = await runner.run_command_job(
+        name=f"Backup VPS -> Unraid ({scope.upper()})",
+        cmd=unraid_script,
+        job_type="backup",
+        description=description or f"Pangolin VPS backup to Unraid ({scope})"
+    )
+
+    if res.get("success"):
+        asyncio.create_task(_monitor_and_index_backup(
+            res["job_id"], backup_id, local_archive_path, scope, description, auto_created
+        ))
+
+    return res
+
+async def _monitor_and_index_backup(
+    job_id: str,
+    backup_id: str,
+    archive_path: Path,
+    scope: str,
+    description: str,
+    auto_created: bool
+):
+    while True:
+        await asyncio.sleep(1)
+        job = runner.get_job(job_id)
+        if not job or job.get("status") in ["completed", "failed"]:
+            break
+
+    if job and job.get("status") == "completed" and archive_path.exists():
+        size_bytes = archive_path.stat().st_size
+        sha256 = compute_sha256(archive_path)
+
+        files_count = 0
+        try:
+            with tarfile.open(archive_path, "r:gz") as tar:
+                files_count = len(tar.getmembers())
+        except Exception:
+            pass
+
+        entry = {
+            "id": backup_id,
+            "filename": archive_path.name,
+            "path": str(archive_path),
+            "scope": scope,
+            "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "size_bytes": size_bytes,
+            "formatted_size": format_bytes(size_bytes),
+            "sha256": sha256,
+            "files_count": files_count,
+            "description": description,
+            "auto_created": auto_created,
+            "verified": True
+        }
+
+        index = load_backup_index()
+        index.insert(0, entry)
+        save_backup_index(index)
+
+        # Auto-prune if retention count exceeded
+        retention = get_setting("retention_count", 10)
+        prune_backups(retention)
+
+def list_backups() -> List[Dict[str, Any]]:
+    index = load_backup_index()
+    backup_dir = get_backup_dir()
+
+    valid = []
+    for item in index:
+        fp = Path(item.get("path", ""))
+        if fp.exists():
+            valid.append(item)
+
+    # Detect any unindexed .tar.gz in local backup_dir
+    known_files = {item["filename"] for item in valid}
+    for f in backup_dir.glob("pangolin-backup_*.tar.gz"):
+        if f.name not in known_files:
+            try:
+                sz = f.stat().st_size
+                mtime = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(f.stat().st_mtime))
+                h = compute_sha256(f)
+                valid.append({
+                    "id": f"bk_imported_{f.stem}",
+                    "filename": f.name,
+                    "path": str(f),
+                    "scope": "pangolin" if "pangolin" in f.name else "full",
+                    "created_at": mtime,
+                    "size_bytes": sz,
+                    "formatted_size": format_bytes(sz),
+                    "sha256": h,
+                    "files_count": 0,
+                    "description": "Detected in Unraid storage",
+                    "auto_created": False,
+                    "verified": True
+                })
+            except Exception:
+                pass
+
+    valid.sort(key=lambda x: x.get("created_at", ""), reverse=True)
+    if len(valid) != len(index):
+        save_backup_index(valid)
+    return valid
+
+def inspect_backup(backup_id: str) -> Dict[str, Any]:
+    index = load_backup_index()
+    target = next((b for b in index if b["id"] == backup_id), None)
+    if not target:
+        return {"success": False, "error": "Backup not found"}
+
+    filepath = Path(target["path"])
+    if not filepath.exists():
+        return {"success": False, "error": "Backup archive file not found on Unraid disk"}
+
+    members = []
+    try:
+        with tarfile.open(filepath, "r:gz") as tar:
+            for m in tar.getmembers():
+                members.append({
+                    "name": m.name,
+                    "size": m.size,
+                    "formatted_size": format_bytes(m.size),
+                    "is_dir": m.isdir(),
+                    "mtime": time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(m.mtime))
+                })
+        return {
+            "success": True,
+            "backup": target,
+            "total_files": len(members),
+            "members": members
+        }
+    except Exception as e:
+        return {"success": False, "error": f"Failed to inspect archive: {str(e)}"}
+
+def verify_backup(backup_id: str) -> Dict[str, Any]:
+    index = load_backup_index()
+    target = next((b for b in index if b["id"] == backup_id), None)
+    if not target:
+        return {"success": False, "error": "Backup not found"}
+
+    filepath = Path(target["path"])
+    if not filepath.exists():
+        return {"success": False, "error": "Backup file not found on Unraid disk"}
+
+    current_hash = compute_sha256(filepath)
+    hash_match = (current_hash == target.get("sha256"))
+
+    tar_valid = False
+    try:
+        with tarfile.open(filepath, "r:gz") as tar:
+            tar.getmembers()
+            tar_valid = True
+    except Exception:
+        tar_valid = False
+
+    is_verified = hash_match and tar_valid
+    target["verified"] = is_verified
+    save_backup_index(index)
+
+    return {
+        "success": True,
+        "verified": is_verified,
+        "sha256_match": hash_match,
+        "tar_integrity": tar_valid,
+        "expected_sha256": target.get("sha256"),
+        "computed_sha256": current_hash
+    }
+
+async def restore_backup(backup_id: str) -> Dict[str, Any]:
+    index = load_backup_index()
+    target = next((b for b in index if b["id"] == backup_id), None)
+    if not target:
+        return {"success": False, "error": "Backup not found"}
+
+    archive_path = Path(target["path"])
+    if not archive_path.exists():
+        return {"success": False, "error": "Archive file does not exist on Unraid disk"}
+
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    p_dir = settings.get("vps_pangolin_dir", "/opt/pangolin").strip()
+    if not vps_host:
+        return {"success": False, "error": "No VPS host configured."}
+
+    ssh_prefix_str = " ".join(build_ssh_prefix(settings))
+    scp_prefix_str = " ".join(build_scp_prefix(settings))
+    vps_user = settings.get("vps_user", "root")
+
+    remote_tar = f"/tmp/restore_{target['filename']}"
+    remote_restore_stage = f"/tmp/restore_stage_{int(time.time())}"
+
+    # Script: Upload archive to VPS -> extract and restore -> restart containers
+    restore_script = f"""
+set -e
+echo "================================================================"
+echo " RESTORING BACKUP FROM UNRAID TO REMOTE VPS ({vps_host})"
+echo " Archive: {target['filename']}"
+echo "================================================================"
+
+echo ""
+echo "=== STEP 1: Uploading backup archive from Unraid to VPS ==="
+{scp_prefix_str} "{archive_path}" "{vps_user}@{vps_host}:{remote_tar}"
+
+echo ""
+echo "=== STEP 2: Applying restore on remote VPS ==="
+{ssh_prefix_str} << 'REMOTE_RESTORE_EOF'
+set -e
+mkdir -p "{remote_restore_stage}"
+tar -xzf "{remote_tar}" -C "{remote_restore_stage}"
+
+# Stop containers on VPS
+if [ -d "{p_dir}" ]; then
+    echo "Stopping existing containers on VPS..."
+    cd "{p_dir}" && docker compose down 2>/dev/null || true
+fi
+
+# Restore Pangolin files
+if [ -d "{remote_restore_stage}/pangolin" ]; then
+    echo "Restoring Pangolin directory files on VPS..."
+    mkdir -p "{p_dir}"
+    rsync -av "{remote_restore_stage}/pangolin/" "{p_dir}/"
+fi
+
+# Restore VPS Config if present
+if [ -d "{remote_restore_stage}/vps_config" ]; then
+    echo "Restoring VPS system configurations..."
+    rsync -av "{remote_restore_stage}/vps_config/" "/"
+fi
+
+# Restart containers on VPS
+if [ -f "{p_dir}/docker-compose.yml" ] || [ -f "{p_dir}/compose.yaml" ]; then
+    echo "Starting Pangolin containers on VPS..."
+    cd "{p_dir}" && docker compose up -d
+    docker compose ps
+fi
+
+# Cleanup VPS tmp
+rm -rf "{remote_restore_stage}" "{remote_tar}"
+echo "Remote VPS restore completed successfully!"
+REMOTE_RESTORE_EOF
+
+echo ""
+echo "=== RESTORE OPERATION COMPLETED SUCCESSFULLY ==="
+"""
+
+    res = await runner.run_command_job(
+        name=f"Restore to VPS ({target['filename']})",
+        cmd=restore_script,
+        job_type="restore",
+        description=f"Restore backup {backup_id} from Unraid to remote VPS ({vps_host})"
+    )
+    return res
+
+def delete_backup(backup_id: str) -> Dict[str, Any]:
+    index = load_backup_index()
+    target = next((b for b in index if b["id"] == backup_id), None)
+    if not target:
+        return {"success": False, "error": "Backup not found"}
+
+    try:
+        p = Path(target["path"])
+        if p.exists():
+            p.unlink()
+    except Exception as e:
+        print(f"Error removing local file {target['path']}: {e}")
+
+    new_index = [b for b in index if b["id"] != backup_id]
+    save_backup_index(new_index)
+    return {"success": True, "message": "Backup deleted from Unraid server"}
+
+def prune_backups(keep_count: int = 10) -> int:
+    index = load_backup_index()
+    if len(index) <= keep_count:
+        return 0
+
+    to_prune = index[keep_count:]
+    kept = index[:keep_count]
+
+    pruned = 0
+    for item in to_prune:
+        try:
+            p = Path(item["path"])
+            if p.exists():
+                p.unlink()
+            pruned += 1
+        except Exception:
+            pass
+
+    save_backup_index(kept)
+    return pruned