Initial commit: Pangolin Sentinel Docker & Unraid Hub

backend/auth.py

@@ -0,0 +1,88 @@
+import hashlib
+import hmac
+import os
+import secrets
+import time
+from typing import Optional, Tuple
+from fastapi import Request, HTTPException, Depends, status
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from backend.config import get_setting, update_setting, load_settings
+
+security = HTTPBearer(auto_error=False)
+
+def hash_password(password: str, salt: Optional[str] = None) -> Tuple[str, str]:
+    if not salt:
+        salt = secrets.token_hex(16)
+    pwd_bytes = password.encode('utf-8')
+    salt_bytes = salt.encode('utf-8')
+    hash_bytes = hashlib.pbkdf2_hmac('sha256', pwd_bytes, salt_bytes, 100000)
+    return hash_bytes.hex(), salt
+
+def verify_password(password: str, password_hash: str, salt: str) -> bool:
+    new_hash, _ = hash_password(password, salt)
+    return hmac.compare_digest(new_hash, password_hash)
+
+def create_session_token(username: str = "admin", expires_in_seconds: int = 86400 * 7) -> str:
+    secret = get_setting("jwt_secret", "fallback-secret-key-32chars-min-ok")
+    expiry = int(time.time()) + expires_in_seconds
+    payload = f"{username}:{expiry}"
+    signature = hmac.new(secret.encode('utf-8'), payload.encode('utf-8'), hashlib.sha256).hexdigest()
+    return f"{payload}:{signature}"
+
+def verify_session_token(token: str) -> bool:
+    try:
+        secret = get_setting("jwt_secret", "fallback-secret-key-32chars-min-ok")
+        parts = token.split(":")
+        if len(parts) != 3:
+            return False
+        username, expiry_str, signature = parts
+        expiry = int(expiry_str)
+        if time.time() > expiry:
+            return False
+        expected_payload = f"{username}:{expiry_str}"
+        expected_sig = hmac.new(secret.encode('utf-8'), expected_payload.encode('utf-8'), hashlib.sha256).hexdigest()
+        return hmac.compare_digest(signature, expected_sig)
+    except Exception:
+        return False
+
+def is_setup_required() -> bool:
+    settings = load_settings()
+    return not bool(settings.get("admin_password_hash"))
+
+async def require_auth(
+    request: Request,
+    credentials: Optional[HTTPAuthorizationCredentials] = Depends(security)
+) -> bool:
+    settings = load_settings()
+    if not settings.get("auth_enabled", True):
+        return True
+    
+    # If setup is required, permit setup endpoints but block others
+    if is_setup_required():
+        path = request.url.path
+        if path in ["/api/auth/status", "/api/auth/setup", "/api/auth/login"]:
+            return True
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Initial setup required. Please configure admin password."
+        )
+
+    token = None
+    # 1. Check Bearer Authorization header
+    if credentials:
+        token = credentials.credentials
+    
+    # 2. Check Cookie
+    if not token and "session_token" in request.cookies:
+        token = request.cookies.get("session_token")
+    
+    # 3. Check Query Parameter (for WebSockets / EventSource / direct download links)
+    if not token and "token" in request.query_params:
+        token = request.query_params.get("token")
+
+    if not token or not verify_session_token(token):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Unauthorized. Please log in."
+        )
+    return True

backend/backup.py

@@ -0,0 +1,454 @@
+import asyncio
+import hashlib
+import json
+import os
+import shutil
+import tarfile
+import time
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+from backend.config import load_settings, get_setting, BACKUP_INDEX_FILE, DEFAULT_LOCAL_BACKUP, DATA_DIR
+from backend.runner import runner
+from backend.remote import run_remote_quick, build_ssh_prefix, build_scp_prefix
+from backend.system import format_bytes
+
+def get_backup_dir() -> Path:
+    s = load_settings()
+    configured = s.get("backup_dir", "").strip()
+    if configured:
+        bdir = Path(configured)
+    else:
+        bdir = Path(DEFAULT_LOCAL_BACKUP)
+
+    try:
+        bdir.mkdir(parents=True, exist_ok=True)
+        return bdir
+    except Exception as e:
+        print(f"Warning: could not create backup directory {bdir}: {e}")
+        fallback = Path("/backups") if os.path.exists("/backups") else (DATA_DIR / "backups")
+        fallback.mkdir(parents=True, exist_ok=True)
+        return fallback
+
+def load_backup_index() -> List[Dict[str, Any]]:
+    if not BACKUP_INDEX_FILE.exists():
+        return []
+    try:
+        with open(BACKUP_INDEX_FILE, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception:
+        return []
+
+def save_backup_index(index: List[Dict[str, Any]]) -> None:
+    try:
+        BACKUP_INDEX_FILE.parent.mkdir(parents=True, exist_ok=True)
+        with open(BACKUP_INDEX_FILE, "w", encoding="utf-8") as f:
+            json.dump(index, f, indent=2)
+    except Exception as e:
+        print(f"Error saving backup index: {e}")
+
+def compute_sha256(filepath: Path) -> str:
+    h = hashlib.sha256()
+    with open(filepath, "rb") as f:
+        while chunk := f.read(65536):
+            h.update(chunk)
+    return h.hexdigest()
+
+async def run_backup_task(
+    scope: str = "pangolin",
+    description: str = "",
+    auto_created: bool = False
+) -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    p_dir = settings.get("vps_pangolin_dir", "/opt/pangolin").strip()
+    backup_dir = get_backup_dir()
+
+    if not vps_host:
+        return {"success": False, "error": "No VPS host configured. Please configure your VPS in Settings first."}
+
+    timestamp = time.strftime("%Y%m%d_%H%M%S")
+    backup_id = f"bk_{timestamp}_{int(time.time()) % 10000:04d}"
+    archive_name = f"pangolin-backup_{scope}_{timestamp}.tar.gz"
+    local_archive_path = backup_dir / archive_name
+    remote_stage = f"/tmp/pangolin_snap_{backup_id}"
+    remote_tar = f"/tmp/{archive_name}"
+
+    # Build multi-step backup workflow script executed on Unraid server
+    # 1. SSH to VPS: stage files, hot backup SQLite, tar on VPS
+    # 2. SCP/rsync from VPS to Unraid server
+    # 3. Clean up remote VPS /tmp
+    # 4. Verify local archive on Unraid server
+
+    ssh_prefix_str = " ".join(build_ssh_prefix(settings))
+    scp_prefix_str = " ".join(build_scp_prefix(settings))
+    vps_user = settings.get("vps_user", "root")
+    vps_port = str(settings.get("vps_port", 22))
+
+    remote_commands = [
+        f'echo "=== STEP 1: Creating Hot-Snapshot on Remote VPS ({vps_host}) ==="',
+        f'mkdir -p "{remote_stage}/pangolin"',
+        f'echo "--- Staging Pangolin files from {p_dir} ---"',
+        f'if [ -d "{p_dir}" ]; then rsync -av --exclude="*.db" --exclude="*.db-wal" --exclude="*.db-shm" "{p_dir}/" "{remote_stage}/pangolin/"; fi',
+        f'echo "--- Performing zero-lock hot-backup of SQLite databases on VPS ---"',
+        f"""
+for db in $(find "{p_dir}" -name "*.db" -o -name "*.sqlite" 2>/dev/null); do
+    rel=$(realpath --relative-to="{p_dir}" "$db")
+    mkdir -p "{remote_stage}/pangolin/$(dirname "$rel")"
+    echo "Hot-backing up $rel..."
+    sqlite3 "$db" ".backup '{remote_stage}/pangolin/$rel'" 2>/dev/null || cp -v "$db" "{remote_stage}/pangolin/$rel"
+done
+"""
+    ]
+
+    if scope in ["vps_config", "full"]:
+        remote_commands.append(f"""
+echo "--- Staging VPS Host Configurations (/etc/ufw, network, cron) ---"
+mkdir -p "{remote_stage}/vps_config/etc"
+for t in /etc/ufw /etc/network /etc/netplan /etc/hosts /etc/hostname /etc/resolv.conf /etc/systemd/system/pangolin*.service /etc/cron* /var/spool/cron; do
+    [ -e "$t" ] && cp -r --parents "$t" "{remote_stage}/vps_config/" 2>/dev/null || true
+done
+""")
+
+    # Metadata & Tar on VPS
+    remote_commands.append(f"""
+cat << 'MANIFEST_EOF' > "{remote_stage}/backup_manifest.json"
+{{
+  "id": "{backup_id}",
+  "scope": "{scope}",
+  "source_vps": "{vps_host}",
+  "timestamp": "{timestamp}",
+  "created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+  "description": "{description}",
+  "auto_created": {str(auto_created).lower()}
+}}
+MANIFEST_EOF
+
+echo "--- Compressing snapshot on VPS into {remote_tar} ---"
+tar -czf "{remote_tar}" -C "{remote_stage}" .
+rm -rf "{remote_stage}"
+echo "Remote snapshot prepared successfully."
+""")
+
+    remote_script_one_liner = "\n".join(remote_commands)
+
+    # Full bash execution pipeline on Unraid server
+    unraid_script = f"""
+set -e
+echo "================================================================"
+echo " Starting Pangolin VPS Backup -> Pull to Unraid Server"
+echo " Remote VPS : {vps_user}@{vps_host}:{vps_port}"
+echo " Local Store: {local_archive_path}"
+echo "================================================================"
+
+# Step 1: Execute remote snapshot creation
+{ssh_prefix_str} << 'REMOTE_EXEC_EOF'
+{remote_script_one_liner}
+REMOTE_EXEC_EOF
+
+echo ""
+echo "=== STEP 2: Transferring backup archive to Unraid Server ==="
+mkdir -p "{backup_dir}"
+{scp_prefix_str} "{vps_user}@{vps_host}:{remote_tar}" "{local_archive_path}"
+
+echo ""
+echo "=== STEP 3: Cleaning up temporary staging on Remote VPS ==="
+{ssh_prefix_str} "rm -f '{remote_tar}'"
+
+echo ""
+echo "=== STEP 4: Verifying archive integrity on Unraid Server ==="
+tar -tzf "{local_archive_path}" > /dev/null && echo "Archive integrity check: PASSED"
+echo -n "Unraid Local SHA256: "
+sha256sum "{local_archive_path}"
+
+echo ""
+echo "=== BACKUP PULL COMPLETED SUCCESSFULLY ==="
+"""
+
+    res = await runner.run_command_job(
+        name=f"Backup VPS -> Unraid ({scope.upper()})",
+        cmd=unraid_script,
+        job_type="backup",
+        description=description or f"Pangolin VPS backup to Unraid ({scope})"
+    )
+
+    if res.get("success"):
+        asyncio.create_task(_monitor_and_index_backup(
+            res["job_id"], backup_id, local_archive_path, scope, description, auto_created
+        ))
+
+    return res
+
+async def _monitor_and_index_backup(
+    job_id: str,
+    backup_id: str,
+    archive_path: Path,
+    scope: str,
+    description: str,
+    auto_created: bool
+):
+    while True:
+        await asyncio.sleep(1)
+        job = runner.get_job(job_id)
+        if not job or job.get("status") in ["completed", "failed"]:
+            break
+
+    if job and job.get("status") == "completed" and archive_path.exists():
+        size_bytes = archive_path.stat().st_size
+        sha256 = compute_sha256(archive_path)
+
+        files_count = 0
+        try:
+            with tarfile.open(archive_path, "r:gz") as tar:
+                files_count = len(tar.getmembers())
+        except Exception:
+            pass
+
+        entry = {
+            "id": backup_id,
+            "filename": archive_path.name,
+            "path": str(archive_path),
+            "scope": scope,
+            "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "size_bytes": size_bytes,
+            "formatted_size": format_bytes(size_bytes),
+            "sha256": sha256,
+            "files_count": files_count,
+            "description": description,
+            "auto_created": auto_created,
+            "verified": True
+        }
+
+        index = load_backup_index()
+        index.insert(0, entry)
+        save_backup_index(index)
+
+        # Auto-prune if retention count exceeded
+        retention = get_setting("retention_count", 10)
+        prune_backups(retention)
+
+def list_backups() -> List[Dict[str, Any]]:
+    index = load_backup_index()
+    backup_dir = get_backup_dir()
+
+    valid = []
+    for item in index:
+        fp = Path(item.get("path", ""))
+        if fp.exists():
+            valid.append(item)
+
+    # Detect any unindexed .tar.gz in local backup_dir
+    known_files = {item["filename"] for item in valid}
+    for f in backup_dir.glob("pangolin-backup_*.tar.gz"):
+        if f.name not in known_files:
+            try:
+                sz = f.stat().st_size
+                mtime = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(f.stat().st_mtime))
+                h = compute_sha256(f)
+                valid.append({
+                    "id": f"bk_imported_{f.stem}",
+                    "filename": f.name,
+                    "path": str(f),
+                    "scope": "pangolin" if "pangolin" in f.name else "full",
+                    "created_at": mtime,
+                    "size_bytes": sz,
+                    "formatted_size": format_bytes(sz),
+                    "sha256": h,
+                    "files_count": 0,
+                    "description": "Detected in Unraid storage",
+                    "auto_created": False,
+                    "verified": True
+                })
+            except Exception:
+                pass
+
+    valid.sort(key=lambda x: x.get("created_at", ""), reverse=True)
+    if len(valid) != len(index):
+        save_backup_index(valid)
+    return valid
+
+def inspect_backup(backup_id: str) -> Dict[str, Any]:
+    index = load_backup_index()
+    target = next((b for b in index if b["id"] == backup_id), None)
+    if not target:
+        return {"success": False, "error": "Backup not found"}
+
+    filepath = Path(target["path"])
+    if not filepath.exists():
+        return {"success": False, "error": "Backup archive file not found on Unraid disk"}
+
+    members = []
+    try:
+        with tarfile.open(filepath, "r:gz") as tar:
+            for m in tar.getmembers():
+                members.append({
+                    "name": m.name,
+                    "size": m.size,
+                    "formatted_size": format_bytes(m.size),
+                    "is_dir": m.isdir(),
+                    "mtime": time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(m.mtime))
+                })
+        return {
+            "success": True,
+            "backup": target,
+            "total_files": len(members),
+            "members": members
+        }
+    except Exception as e:
+        return {"success": False, "error": f"Failed to inspect archive: {str(e)}"}
+
+def verify_backup(backup_id: str) -> Dict[str, Any]:
+    index = load_backup_index()
+    target = next((b for b in index if b["id"] == backup_id), None)
+    if not target:
+        return {"success": False, "error": "Backup not found"}
+
+    filepath = Path(target["path"])
+    if not filepath.exists():
+        return {"success": False, "error": "Backup file not found on Unraid disk"}
+
+    current_hash = compute_sha256(filepath)
+    hash_match = (current_hash == target.get("sha256"))
+
+    tar_valid = False
+    try:
+        with tarfile.open(filepath, "r:gz") as tar:
+            tar.getmembers()
+            tar_valid = True
+    except Exception:
+        tar_valid = False
+
+    is_verified = hash_match and tar_valid
+    target["verified"] = is_verified
+    save_backup_index(index)
+
+    return {
+        "success": True,
+        "verified": is_verified,
+        "sha256_match": hash_match,
+        "tar_integrity": tar_valid,
+        "expected_sha256": target.get("sha256"),
+        "computed_sha256": current_hash
+    }
+
+async def restore_backup(backup_id: str) -> Dict[str, Any]:
+    index = load_backup_index()
+    target = next((b for b in index if b["id"] == backup_id), None)
+    if not target:
+        return {"success": False, "error": "Backup not found"}
+
+    archive_path = Path(target["path"])
+    if not archive_path.exists():
+        return {"success": False, "error": "Archive file does not exist on Unraid disk"}
+
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    p_dir = settings.get("vps_pangolin_dir", "/opt/pangolin").strip()
+    if not vps_host:
+        return {"success": False, "error": "No VPS host configured."}
+
+    ssh_prefix_str = " ".join(build_ssh_prefix(settings))
+    scp_prefix_str = " ".join(build_scp_prefix(settings))
+    vps_user = settings.get("vps_user", "root")
+
+    remote_tar = f"/tmp/restore_{target['filename']}"
+    remote_restore_stage = f"/tmp/restore_stage_{int(time.time())}"
+
+    # Script: Upload archive to VPS -> extract and restore -> restart containers
+    restore_script = f"""
+set -e
+echo "================================================================"
+echo " RESTORING BACKUP FROM UNRAID TO REMOTE VPS ({vps_host})"
+echo " Archive: {target['filename']}"
+echo "================================================================"
+
+echo ""
+echo "=== STEP 1: Uploading backup archive from Unraid to VPS ==="
+{scp_prefix_str} "{archive_path}" "{vps_user}@{vps_host}:{remote_tar}"
+
+echo ""
+echo "=== STEP 2: Applying restore on remote VPS ==="
+{ssh_prefix_str} << 'REMOTE_RESTORE_EOF'
+set -e
+mkdir -p "{remote_restore_stage}"
+tar -xzf "{remote_tar}" -C "{remote_restore_stage}"
+
+# Stop containers on VPS
+if [ -d "{p_dir}" ]; then
+    echo "Stopping existing containers on VPS..."
+    cd "{p_dir}" && docker compose down 2>/dev/null || true
+fi
+
+# Restore Pangolin files
+if [ -d "{remote_restore_stage}/pangolin" ]; then
+    echo "Restoring Pangolin directory files on VPS..."
+    mkdir -p "{p_dir}"
+    rsync -av "{remote_restore_stage}/pangolin/" "{p_dir}/"
+fi
+
+# Restore VPS Config if present
+if [ -d "{remote_restore_stage}/vps_config" ]; then
+    echo "Restoring VPS system configurations..."
+    rsync -av "{remote_restore_stage}/vps_config/" "/"
+fi
+
+# Restart containers on VPS
+if [ -f "{p_dir}/docker-compose.yml" ] || [ -f "{p_dir}/compose.yaml" ]; then
+    echo "Starting Pangolin containers on VPS..."
+    cd "{p_dir}" && docker compose up -d
+    docker compose ps
+fi
+
+# Cleanup VPS tmp
+rm -rf "{remote_restore_stage}" "{remote_tar}"
+echo "Remote VPS restore completed successfully!"
+REMOTE_RESTORE_EOF
+
+echo ""
+echo "=== RESTORE OPERATION COMPLETED SUCCESSFULLY ==="
+"""
+
+    res = await runner.run_command_job(
+        name=f"Restore to VPS ({target['filename']})",
+        cmd=restore_script,
+        job_type="restore",
+        description=f"Restore backup {backup_id} from Unraid to remote VPS ({vps_host})"
+    )
+    return res
+
+def delete_backup(backup_id: str) -> Dict[str, Any]:
+    index = load_backup_index()
+    target = next((b for b in index if b["id"] == backup_id), None)
+    if not target:
+        return {"success": False, "error": "Backup not found"}
+
+    try:
+        p = Path(target["path"])
+        if p.exists():
+            p.unlink()
+    except Exception as e:
+        print(f"Error removing local file {target['path']}: {e}")
+
+    new_index = [b for b in index if b["id"] != backup_id]
+    save_backup_index(new_index)
+    return {"success": True, "message": "Backup deleted from Unraid server"}
+
+def prune_backups(keep_count: int = 10) -> int:
+    index = load_backup_index()
+    if len(index) <= keep_count:
+        return 0
+
+    to_prune = index[keep_count:]
+    kept = index[:keep_count]
+
+    pruned = 0
+    for item in to_prune:
+        try:
+            p = Path(item["path"])
+            if p.exists():
+                p.unlink()
+            pruned += 1
+        except Exception:
+            pass
+
+    save_backup_index(kept)
+    return pruned

backend/config.py

@@ -0,0 +1,111 @@
+import os
+import json
+import secrets
+from pathlib import Path
+from typing import Dict, Any
+
+# Project base directory: can be overridden via APP_DIR or auto-detected from file location
+DEFAULT_APP_DIR = Path(__file__).resolve().parent.parent
+BASE_DIR = Path(os.environ.get("APP_DIR", str(DEFAULT_APP_DIR)))
+
+# Persistent data directory: holds settings, SSH keys, job history, and backup index
+DATA_DIR = Path(os.environ.get("DATA_DIR", str(BASE_DIR / "data")))
+
+CONFIG_FILE = DATA_DIR / "settings.json"
+BACKUP_INDEX_FILE = DATA_DIR / "backups_index.json"
+JOBS_FILE = DATA_DIR / "jobs_history.json"
+DEFAULT_KEY_FILE = DATA_DIR / "id_ed25519"
+DEFAULT_PUB_KEY_FILE = DATA_DIR / "id_ed25519.pub"
+
+def get_default_backup_dir() -> str:
+    if "BACKUP_DIR" in os.environ and os.environ["BACKUP_DIR"].strip():
+        return os.environ["BACKUP_DIR"].strip()
+    if os.path.exists("/backups"):
+        return "/backups"
+    if os.path.exists("/mnt/user"):
+        return "/mnt/user/backups/pangolin"
+    if os.path.exists("/var/backups"):
+        return "/var/backups/pangolin"
+    return str(DATA_DIR / "backups")
+
+DEFAULT_LOCAL_BACKUP = get_default_backup_dir()
+
+DEFAULT_SETTINGS = {
+    "auth_enabled": True,
+    "admin_password_hash": "",
+    "admin_password_salt": "",
+    "jwt_secret": secrets.token_hex(32),
+    
+    # Remote VPS Configuration
+    "vps_host": "",                  # IP or Hostname of the Pangolin VPS
+    "vps_port": 22,                  # SSH Port
+    "vps_user": "root",              # SSH Username
+    "vps_auth_method": "key",        # "key" or "password"
+    "vps_ssh_key_path": str(DEFAULT_KEY_FILE),
+    "vps_ssh_password": "",          # SSH Password (if method == "password")
+    "vps_pangolin_dir": "/opt/pangolin", # Path to Pangolin on remote VPS
+    
+    # Local Unraid Storage & Retention
+    "backup_dir": DEFAULT_LOCAL_BACKUP,
+    "retention_count": 10,
+    
+    # Automation & Scheduler
+    "scheduled_backup_enabled": True,
+    "scheduled_backup_cron": "0 3 * * *",
+    "scheduled_backup_scope": "pangolin",
+    "scheduled_update_check_enabled": True,
+    
+    # Webhooks
+    "webhook_url": "",
+    "webhook_type": "discord",
+    "telegram_bot_token": "",
+    "telegram_chat_id": "",
+    "notify_on_backup": True,
+    "notify_on_update": True,
+    
+    # App daemon
+    "app_port": int(os.environ.get("PORT", 8888)),
+    "app_host": os.environ.get("HOST", "0.0.0.0")
+}
+
+def load_settings() -> Dict[str, Any]:
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+    if not CONFIG_FILE.exists():
+        save_settings(DEFAULT_SETTINGS)
+        return DEFAULT_SETTINGS.copy()
+    
+    try:
+        with open(CONFIG_FILE, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            for k, v in DEFAULT_SETTINGS.items():
+                if k not in data:
+                    data[k] = v
+            
+            # Auto-heal SSH key path if configured path does not exist but default key does
+            cur_key_path = data.get("vps_ssh_key_path", "")
+            if (not cur_key_path or not os.path.exists(cur_key_path)) and DEFAULT_KEY_FILE.exists():
+                data["vps_ssh_key_path"] = str(DEFAULT_KEY_FILE)
+
+            # Auto-heal backup_dir if set to old non-existent path and /backups is mounted in container
+            cur_backup_dir = data.get("backup_dir", "")
+            if os.path.exists("/backups") and (not cur_backup_dir or not os.path.exists(cur_backup_dir)):
+                data["backup_dir"] = "/backups"
+
+            return data
+    except Exception as e:
+        print(f"Error loading settings: {e}. Using defaults.")
+        return DEFAULT_SETTINGS.copy()
+
+def save_settings(settings: Dict[str, Any]) -> None:
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+    with open(CONFIG_FILE, "w", encoding="utf-8") as f:
+        json.dump(settings, f, indent=2)
+
+def get_setting(key: str, default: Any = None) -> Any:
+    settings = load_settings()
+    return settings.get(key, default)
+
+def update_setting(key: str, value: Any) -> None:
+    settings = load_settings()
+    settings[key] = value
+    save_settings(settings)

backend/main.py

@@ -0,0 +1,422 @@
+import asyncio
+import os
+import shutil
+import sys
+import time
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+# Ensure project root is in sys.path so backend modules can be imported directly
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from fastapi import (
+    FastAPI, HTTPException, Depends, Request, Response,
+    WebSocket, WebSocketDisconnect, UploadFile, File, Form, status
+)
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+
+from backend.config import (
+    load_settings, save_settings, get_setting, update_setting,
+    BASE_DIR, DATA_DIR, DEFAULT_KEY_FILE, DEFAULT_PUB_KEY_FILE
+)
+from backend.auth import (
+    require_auth, hash_password, verify_password,
+    create_session_token, is_setup_required
+)
+from backend.system import (
+    get_system_stats, check_upgradable_packages,
+    trigger_system_upgrade, trigger_system_reboot
+)
+from backend.pangolin import (
+    get_pangolin_stack_status,
+    trigger_pangolin_update, container_action,
+    get_container_logs, get_stack_config_files,
+    save_stack_config_file
+)
+from backend.backup import (
+    list_backups, run_backup_task, inspect_backup,
+    verify_backup, restore_backup, delete_backup,
+    get_backup_dir, load_backup_index, save_backup_index,
+    compute_sha256
+)
+from backend.remote import (
+    get_public_key, ensure_ssh_key, test_vps_connection
+)
+from backend.runner import runner
+from backend.scheduler import start_scheduler, shutdown_scheduler, reload_scheduler
+from backend.notifier import test_webhook, send_notification
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    print("[APP] Initializing Pangolin Sentinel (Unraid -> VPS)...")
+    ensure_ssh_key()
+    start_scheduler()
+    yield
+    print("[APP] Shutting down Pangolin Sentinel...")
+    shutdown_scheduler()
+
+app = FastAPI(
+    title="Pangolin Sentinel — Unraid to VPS Hub",
+    description="Full web app on Unraid for updating and backing up remote Pangolin VPS",
+    version="2.0.0",
+    lifespan=lifespan
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+FRONTEND_DIR = BASE_DIR / "frontend"
+ASSETS_DIR = FRONTEND_DIR / "assets"
+ASSETS_DIR.mkdir(parents=True, exist_ok=True)
+
+app.mount("/assets", StaticFiles(directory=str(ASSETS_DIR)), name="assets")
+
+# ----------------- Auth Endpoints -----------------
+
+class SetupRequest(BaseModel):
+    password: str
+
+class LoginRequest(BaseModel):
+    password: str
+
+class ChangePasswordRequest(BaseModel):
+    old_password: str
+    new_password: str
+
+@app.get("/api/auth/status")
+async def get_auth_status():
+    settings = load_settings()
+    return {
+        "auth_enabled": settings.get("auth_enabled", True),
+        "setup_required": is_setup_required()
+    }
+
+@app.post("/api/auth/setup")
+async def initial_setup(req: SetupRequest, response: Response):
+    if not is_setup_required():
+        raise HTTPException(status_code=400, detail="Admin password is already configured.")
+    if len(req.password) < 6:
+        raise HTTPException(status_code=400, detail="Password must be at least 6 characters.")
+    
+    pwd_hash, salt = hash_password(req.password)
+    update_setting("admin_password_hash", pwd_hash)
+    update_setting("admin_password_salt", salt)
+    
+    token = create_session_token()
+    response.set_cookie("session_token", token, httponly=True, max_age=86400 * 7, samesite="lax")
+    return {"success": True, "token": token}
+
+@app.post("/api/auth/login")
+async def login(req: LoginRequest, response: Response):
+    settings = load_settings()
+    if not settings.get("auth_enabled", True):
+        token = create_session_token()
+        return {"success": True, "token": token}
+
+    pwd_hash = settings.get("admin_password_hash", "")
+    salt = settings.get("admin_password_salt", "")
+
+    if not pwd_hash:
+        raise HTTPException(status_code=400, detail="Setup required. Please set admin password first.")
+
+    if not verify_password(req.password, pwd_hash, salt):
+        raise HTTPException(status_code=401, detail="Invalid password.")
+
+    token = create_session_token()
+    response.set_cookie("session_token", token, httponly=True, max_age=86400 * 7, samesite="lax")
+    return {"success": True, "token": token}
+
+@app.post("/api/auth/logout")
+async def logout(response: Response):
+    response.delete_cookie("session_token")
+    return {"success": True}
+
+@app.post("/api/auth/change-password", dependencies=[Depends(require_auth)])
+async def change_password(req: ChangePasswordRequest):
+    settings = load_settings()
+    pwd_hash = settings.get("admin_password_hash", "")
+    salt = settings.get("admin_password_salt", "")
+
+    if not verify_password(req.old_password, pwd_hash, salt):
+        raise HTTPException(status_code=400, detail="Incorrect current password.")
+
+    if len(req.new_password) < 6:
+        raise HTTPException(status_code=400, detail="New password must be at least 6 characters.")
+
+    new_hash, new_salt = hash_password(req.new_password)
+    update_setting("admin_password_hash", new_hash)
+    update_setting("admin_password_salt", new_salt)
+    return {"success": True, "message": "Password updated successfully."}
+
+# ----------------- Remote VPS Connection Endpoints -----------------
+
+@app.get("/api/remote/public-key", dependencies=[Depends(require_auth)])
+async def api_get_public_key():
+    pub = ensure_ssh_key()
+    return {
+        "public_key": pub,
+        "key_type": "ed25519",
+        "command_hint": f'echo "{pub}" >> ~/.ssh/authorized_keys'
+    }
+
+class TestConnectionRequest(BaseModel):
+    settings: Optional[Dict[str, Any]] = None
+
+@app.post("/api/remote/test-connection", dependencies=[Depends(require_auth)])
+async def api_test_vps_connection(req: TestConnectionRequest):
+    res = await test_vps_connection(req.settings)
+    return res
+
+# ----------------- System Endpoints (Remote VPS) -----------------
+
+@app.get("/api/system/stats", dependencies=[Depends(require_auth)])
+async def api_system_stats():
+    return await get_system_stats()
+
+@app.get("/api/system/updates", dependencies=[Depends(require_auth)])
+async def api_check_updates():
+    return await check_upgradable_packages()
+
+class UpgradeRequest(BaseModel):
+    dist_upgrade: bool = False
+
+@app.post("/api/system/upgrade", dependencies=[Depends(require_auth)])
+async def api_trigger_upgrade(req: UpgradeRequest):
+    res = await trigger_system_upgrade(req.dist_upgrade)
+    if not res.get("success"):
+        raise HTTPException(status_code=409, detail=res.get("error"))
+    return res
+
+@app.post("/api/system/reboot", dependencies=[Depends(require_auth)])
+async def api_trigger_reboot():
+    res = await trigger_system_reboot()
+    return res
+
+# ----------------- Pangolin Stack Endpoints (Remote VPS) -----------------
+
+@app.get("/api/pangolin/status", dependencies=[Depends(require_auth)])
+async def api_pangolin_status():
+    return await get_pangolin_stack_status()
+
+@app.post("/api/pangolin/update", dependencies=[Depends(require_auth)])
+async def api_trigger_pangolin_update():
+    res = await trigger_pangolin_update()
+    if not res.get("success"):
+        raise HTTPException(status_code=400, detail=res.get("error"))
+    return res
+
+class ContainerActionRequest(BaseModel):
+    action: str
+
+@app.post("/api/pangolin/containers/{name}/action", dependencies=[Depends(require_auth)])
+async def api_container_action(name: str, req: ContainerActionRequest):
+    res = await container_action(name, req.action)
+    if not res.get("success"):
+        raise HTTPException(status_code=400, detail=res.get("error"))
+    return res
+
+@app.get("/api/pangolin/containers/{name}/logs", dependencies=[Depends(require_auth)])
+async def api_container_logs(name: str, lines: int = 150):
+    return await get_container_logs(name, lines)
+
+@app.get("/api/pangolin/files", dependencies=[Depends(require_auth)])
+async def api_get_stack_files():
+    return await get_stack_config_files()
+
+class SaveFileRequest(BaseModel):
+    filename: str
+    content: str
+
+@app.post("/api/pangolin/files", dependencies=[Depends(require_auth)])
+async def api_save_stack_file(req: SaveFileRequest):
+    res = await save_stack_config_file(req.filename, req.content)
+    if not res.get("success"):
+        raise HTTPException(status_code=400, detail=res.get("error"))
+    return res
+
+# ----------------- Backups Endpoints (Stored on Unraid) -----------------
+
+@app.get("/api/backups", dependencies=[Depends(require_auth)])
+async def api_list_backups():
+    return list_backups()
+
+class CreateBackupRequest(BaseModel):
+    scope: str = "pangolin"
+    description: str = ""
+
+@app.post("/api/backups", dependencies=[Depends(require_auth)])
+async def api_create_backup(req: CreateBackupRequest):
+    res = await run_backup_task(
+        scope=req.scope,
+        description=req.description or "Manual backup pulled from VPS to Unraid",
+        auto_created=False
+    )
+    if not res.get("success"):
+        raise HTTPException(status_code=409, detail=res.get("error"))
+    return res
+
+@app.get("/api/backups/{backup_id}/inspect", dependencies=[Depends(require_auth)])
+async def api_inspect_backup(backup_id: str):
+    res = inspect_backup(backup_id)
+    if not res.get("success"):
+        raise HTTPException(status_code=404, detail=res.get("error"))
+    return res
+
+@app.post("/api/backups/{backup_id}/verify", dependencies=[Depends(require_auth)])
+async def api_verify_backup(backup_id: str):
+    res = verify_backup(backup_id)
+    if not res.get("success"):
+        raise HTTPException(status_code=404, detail=res.get("error"))
+    return res
+
+@app.post("/api/backups/{backup_id}/restore", dependencies=[Depends(require_auth)])
+async def api_restore_backup(backup_id: str):
+    res = await restore_backup(backup_id)
+    if not res.get("success"):
+        raise HTTPException(status_code=409, detail=res.get("error"))
+    return res
+
+@app.delete("/api/backups/{backup_id}", dependencies=[Depends(require_auth)])
+async def api_delete_backup(backup_id: str):
+    res = delete_backup(backup_id)
+    if not res.get("success"):
+        raise HTTPException(status_code=404, detail=res.get("error"))
+    return res
+
+@app.get("/api/backups/{backup_id}/download", dependencies=[Depends(require_auth)])
+async def api_download_backup(backup_id: str):
+    index = load_backup_index()
+    target = next((b for b in index if b["id"] == backup_id), None)
+    if not target:
+        raise HTTPException(status_code=404, detail="Backup not found")
+    filepath = Path(target["path"])
+    if not filepath.exists():
+        raise HTTPException(status_code=404, detail="File does not exist on disk")
+    return FileResponse(
+        path=str(filepath),
+        filename=target["filename"],
+        media_type="application/gzip"
+    )
+
+@app.post("/api/backups/upload", dependencies=[Depends(require_auth)])
+async def api_upload_backup(file: UploadFile = File(...)):
+    if not file.filename.endswith((".tar.gz", ".tgz")):
+        raise HTTPException(status_code=400, detail="Only .tar.gz archives are supported.")
+    
+    bdir = get_backup_dir()
+    safe_filename = Path(file.filename).name
+    dest_path = bdir / safe_filename
+    if dest_path.exists():
+        dest_path = bdir / f"{dest_path.stem}_{int(time.time())}{dest_path.suffix}"
+
+    with open(dest_path, "wb") as buffer:
+        shutil.copyfileobj(file.file, buffer)
+
+    size = dest_path.stat().st_size
+    sha256 = compute_sha256(dest_path)
+
+    index = load_backup_index()
+    backup_id = f"bk_upload_{int(time.time())}"
+    entry = {
+        "id": backup_id,
+        "filename": dest_path.name,
+        "path": str(dest_path),
+        "scope": "uploaded",
+        "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "size_bytes": size,
+        "formatted_size": f"{size / (1024*1024):.2f} MB",
+        "sha256": sha256,
+        "files_count": 0,
+        "description": f"Uploaded to Unraid: {file.filename}",
+        "auto_created": False,
+        "verified": True
+    }
+    index.insert(0, entry)
+    save_backup_index(index)
+    return {"success": True, "backup": entry}
+
+# ----------------- Settings Endpoints -----------------
+
+@app.get("/api/settings", dependencies=[Depends(require_auth)])
+async def api_get_settings():
+    s = load_settings().copy()
+    s.pop("admin_password_hash", None)
+    s.pop("admin_password_salt", None)
+    s.pop("jwt_secret", None)
+    return s
+
+class UpdateSettingsRequest(BaseModel):
+    settings: Dict[str, Any]
+
+@app.post("/api/settings", dependencies=[Depends(require_auth)])
+async def api_update_settings(req: UpdateSettingsRequest):
+    allowed_keys = [
+        "auth_enabled", "vps_host", "vps_port", "vps_user", "vps_auth_method",
+        "vps_ssh_key_path", "vps_ssh_password", "vps_pangolin_dir",
+        "backup_dir", "retention_count",
+        "scheduled_backup_enabled", "scheduled_backup_cron", "scheduled_backup_scope",
+        "scheduled_update_check_enabled", "webhook_url", "webhook_type",
+        "telegram_bot_token", "telegram_chat_id", "notify_on_backup", "notify_on_update"
+    ]
+    cur = load_settings()
+    for k, v in req.settings.items():
+        if k in allowed_keys:
+            cur[k] = v
+    save_settings(cur)
+    reload_scheduler()
+    return {"success": True, "settings": cur}
+
+@app.post("/api/notifications/test", dependencies=[Depends(require_auth)])
+async def api_test_notification():
+    return await test_webhook()
+
+# ----------------- Jobs & WebSocket -----------------
+
+@app.get("/api/jobs", dependencies=[Depends(require_auth)])
+async def api_list_jobs(limit: int = 25):
+    return runner.list_jobs(limit)
+
+@app.get("/api/jobs/{job_id}", dependencies=[Depends(require_auth)])
+async def api_get_job(job_id: str):
+    job = runner.get_job(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Job not found")
+    return job
+
+@app.websocket("/ws/jobs")
+async def websocket_jobs(websocket: WebSocket):
+    await websocket.accept()
+    await runner.register_global(websocket)
+    try:
+        while True:
+            await websocket.receive_text()
+    except WebSocketDisconnect:
+        runner.unregister_global(websocket)
+    except Exception:
+        runner.unregister_global(websocket)
+
+# ----------------- SPA Frontend Route -----------------
+
+@app.get("/{full_path:path}")
+async def serve_spa(full_path: str):
+    index_path = FRONTEND_DIR / "index.html"
+    if not index_path.exists():
+        return HTMLResponse("<h1>Pangolin Sentinel is initializing...</h1>")
+    return FileResponse(str(index_path))
+
+if __name__ == "__main__":
+    import uvicorn
+    port = int(os.environ.get("PORT", get_setting("app_port", 8888)))
+    host = os.environ.get("HOST", get_setting("app_host", "0.0.0.0"))
+    uvicorn.run("backend.main:app", host=host, port=port, reload=False)

backend/notifier.py

@@ -0,0 +1,95 @@
+import asyncio
+import time
+import httpx
+from typing import Dict, Any, Optional
+from backend.config import get_setting
+
+async def send_notification(
+    title: str,
+    message: str,
+    status: str = "info",  # "success", "error", "warning", "info"
+    details: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    webhook_url = get_setting("webhook_url", "")
+    webhook_type = get_setting("webhook_type", "discord")
+    bot_token = get_setting("telegram_bot_token", "")
+    chat_id = get_setting("telegram_chat_id", "")
+
+    if webhook_type == "telegram":
+        if not bot_token or not chat_id:
+            return {"success": False, "error": "Telegram Bot Token and Chat ID required"}
+    elif not webhook_url:
+        return {"success": False, "error": "Webhook URL not configured"}
+
+    color_map = {
+        "success": 0x10B981,  # Emerald green
+        "error": 0xEF4444,    # Red
+        "warning": 0xF59E0B,  # Amber
+        "info": 0x3B82F6      # Blue
+    }
+    color = color_map.get(status, 0x3B82F6)
+
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            if webhook_type == "discord":
+                embed = {
+                    "title": title,
+                    "description": message,
+                    "color": color,
+                    "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                    "footer": {"text": "Pangolin VPS Sentinel"}
+                }
+                if details:
+                    fields = []
+                    for k, v in details.items():
+                        fields.append({"name": str(k).capitalize(), "value": str(v), "inline": True})
+                    embed["fields"] = fields
+
+                payload = {
+                    "username": "Pangolin Sentinel",
+                    "embeds": [embed]
+                }
+                resp = await client.post(webhook_url, json=payload)
+                return {"success": resp.is_success, "status_code": resp.status_code, "text": resp.text}
+
+            elif webhook_type == "telegram":
+                tg_url = f"https://api.telegram.org/bot{bot_token}/sendMessage"
+                icon = "✅" if status == "success" else "❌" if status == "error" else "ℹ️"
+                text = f"<b>{icon} {title}</b>\n\n{message}\n"
+                if details:
+                    text += "\n"
+                    for k, v in details.items():
+                        text += f"• <b>{k.capitalize()}:</b> {v}\n"
+                
+                resp = await client.post(tg_url, json={
+                    "chat_id": chat_id,
+                    "text": text,
+                    "parse_mode": "HTML"
+                })
+                return {"success": resp.is_success, "status_code": resp.status_code, "text": resp.text}
+
+            else:  # Generic webhook
+                payload = {
+                    "event": "pangolin_event",
+                    "title": title,
+                    "message": message,
+                    "status": status,
+                    "details": details or {},
+                    "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+                }
+                resp = await client.post(webhook_url, json=payload)
+                return {"success": resp.is_success, "status_code": resp.status_code, "text": resp.text}
+
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+
+async def test_webhook() -> Dict[str, Any]:
+    return await send_notification(
+        title="🔔 Test Notification from Pangolin Sentinel",
+        message="Your webhook configuration is active and working properly on your Pangolin VPS.",
+        status="info",
+        details={
+            "vps": get_setting("app_host", "Pangolin-VPS"),
+            "time": time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime())
+        }
+    )

backend/pangolin.py

@@ -0,0 +1,239 @@
+import asyncio
+import json
+import os
+import time
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+import yaml
+from backend.runner import runner
+from backend.config import load_settings, get_setting
+from backend.remote import run_remote_quick, build_ssh_prefix
+
+async def get_pangolin_stack_status() -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    p_dir = settings.get("vps_pangolin_dir", "/opt/pangolin").strip()
+
+    if not vps_host:
+        return {
+            "vps_configured": False,
+            "installed": False,
+            "path": p_dir,
+            "compose_exists": False,
+            "config_exists": False,
+            "database": {"found": False, "path": None, "size_bytes": 0},
+            "containers": [],
+            "running_count": 0,
+            "total_count": 0,
+            "api_online": False,
+            "message": "Configure VPS Host and SSH credentials in Settings to manage Pangolin."
+        }
+
+    # Query Remote VPS for files, DB, and Docker status
+    check_script = f"""
+[ -f "{p_dir}/docker-compose.yml" ] || [ -f "{p_dir}/compose.yaml" ] && echo "COMPOSE_EXISTS" || echo "COMPOSE_NO"
+[ -f "{p_dir}/config/config.yml" ] || [ -f "{p_dir}/config.yml" ] && echo "CONFIG_EXISTS" || echo "CONFIG_NO"
+
+# Check DB
+db=$(find "{p_dir}" -name "*.db" -o -name "*.sqlite" 2>/dev/null | head -n 1)
+if [ -n "$db" ]; then
+    echo "DB_PATH:$db"
+    wc -c "$db" 2>/dev/null | awk '{{print $1}}'
+else
+    echo "DB_PATH:NONE"
+    echo "0"
+fi
+
+# Docker containers
+docker ps -a --format '{{json .}}' 2>/dev/null || true
+"""
+    ret, stdout, stderr = await run_remote_quick(check_script, settings)
+
+    if ret != 0:
+        return {
+            "vps_configured": True,
+            "connected": False,
+            "installed": False,
+            "path": p_dir,
+            "compose_exists": False,
+            "config_exists": False,
+            "database": {"found": False, "path": None, "size_bytes": 0},
+            "containers": [],
+            "running_count": 0,
+            "total_count": 0,
+            "api_online": False,
+            "error": stderr.strip() or "SSH connection to VPS failed."
+        }
+
+    lines = stdout.splitlines()
+    compose_exists = any("COMPOSE_EXISTS" in l for l in lines)
+    config_exists = any("CONFIG_EXISTS" in l for l in lines)
+
+    db_path = None
+    db_size = 0
+    for idx, l in enumerate(lines):
+        if l.startswith("DB_PATH:") and not l.endswith(":NONE"):
+            db_path = l.split(":", 1)[1].strip()
+            if idx + 1 < len(lines) and lines[idx + 1].strip().isdigit():
+                db_size = int(lines[idx + 1].strip())
+            break
+
+    # Parse Docker JSON lines
+    containers = []
+    for l in lines:
+        l = l.strip()
+        if not l or not l.startswith("{"):
+            continue
+        try:
+            data = json.loads(l)
+            names = data.get("Names", "")
+            image = data.get("Image", "")
+            if any(k in names.lower() or k in image.lower() for k in ["pangolin", "gerbil", "traefik", "newt"]):
+                containers.append({
+                    "id": data.get("ID", ""),
+                    "name": names,
+                    "image": image,
+                    "status": data.get("Status", ""),
+                    "state": data.get("State", ""),
+                    "created": data.get("CreatedAt", ""),
+                    "ports": data.get("Ports", "")
+                })
+        except Exception:
+            continue
+
+    running = sum(1 for c in containers if c.get("state") == "running")
+
+    return {
+        "vps_configured": True,
+        "connected": True,
+        "installed": compose_exists,
+        "path": p_dir,
+        "compose_exists": compose_exists,
+        "config_exists": config_exists,
+        "database": {
+            "found": db_path is not None,
+            "path": db_path,
+            "size_bytes": db_size
+        },
+        "containers": containers,
+        "running_count": running,
+        "total_count": len(containers),
+        "api_online": running > 0
+    }
+
+async def trigger_pangolin_update() -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    p_dir = settings.get("vps_pangolin_dir", "/opt/pangolin").strip()
+
+    if not vps_host:
+        return {"success": False, "error": "No VPS configured. Set VPS Host in Settings."}
+
+    ssh_prefix = build_ssh_prefix(settings)
+    
+    # Remote update script executed on the VPS
+    remote_script = f"""
+echo "=== STEP 1: Connecting to Pangolin VPS ({vps_host}) ==="
+cd "{p_dir}" || exit 1
+
+echo ""
+echo "=== STEP 2: Pulling latest images on VPS ==="
+docker compose pull
+
+echo ""
+echo "=== STEP 3: Applying stack updates & recreating containers ==="
+docker compose up -d --remove-orphans
+
+echo ""
+echo "=== STEP 4: Verifying running container status ==="
+docker compose ps
+
+echo ""
+echo "=== STEP 5: Cleaning up unused images on VPS ==="
+docker image prune -f
+
+echo ""
+echo "=== Remote Pangolin Stack update completed successfully! ==="
+"""
+    full_cmd = ssh_prefix + [remote_script]
+
+    res = await runner.run_command_job(
+        name=f"Update Remote Pangolin Stack ({vps_host})",
+        cmd=full_cmd,
+        job_type="update_pangolin",
+        description=f"Pull latest docker images and recreate Pangolin stack on remote VPS ({vps_host})"
+    )
+    return res
+
+async def container_action(container_name: str, action: str) -> Dict[str, Any]:
+    if action not in ["start", "stop", "restart"]:
+        return {"success": False, "error": "Invalid action"}
+
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    if not vps_host:
+        return {"success": False, "error": "No VPS configured."}
+
+    ret, stdout, stderr = await run_remote_quick(f"docker {action} {container_name}", settings)
+    if ret == 0:
+        return {"success": True, "output": stdout.strip()}
+    return {"success": False, "error": stderr.strip()}
+
+async def get_container_logs(container_name: str, lines: int = 150) -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    if not vps_host:
+        return {"success": False, "error": "No VPS configured."}
+
+    ret, stdout, stderr = await run_remote_quick(f"docker logs --tail {lines} {container_name} 2>&1", settings)
+    return {"success": True, "logs": stdout or stderr or "No logs available."}
+
+async def get_stack_config_files() -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    p_dir = settings.get("vps_pangolin_dir", "/opt/pangolin").strip()
+
+    if not vps_host:
+        return {}
+
+    candidates = ["docker-compose.yml", "compose.yaml", ".env", "config/config.yml", "config.yml"]
+    result = {}
+
+    for c in candidates:
+        ret, stdout, _ = await run_remote_quick(f'[ -f "{p_dir}/{c}" ] && cat "{p_dir}/{c}"', settings)
+        if ret == 0 and stdout:
+            result[c] = stdout
+    return result
+
+async def save_stack_config_file(filename: str, content: str) -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    p_dir = settings.get("vps_pangolin_dir", "/opt/pangolin").strip()
+
+    if not vps_host:
+        return {"success": False, "error": "No VPS configured."}
+    if ".." in filename or filename.startswith("/"):
+        return {"success": False, "error": "Invalid filename path."}
+
+    # Validate YAML locally before pushing to VPS
+    if filename.endswith((".yml", ".yaml")):
+        try:
+            yaml.safe_load(content)
+        except yaml.YAMLError as ye:
+            return {"success": False, "error": f"Invalid YAML syntax: {str(ye)}"}
+
+    # Push to remote VPS with backup of old file
+    safe_target = f"{p_dir}/{filename}"
+    remote_write_script = f"""
+mkdir -p "$(dirname '{safe_target}')"
+if [ -f '{safe_target}' ]; then
+    cp '{safe_target}' '{safe_target}.bak.{int(time.time())}'
+fi
+cat << 'PANGOLIN_EOF_SAVE' > '{safe_target}'
+{content}
+PANGOLIN_EOF_SAVE
+"""
+    ret, _, stderr = await run_remote_quick(remote_write_script, settings)
+    if ret == 0:
+        return {"success": True, "message": f"Saved {filename} on remote VPS with automatic snapshot backup."}
+    return {"success": False, "error": f"Failed to save on remote VPS: {stderr}"}

backend/remote.py

@@ -0,0 +1,177 @@
+import asyncio
+import os
+import shutil
+import time
+from pathlib import Path
+from typing import Dict, Any, List, Optional, Tuple
+from backend.config import load_settings, DEFAULT_KEY_FILE, DEFAULT_PUB_KEY_FILE
+
+def get_public_key() -> str:
+    if DEFAULT_PUB_KEY_FILE.exists():
+        return DEFAULT_PUB_KEY_FILE.read_text(encoding="utf-8").strip()
+    return ""
+
+def ensure_ssh_key() -> str:
+    if not DEFAULT_KEY_FILE.exists():
+        DEFAULT_KEY_FILE.parent.mkdir(parents=True, exist_ok=True)
+        os.system(f'ssh-keygen -t ed25519 -f "{DEFAULT_KEY_FILE}" -N "" -C "pangolin-sentinel@unraid" -q')
+    if DEFAULT_KEY_FILE.exists():
+        try:
+            os.chmod(DEFAULT_KEY_FILE, 0o600)
+        except Exception:
+            pass
+    return get_public_key()
+
+def resolve_key_path(configured_path: Optional[str] = None) -> str:
+    if configured_path and os.path.exists(configured_path):
+        return configured_path
+    if DEFAULT_KEY_FILE.exists():
+        return str(DEFAULT_KEY_FILE)
+    return configured_path or str(DEFAULT_KEY_FILE)
+
+def build_ssh_prefix(settings: Optional[Dict[str, Any]] = None) -> List[str]:
+    if not settings:
+        settings = load_settings()
+
+    host = settings.get("vps_host", "").strip()
+    port = str(settings.get("vps_port", 22))
+    user = settings.get("vps_user", "root").strip()
+    auth_method = settings.get("vps_auth_method", "key")
+    raw_key_path = settings.get("vps_ssh_key_path", str(DEFAULT_KEY_FILE))
+    key_path = resolve_key_path(raw_key_path)
+    password = settings.get("vps_ssh_password", "")
+
+    ssh_opts = [
+        "-o", "StrictHostKeyChecking=accept-new",
+        "-o", "ConnectTimeout=8",
+        "-o", "ServerAliveInterval=15",
+        "-p", port
+    ]
+
+    if auth_method == "password" and password:
+        return ["sshpass", "-p", password, "ssh"] + ssh_opts + [f"{user}@{host}"]
+    else:
+        # Use Key Auth
+        if os.path.exists(key_path):
+            try:
+                os.chmod(key_path, 0o600)
+            except Exception:
+                pass
+            return ["ssh", "-i", key_path, "-o", "BatchMode=yes"] + ssh_opts + [f"{user}@{host}"]
+        return ["ssh", "-o", "BatchMode=yes"] + ssh_opts + [f"{user}@{host}"]
+
+def build_scp_prefix(settings: Optional[Dict[str, Any]] = None) -> List[str]:
+    if not settings:
+        settings = load_settings()
+
+    port = str(settings.get("vps_port", 22))
+    auth_method = settings.get("vps_auth_method", "key")
+    raw_key_path = settings.get("vps_ssh_key_path", str(DEFAULT_KEY_FILE))
+    key_path = resolve_key_path(raw_key_path)
+    password = settings.get("vps_ssh_password", "")
+
+    scp_opts = [
+        "-P", port,
+        "-o", "StrictHostKeyChecking=accept-new",
+        "-o", "ConnectTimeout=10"
+    ]
+
+    if auth_method == "password" and password:
+        return ["sshpass", "-p", password, "scp"] + scp_opts
+    else:
+        if os.path.exists(key_path):
+            try:
+                os.chmod(key_path, 0o600)
+            except Exception:
+                pass
+            return ["scp", "-i", key_path, "-o", "BatchMode=yes"] + scp_opts
+        return ["scp", "-o", "BatchMode=yes"] + scp_opts
+
+async def run_remote_quick(cmd: str, settings: Optional[Dict[str, Any]] = None) -> Tuple[int, str, str]:
+    prefix = build_ssh_prefix(settings)
+    full_cmd = prefix + [cmd]
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *full_cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        stdout, stderr = await proc.communicate()
+        return proc.returncode, stdout.decode('utf-8', errors='replace'), stderr.decode('utf-8', errors='replace')
+    except Exception as e:
+        return -1, "", str(e)
+
+async def test_vps_connection(custom_settings: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    s = custom_settings or load_settings()
+    host = s.get("vps_host", "").strip()
+    if not host:
+        return {
+            "success": False,
+            "connected": False,
+            "error": "No VPS Host / IP configured. Please specify the VPS address in Settings."
+        }
+
+    # Run comprehensive diagnostic script on remote VPS
+    diag_script = """
+echo "===PING_OK==="
+uname -n
+cat /etc/os-release | grep PRETTY_NAME | cut -d= -f2 | tr -d '"'
+uname -r
+uptime -p 2>/dev/null || uptime
+which docker >/dev/null 2>&1 && echo "DOCKER_FOUND" || echo "DOCKER_NOT_FOUND"
+docker ps >/dev/null 2>&1 && echo "DOCKER_RUNNING" || echo "DOCKER_STOPPED"
+"""
+    ret, stdout, stderr = await run_remote_quick(diag_script, s)
+
+    if ret != 0 or "===PING_OK===" not in stdout:
+        err = stderr.strip() or "Connection failed. Please check Host, Port, and SSH credentials."
+        if "Permission denied" in err:
+            err = "Permission denied (SSH authentication failed). Please ensure your SSH public key or password is correct on the VPS."
+        elif "Connection timed out" in err or "No route to host" in err:
+            err = f"Could not reach {host}:{s.get('vps_port', 22)}. Verify firewall, IP address, and SSH port."
+        return {
+            "success": False,
+            "connected": False,
+            "error": err
+        }
+
+    lines = [line.strip() for line in stdout.splitlines() if line.strip()]
+    hostname = lines[1] if len(lines) > 1 else host
+    os_name = lines[2] if len(lines) > 2 else "Linux"
+    kernel = lines[3] if len(lines) > 3 else ""
+    uptime_str = lines[4] if len(lines) > 4 else ""
+    docker_found = "DOCKER_FOUND" in stdout
+    docker_running = "DOCKER_RUNNING" in stdout
+
+    # Check Pangolin path on VPS
+    p_dir = s.get("vps_pangolin_dir", "/opt/pangolin")
+    p_script = f"""
+[ -d "{p_dir}" ] && echo "DIR_OK" || echo "DIR_MISSING"
+[ -f "{p_dir}/docker-compose.yml" ] || [ -f "{p_dir}/compose.yaml" ] && echo "COMPOSE_OK" || echo "COMPOSE_MISSING"
+find "{p_dir}" -name "*.db" -o -name "*.sqlite" 2>/dev/null | head -n 1
+"""
+    _, p_out, _ = await run_remote_quick(p_script, s)
+    p_lines = [l.strip() for l in p_out.splitlines() if l.strip()]
+
+    dir_ok = "DIR_OK" in p_lines
+    compose_ok = "COMPOSE_OK" in p_lines
+    db_file = None
+    for l in p_lines:
+        if l.endswith((".db", ".sqlite")):
+            db_file = l
+            break
+
+    return {
+        "success": True,
+        "connected": True,
+        "hostname": hostname,
+        "os": os_name,
+        "kernel": kernel,
+        "uptime": uptime_str,
+        "docker_installed": docker_found,
+        "docker_running": docker_running,
+        "pangolin_dir_exists": dir_ok,
+        "compose_file_exists": compose_ok,
+        "database_file": db_file,
+        "message": f"Successfully connected to {hostname} ({host}) via SSH!"
+    }

backend/runner.py

@@ -0,0 +1,215 @@
+import asyncio
+import os
+import json
+import time
+import secrets
+from typing import Dict, List, Optional, Any, Set
+from fastapi import WebSocket
+from backend.config import JOBS_FILE
+
+class JobManager:
+    def __init__(self):
+        self.active_jobs: Dict[str, Dict[str, Any]] = {}
+        self.subscribers: Dict[str, Set[WebSocket]] = {}
+        self.global_subscribers: Set[WebSocket] = set()
+        self.lock = asyncio.Lock()
+        self.type_locks: Dict[str, asyncio.Lock] = {
+            "backup": asyncio.Lock(),
+            "restore": asyncio.Lock(),
+            "update_pangolin": asyncio.Lock(),
+            "update_system": asyncio.Lock(),
+        }
+
+    def _load_history(self) -> List[Dict[str, Any]]:
+        if not JOBS_FILE.exists():
+            return []
+        try:
+            with open(JOBS_FILE, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except Exception:
+            return []
+
+    def _save_history(self, history: List[Dict[str, Any]]) -> None:
+        try:
+            with open(JOBS_FILE, "w", encoding="utf-8") as f:
+                json.dump(history[:50], f, indent=2)
+        except Exception as e:
+            print(f"Error saving jobs history: {e}")
+
+    def list_jobs(self, limit: int = 25) -> List[Dict[str, Any]]:
+        history = self._load_history()
+        # Merge active jobs at the beginning
+        active_list = list(self.active_jobs.values())
+        seen_ids = {j["id"] for j in active_list}
+        merged = active_list + [j for j in history if j["id"] not in seen_ids]
+        merged.sort(key=lambda x: x.get("start_time", 0), reverse=True)
+        return merged[:limit]
+
+    def get_job(self, job_id: str) -> Optional[Dict[str, Any]]:
+        if job_id in self.active_jobs:
+            return self.active_jobs[job_id]
+        history = self._load_history()
+        for j in history:
+            if j["id"] == job_id:
+                return j
+        return None
+
+    async def register_subscriber(self, job_id: str, ws: WebSocket):
+        if job_id not in self.subscribers:
+            self.subscribers[job_id] = set()
+        self.subscribers[job_id].add(ws)
+
+    def unregister_subscriber(self, job_id: str, ws: WebSocket):
+        if job_id in self.subscribers:
+            self.subscribers[job_id].discard(ws)
+            if not self.subscribers[job_id]:
+                del self.subscribers[job_id]
+
+    async def register_global(self, ws: WebSocket):
+        self.global_subscribers.add(ws)
+
+    def unregister_global(self, ws: WebSocket):
+        self.global_subscribers.discard(ws)
+
+    async def broadcast(self, job_id: str, message: Dict[str, Any]):
+        # Broadcast to specific job subscribers
+        if job_id in self.subscribers:
+            dead = set()
+            for ws in list(self.subscribers[job_id]):
+                try:
+                    await ws.send_json(message)
+                except Exception:
+                    dead.add(ws)
+            for ws in dead:
+                self.subscribers[job_id].discard(ws)
+
+        # Broadcast to global subscribers
+        dead_global = set()
+        for ws in list(self.global_subscribers):
+            try:
+                await ws.send_json(message)
+            except Exception:
+                dead_global.add(ws)
+        for ws in dead_global:
+            self.global_subscribers.discard(ws)
+
+    async def run_command_job(
+        self,
+        name: str,
+        cmd: List[str] | str,
+        job_type: str = "general",
+        env: Optional[Dict[str, str]] = None,
+        cwd: Optional[str] = None,
+        description: str = ""
+    ) -> Dict[str, Any]:
+        lock = self.type_locks.get(job_type)
+        if lock and lock.locked():
+            return {
+                "success": False,
+                "error": f"A {job_type} task is already running in the background. Please wait for it to complete."
+            }
+
+        job_id = f"job_{int(time.time())}_{secrets.token_hex(4)}"
+        job = {
+            "id": job_id,
+            "name": name,
+            "type": job_type,
+            "description": description,
+            "cmd": cmd if isinstance(cmd, str) else " ".join(cmd),
+            "status": "running",
+            "start_time": time.time(),
+            "end_time": None,
+            "duration": None,
+            "exit_code": None,
+            "logs": []
+        }
+
+        self.active_jobs[job_id] = job
+        await self.broadcast(job_id, {"type": "job_started", "job": job})
+
+        # Run in background asyncio task
+        asyncio.create_task(self._execute_process(job_id, cmd, env, cwd, lock))
+        return {"success": True, "job_id": job_id, "job": job}
+
+    async def _execute_process(
+        self,
+        job_id: str,
+        cmd: List[str] | str,
+        env: Optional[Dict[str, str]],
+        cwd: Optional[str],
+        lock: Optional[asyncio.Lock]
+    ):
+        if lock:
+            await lock.acquire()
+
+        job = self.active_jobs[job_id]
+        run_env = os.environ.copy()
+        run_env["DEBIAN_FRONTEND"] = "noninteractive"
+        run_env["TERM"] = "xterm-256color"
+        run_env["COLUMNS"] = "120"
+        if env:
+            run_env.update(env)
+
+        try:
+            if isinstance(cmd, list):
+                proc = await asyncio.create_subprocess_exec(
+                    *cmd,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.STDOUT,
+                    env=run_env,
+                    cwd=cwd
+                )
+            else:
+                proc = await asyncio.create_subprocess_shell(
+                    cmd,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.STDOUT,
+                    env=run_env,
+                    cwd=cwd,
+                    executable="/bin/bash" if os.path.exists("/bin/bash") else None
+                )
+
+            while True:
+                line = await proc.stdout.readline()
+                if not line:
+                    break
+                text = line.decode('utf-8', errors='replace').rstrip('\r\n')
+                timestamp = time.strftime("%H:%M:%S")
+                log_entry = f"[{timestamp}] {text}"
+                job["logs"].append(log_entry)
+                # Keep up to 2000 lines
+                if len(job["logs"]) > 2000:
+                    job["logs"] = job["logs"][-2000:]
+                await self.broadcast(job_id, {"type": "log", "job_id": job_id, "data": log_entry})
+
+            await proc.wait()
+            exit_code = proc.returncode
+            job["exit_code"] = exit_code
+            job["status"] = "completed" if exit_code == 0 else "failed"
+
+        except Exception as e:
+            err_msg = f"Execution error: {str(e)}"
+            job["logs"].append(f"[{time.strftime('%H:%M:%S')}] ERROR: {err_msg}")
+            job["status"] = "failed"
+            job["exit_code"] = -1
+            await self.broadcast(job_id, {"type": "log", "job_id": job_id, "data": err_msg})
+
+        finally:
+            job["end_time"] = time.time()
+            job["duration"] = round(job["end_time"] - job["start_time"], 1)
+            await self.broadcast(job_id, {"type": "job_finished", "job": job})
+
+            # Save to history file
+            history = self._load_history()
+            history.insert(0, job.copy())
+            self._save_history(history)
+
+            # Clean from active_jobs after a short delay so clients can finish reading
+            await asyncio.sleep(5)
+            if job_id in self.active_jobs:
+                del self.active_jobs[job_id]
+
+            if lock:
+                lock.release()
+
+runner = JobManager()

backend/scheduler.py

@@ -0,0 +1,108 @@
+import asyncio
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.triggers.cron import CronTrigger
+from apscheduler.triggers.interval import IntervalTrigger
+from backend.config import get_setting, load_settings
+from backend.backup import run_backup_task
+from backend.system import check_upgradable_packages
+from backend.notifier import send_notification
+
+scheduler = AsyncIOScheduler()
+
+async def scheduled_backup_job():
+    print("[SCHEDULER] Triggering scheduled automated backup...")
+    settings = load_settings()
+    scope = settings.get("scheduled_backup_scope", "pangolin")
+    
+    try:
+        res = await run_backup_task(
+            scope=scope,
+            description="Automated scheduled backup",
+            auto_created=True
+        )
+        if settings.get("notify_on_backup"):
+            await send_notification(
+                title=f"Automated Backup Started ({scope.upper()})",
+                message=f"A scheduled backup job has been initiated.",
+                status="info"
+            )
+    except Exception as e:
+        print(f"[SCHEDULER] Error during scheduled backup: {e}")
+        if settings.get("notify_on_backup"):
+            await send_notification(
+                title="Automated Backup Failed",
+                message=f"Scheduled backup failed to start: {str(e)}",
+                status="error"
+            )
+
+async def scheduled_update_check_job():
+    print("[SCHEDULER] Checking for available updates...")
+    settings = load_settings()
+    if not settings.get("scheduled_update_check_enabled", True):
+        return
+
+    try:
+        pkgs = await check_upgradable_packages()
+        total = pkgs.get("total", 0)
+        sec = pkgs.get("security_count", 0)
+
+        if total > 0 and settings.get("notify_on_update"):
+            await send_notification(
+                title=f"📦 VPS Updates Available ({total} packages)",
+                message=f"There are {total} upgradable package(s) available, including {sec} security update(s).",
+                status="warning" if sec > 0 else "info",
+                details={
+                    "total_upgrades": total,
+                    "security_updates": sec
+                }
+            )
+    except Exception as e:
+        print(f"[SCHEDULER] Error during update check: {e}")
+
+def reload_scheduler():
+    settings = load_settings()
+    
+    # Remove existing jobs if any
+    for job_id in ["scheduled_backup", "scheduled_update_check"]:
+        if scheduler.get_job(job_id):
+            scheduler.remove_job(job_id)
+
+    # 1. Scheduled Backup
+    if settings.get("scheduled_backup_enabled", True):
+        cron_expr = settings.get("scheduled_backup_cron", "0 3 * * *")
+        parts = cron_expr.strip().split()
+        if len(parts) == 5:
+            trigger = CronTrigger(
+                minute=parts[0],
+                hour=parts[1],
+                day=parts[2],
+                month=parts[3],
+                day_of_week=parts[4]
+            )
+            scheduler.add_job(
+                scheduled_backup_job,
+                trigger=trigger,
+                id="scheduled_backup",
+                replace_existing=True
+            )
+            print(f"[SCHEDULER] Scheduled backup registered with cron: {cron_expr}")
+
+    # 2. Scheduled Update Check (every 12 hours)
+    if settings.get("scheduled_update_check_enabled", True):
+        scheduler.add_job(
+            scheduled_update_check_job,
+            trigger=IntervalTrigger(hours=12),
+            id="scheduled_update_check",
+            replace_existing=True
+        )
+        print("[SCHEDULER] Scheduled update checks registered (every 12 hours).")
+
+def start_scheduler():
+    if not scheduler.running:
+        reload_scheduler()
+        scheduler.start()
+        print("[SCHEDULER] Scheduler started.")
+
+def shutdown_scheduler():
+    if scheduler.running:
+        scheduler.shutdown()

backend/system.py

@@ -0,0 +1,273 @@
+import asyncio
+import json
+import os
+import platform
+import re
+import shutil
+import time
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+import psutil
+from backend.runner import runner
+from backend.config import get_setting, load_settings, get_default_backup_dir
+from backend.remote import run_remote_quick, build_ssh_prefix
+
+def format_bytes(size: int | float) -> str:
+    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
+        if size < 1024.0:
+            return f"{size:.2f} {unit}"
+        size /= 1024.0
+    return f"{size:.2f} PB"
+
+def format_uptime(seconds: float) -> str:
+    days = int(seconds // (24 * 3600))
+    seconds %= (24 * 3600)
+    hours = int(seconds // 3600)
+    seconds %= 3600
+    minutes = int(seconds // 60)
+    parts = []
+    if days > 0:
+        parts.append(f"{days}d")
+    if hours > 0 or days > 0:
+        parts.append(f"{hours}h")
+    parts.append(f"{minutes}m")
+    return " ".join(parts)
+
+async def get_system_stats() -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+
+    # Local Storage Info (Unraid Server or Docker Backup Volume)
+    configured_dir = settings.get("backup_dir", "").strip() or get_default_backup_dir()
+    backup_path = configured_dir
+    unraid_disk = None
+    try:
+        p = Path(backup_path)
+        if not p.exists():
+            p.mkdir(parents=True, exist_ok=True)
+        bd = psutil.disk_usage(str(p))
+        unraid_disk = {
+            "path": str(p),
+            "total": bd.total,
+            "used": bd.used,
+            "free": bd.free,
+            "percent": bd.percent,
+            "formatted_total": format_bytes(bd.total),
+            "formatted_used": format_bytes(bd.used),
+            "formatted_free": format_bytes(bd.free)
+        }
+    except Exception:
+        pass
+
+    # If no VPS configured yet, return local info and indicator
+    if not vps_host:
+        local_cpu = psutil.cpu_percent(interval=0.05)
+        local_mem = psutil.virtual_memory()
+        local_uptime = time.time() - psutil.boot_time()
+        return {
+            "vps_configured": False,
+            "hostname": "No VPS Configured (Local Host)",
+            "os": "Unraid / Docker Host",
+            "kernel": platform.release(),
+            "uptime_formatted": format_uptime(local_uptime),
+            "cpu": {"percent": local_cpu, "cores": psutil.cpu_count(logical=True), "load_avg": [0, 0, 0]},
+            "memory": {
+                "percent": local_mem.percent,
+                "formatted_total": format_bytes(local_mem.total),
+                "formatted_used": format_bytes(local_mem.used),
+                "formatted_available": format_bytes(local_mem.available)
+            },
+            "disk": unraid_disk or {"percent": 0, "formatted_used": "0 B", "formatted_total": "0 B", "formatted_free": "0 B"},
+            "unraid_disk": unraid_disk,
+            "reboot_required": False,
+            "reboot_packages": []
+        }
+
+    # Gather metrics from Remote VPS via SSH
+    remote_script = """
+uname -n
+cat /etc/os-release | grep PRETTY_NAME | cut -d= -f2 | tr -d '"'
+uname -r
+cat /proc/uptime | awk '{print $1}'
+grep -c ^processor /proc/cpuinfo
+cat /proc/loadavg | awk '{print $1,$2,$3}'
+free -b | grep Mem: | awk '{print $2,$3,$7}'
+df -B1 / | tail -n 1 | awk '{print $2,$3,$4}'
+[ -f /var/run/reboot-required ] && echo "REBOOT_YES" || echo "REBOOT_NO"
+[ -f /var/run/reboot-required.pkgs ] && cat /var/run/reboot-required.pkgs || true
+"""
+    ret, stdout, stderr = await run_remote_quick(remote_script, settings)
+
+    if ret != 0:
+        return {
+            "vps_configured": True,
+            "vps_connected": False,
+            "hostname": vps_host,
+            "os": "Remote VPS (Unreachable)",
+            "kernel": "SSH Error",
+            "uptime_formatted": "Offline",
+            "error": stderr.strip() or "Failed to connect over SSH",
+            "cpu": {"percent": 0, "cores": 1, "load_avg": [0, 0, 0]},
+            "memory": {"percent": 0, "formatted_total": "0 B", "formatted_used": "0 B", "formatted_available": "0 B"},
+            "disk": {"percent": 0, "formatted_total": "0 B", "formatted_used": "0 B", "formatted_free": "0 B"},
+            "unraid_disk": unraid_disk,
+            "reboot_required": False,
+            "reboot_packages": []
+        }
+
+    lines = [l.strip() for l in stdout.splitlines() if l.strip()]
+    try:
+        r_host = lines[0]
+        r_os = lines[1]
+        r_kernel = lines[2]
+        r_uptime_sec = float(lines[3])
+        r_cores = int(lines[4])
+        r_load = [float(x) for x in lines[5].split()[:3]]
+
+        mem_parts = [int(x) for x in lines[6].split()]
+        mem_total, mem_used, mem_avail = mem_parts[0], mem_parts[1], mem_parts[2]
+        mem_pct = round((mem_used / mem_total) * 100, 1) if mem_total else 0
+
+        disk_parts = [int(x) for x in lines[7].split()]
+        disk_total, disk_used, disk_avail = disk_parts[0], disk_parts[1], disk_parts[2]
+        disk_pct = round((disk_used / disk_total) * 100, 1) if disk_total else 0
+
+        reboot_req = "REBOOT_YES" in stdout
+        reboot_pkgs = []
+        capture_pkgs = False
+        for l in lines:
+            if capture_pkgs and not l.startswith("REBOOT_"):
+                reboot_pkgs.append(l)
+            if l == "REBOOT_YES":
+                capture_pkgs = True
+
+        return {
+            "vps_configured": True,
+            "vps_connected": True,
+            "hostname": r_host,
+            "os": r_os,
+            "kernel": r_kernel,
+            "uptime_formatted": format_uptime(r_uptime_sec),
+            "cpu": {
+                "percent": min(100, round((r_load[0] / max(1, r_cores)) * 100, 1)),
+                "cores": r_cores,
+                "load_avg": r_load
+            },
+            "memory": {
+                "total": mem_total,
+                "used": mem_used,
+                "available": mem_avail,
+                "percent": mem_pct,
+                "formatted_total": format_bytes(mem_total),
+                "formatted_used": format_bytes(mem_used),
+                "formatted_available": format_bytes(mem_avail)
+            },
+            "disk": {
+                "total": disk_total,
+                "used": disk_used,
+                "free": disk_avail,
+                "percent": disk_pct,
+                "formatted_total": format_bytes(disk_total),
+                "formatted_used": format_bytes(disk_used),
+                "formatted_free": format_bytes(disk_avail)
+            },
+            "unraid_disk": unraid_disk,
+            "reboot_required": reboot_req,
+            "reboot_packages": reboot_pkgs
+        }
+    except Exception as e:
+        return {
+            "vps_configured": True,
+            "vps_connected": True,
+            "hostname": vps_host,
+            "os": "Linux VPS",
+            "kernel": "Connected",
+            "uptime_formatted": "Connected",
+            "cpu": {"percent": 0, "cores": 1, "load_avg": [0, 0, 0]},
+            "memory": {"percent": 0, "formatted_total": "0 B", "formatted_used": "0 B", "formatted_available": "0 B"},
+            "disk": {"percent": 0, "formatted_total": "0 B", "formatted_used": "0 B", "formatted_free": "0 B"},
+            "unraid_disk": unraid_disk,
+            "reboot_required": False,
+            "reboot_packages": []
+        }
+
+async def check_upgradable_packages() -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    if not vps_host:
+        return {"total": 0, "security_count": 0, "packages": [], "message": "No VPS configured."}
+
+    # Run apt update silently on the VPS
+    await run_remote_quick("apt-get update -qq", settings)
+
+    # Run apt list --upgradable on the VPS
+    ret, stdout, _ = await run_remote_quick("apt list --upgradable 2>/dev/null", settings)
+    packages: List[Dict[str, str]] = []
+    security_count = 0
+
+    if ret == 0:
+        lines = stdout.splitlines()
+        pattern = re.compile(r"^([^/]+)/([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+\[upgradable from:\s+([^\]]+)\]")
+        for line in lines:
+            line = line.strip()
+            if not line or line.startswith("Listing..."):
+                continue
+            match = pattern.match(line)
+            if match:
+                pkg_name, repo, new_ver, arch, cur_ver = match.groups()
+                is_sec = "security" in repo.lower()
+                if is_sec:
+                    security_count += 1
+                packages.append({
+                    "name": pkg_name,
+                    "repository": repo,
+                    "current_version": cur_ver,
+                    "new_version": new_ver,
+                    "arch": arch,
+                    "is_security": is_sec
+                })
+
+    return {
+        "total": len(packages),
+        "security_count": security_count,
+        "packages": packages
+    }
+
+async def trigger_system_upgrade(dist_upgrade: bool = False) -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    if not vps_host:
+        return {"success": False, "error": "No VPS configured. Set VPS Host in Settings."}
+
+    cmd_type = "dist-upgrade" if dist_upgrade else "upgrade"
+    ssh_prefix = build_ssh_prefix(settings)
+    
+    # Remote upgrade command with proper noninteractive env
+    remote_cmd = f"DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold {cmd_type} -y"
+    full_cmd = ssh_prefix + [remote_cmd]
+
+    res = await runner.run_command_job(
+        name=f"Remote VPS Packages {cmd_type.capitalize()}",
+        cmd=full_cmd,
+        job_type="update_system",
+        description=f"Automated package upgrade on remote VPS ({vps_host}) via apt-get {cmd_type} -y"
+    )
+    return res
+
+async def trigger_system_reboot() -> Dict[str, Any]:
+    settings = load_settings()
+    vps_host = settings.get("vps_host", "").strip()
+    if not vps_host:
+        return {"success": False, "error": "No VPS configured."}
+
+    ssh_prefix = build_ssh_prefix(settings)
+    remote_cmd = "shutdown -r +1 'Reboot initiated from Unraid Pangolin Sentinel'"
+    full_cmd = ssh_prefix + [remote_cmd]
+
+    res = await runner.run_command_job(
+        name=f"Remote VPS Reboot ({vps_host})",
+        cmd=full_cmd,
+        job_type="general",
+        description="Initiate safe remote VPS reboot scheduled in 1 minute"
+    )
+    return res