404 lines
16 KiB
Python
404 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Migrate Nginx Proxy Manager proxy hosts from SQLite into RustProxy.
|
|
|
|
The migrator intentionally focuses on proxy-host identity and upstream routing. It
|
|
never imports certificate/key material from NPM. NPM hosts that used TLS are attached
|
|
to RustProxy's TLS listener and use the RustProxy default certificate unless a named
|
|
certificate is supplied explicitly.
|
|
|
|
Run without --apply first. The script logs in to the RustProxy administration API,
|
|
reads the current revision, builds a merged configuration and prints a review report.
|
|
Only --apply performs the revisioned PUT /api/config.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from dataclasses import dataclass, asdict
|
|
import getpass
|
|
import http.cookiejar
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import sqlite3
|
|
import ssl
|
|
import sys
|
|
from typing import Any
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
MAX_CONFIG_BYTES = 262_144
|
|
|
|
|
|
class MigrationError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class NpmProxyHost:
|
|
id: int
|
|
domain_names: list[str]
|
|
forward_scheme: str
|
|
forward_host: str
|
|
forward_port: int
|
|
allow_websocket_upgrade: bool
|
|
http2_support: bool
|
|
enabled: bool
|
|
certificate_id: int
|
|
ssl_forced: bool
|
|
access_list_id: int
|
|
advanced_config: str
|
|
locations: list[Any]
|
|
caching_enabled: bool
|
|
block_exploits: bool
|
|
meta: dict[str, Any]
|
|
|
|
|
|
@dataclass
|
|
class Decision:
|
|
npm_id: int
|
|
action: str
|
|
hosts: list[str]
|
|
backend: str | None = None
|
|
rustproxy_ids: list[str] | None = None
|
|
warnings: list[str] | None = None
|
|
reason: str | None = None
|
|
|
|
|
|
def json_value(raw: Any, fallback: Any) -> Any:
|
|
if raw is None or raw == "":
|
|
return fallback
|
|
if isinstance(raw, (list, dict)):
|
|
return raw
|
|
try:
|
|
return json.loads(raw)
|
|
except (TypeError, json.JSONDecodeError):
|
|
return fallback
|
|
|
|
|
|
def bool_value(raw: Any) -> bool:
|
|
if isinstance(raw, str):
|
|
return raw.strip().lower() in {"1", "true", "yes", "on"}
|
|
return bool(raw)
|
|
|
|
|
|
def normalize_host_pattern(value: str) -> str:
|
|
value = value.strip().lower().rstrip(".")
|
|
wildcard = value.startswith("*.")
|
|
base = value[2:] if wildcard else value
|
|
if not base or len(base) > 253 or "*" in base or ":" in base or "/" in base:
|
|
raise MigrationError(f"invalid hostname {value!r}")
|
|
labels = base.split(".")
|
|
if wildcard and len(labels) < 2:
|
|
raise MigrationError(f"invalid wildcard hostname {value!r}")
|
|
for label in labels:
|
|
if not label or len(label) > 63 or label.startswith("-") or label.endswith("-"):
|
|
raise MigrationError(f"invalid hostname {value!r}")
|
|
if not re.fullmatch(r"[a-z0-9-]+", label):
|
|
raise MigrationError(f"hostname must be ASCII/punycode: {value!r}")
|
|
return ("*." if wildcard else "") + base
|
|
|
|
|
|
def sql_columns(connection: sqlite3.Connection, table: str) -> set[str]:
|
|
return {str(row[1]) for row in connection.execute(f"PRAGMA table_info({table})")}
|
|
|
|
|
|
def load_npm_hosts(database: Path) -> list[NpmProxyHost]:
|
|
if database.is_symlink() or not database.is_file():
|
|
raise MigrationError(f"NPM SQLite database not found or unsafe: {database}")
|
|
uri = f"file:{urllib.parse.quote(str(database.resolve()))}?mode=ro"
|
|
connection = sqlite3.connect(uri, uri=True)
|
|
connection.row_factory = sqlite3.Row
|
|
try:
|
|
tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
if "proxy_host" not in tables:
|
|
raise MigrationError("NPM database does not contain the proxy_host table")
|
|
columns = sql_columns(connection, "proxy_host")
|
|
required = {"id", "domain_names", "forward_scheme", "forward_host", "forward_port"}
|
|
missing = sorted(required - columns)
|
|
if missing:
|
|
raise MigrationError("NPM proxy_host table is missing required columns: " + ", ".join(missing))
|
|
wanted = [
|
|
"id", "domain_names", "forward_scheme", "forward_host", "forward_port",
|
|
"allow_websocket_upgrade", "http2_support", "enabled", "certificate_id",
|
|
"ssl_forced", "access_list_id", "advanced_config", "locations",
|
|
"caching_enabled", "block_exploits", "meta", "is_deleted",
|
|
]
|
|
select = ", ".join(name for name in wanted if name in columns)
|
|
where = " WHERE is_deleted = 0" if "is_deleted" in columns else ""
|
|
rows = connection.execute(f"SELECT {select} FROM proxy_host{where} ORDER BY id").fetchall()
|
|
finally:
|
|
connection.close()
|
|
|
|
result: list[NpmProxyHost] = []
|
|
for row in rows:
|
|
data = dict(row)
|
|
domains_raw = json_value(data.get("domain_names"), [])
|
|
if not isinstance(domains_raw, list):
|
|
domains_raw = []
|
|
domains: list[str] = []
|
|
for item in domains_raw:
|
|
if isinstance(item, str) and item.strip():
|
|
domains.append(item.strip())
|
|
locations = json_value(data.get("locations"), [])
|
|
if not isinstance(locations, list):
|
|
locations = []
|
|
meta = json_value(data.get("meta"), {})
|
|
if not isinstance(meta, dict):
|
|
meta = {}
|
|
try:
|
|
port = int(data.get("forward_port"))
|
|
except (TypeError, ValueError):
|
|
port = 0
|
|
result.append(NpmProxyHost(
|
|
id=int(data["id"]),
|
|
domain_names=domains,
|
|
forward_scheme=str(data.get("forward_scheme") or "").strip().lower(),
|
|
forward_host=str(data.get("forward_host") or "").strip(),
|
|
forward_port=port,
|
|
allow_websocket_upgrade=bool_value(data.get("allow_websocket_upgrade", False)),
|
|
http2_support=bool_value(data.get("http2_support", True)),
|
|
enabled=bool_value(data.get("enabled", True)),
|
|
certificate_id=int(data.get("certificate_id") or 0),
|
|
ssl_forced=bool_value(data.get("ssl_forced", False)),
|
|
access_list_id=int(data.get("access_list_id") or 0),
|
|
advanced_config=str(data.get("advanced_config") or ""),
|
|
locations=locations,
|
|
caching_enabled=bool_value(data.get("caching_enabled", False)),
|
|
block_exploits=bool_value(data.get("block_exploits", False)),
|
|
meta=meta,
|
|
))
|
|
return result
|
|
|
|
|
|
def backend_url(item: NpmProxyHost) -> str:
|
|
if item.forward_scheme not in {"http", "https"}:
|
|
raise MigrationError(f"unsupported upstream scheme {item.forward_scheme!r}")
|
|
if not 1 <= item.forward_port <= 65535:
|
|
raise MigrationError(f"invalid upstream port {item.forward_port}")
|
|
host = item.forward_host.strip()
|
|
if not host or any(ch.isspace() for ch in host) or "/" in host or "@" in host:
|
|
raise MigrationError(f"invalid upstream host {host!r}")
|
|
if host.startswith("[") and host.endswith("]"):
|
|
rendered = host
|
|
else:
|
|
try:
|
|
ip = ipaddress.ip_address(host)
|
|
rendered = f"[{ip}]" if ip.version == 6 else str(ip)
|
|
except ValueError:
|
|
if ":" in host:
|
|
raise MigrationError(f"invalid upstream host {host!r}")
|
|
rendered = host
|
|
return f"{item.forward_scheme}://{rendered}:{item.forward_port}"
|
|
|
|
|
|
def slug(value: str) -> str:
|
|
value = value.lower().replace("*.", "wildcard-")
|
|
value = re.sub(r"[^a-z0-9_-]+", "-", value).strip("-_")
|
|
return value[:40] or "host"
|
|
|
|
|
|
def unique_id(base: str, used: set[str]) -> str:
|
|
base = re.sub(r"[^A-Za-z0-9_-]+", "-", base)[:64].strip("-_") or "npm"
|
|
candidate = base
|
|
index = 2
|
|
while candidate in used:
|
|
suffix = f"-{index}"
|
|
candidate = base[:64 - len(suffix)] + suffix
|
|
index += 1
|
|
used.add(candidate)
|
|
return candidate
|
|
|
|
|
|
def listener_inventory(config: dict[str, Any]) -> tuple[list[str], list[str]]:
|
|
http_ids: list[str] = []
|
|
https_ids: list[str] = []
|
|
for entry in ((config.get("listeners") or {}).get("endpoints") or []):
|
|
if not isinstance(entry, dict) or not entry.get("enabled", True):
|
|
continue
|
|
listener_id = str(entry.get("id") or "").strip()
|
|
binding = entry.get("binding") or {}
|
|
if not listener_id or not isinstance(binding, dict):
|
|
continue
|
|
(https_ids if binding.get("tls") else http_ids).append(listener_id)
|
|
return http_ids, https_ids
|
|
|
|
|
|
def selected_tls_ports(config: dict[str, Any], listener_ids: list[str]) -> set[int]:
|
|
selected = set(listener_ids)
|
|
ports: set[int] = set()
|
|
for entry in ((config.get("listeners") or {}).get("endpoints") or []):
|
|
if not isinstance(entry, dict) or str(entry.get("id") or "") not in selected:
|
|
continue
|
|
binding = entry.get("binding") or {}
|
|
if not isinstance(binding, dict) or not binding.get("tls"):
|
|
continue
|
|
for address in [binding.get("address"), *(entry.get("additional_addresses") or [])]:
|
|
if not isinstance(address, str) or ":" not in address:
|
|
raise MigrationError(f"invalid RustProxy listener address for Force SSL: {address!r}")
|
|
try:
|
|
port = int(address.rsplit(":", 1)[1])
|
|
except (TypeError, ValueError) as error:
|
|
raise MigrationError(f"invalid RustProxy listener address for Force SSL: {address!r}") from error
|
|
if not 1 <= port <= 65535:
|
|
raise MigrationError(f"invalid RustProxy listener port for Force SSL: {address!r}")
|
|
ports.add(port)
|
|
return ports
|
|
|
|
|
|
def existing_host_owners(config: dict[str, Any]) -> dict[str, str]:
|
|
result: dict[str, str] = {}
|
|
for vhost in config.get("vhosts") or []:
|
|
if not isinstance(vhost, dict):
|
|
continue
|
|
vid = str(vhost.get("id") or "?")
|
|
frontend = vhost.get("frontend") or {}
|
|
for raw in frontend.get("hosts") or []:
|
|
if isinstance(raw, str):
|
|
try:
|
|
result[normalize_host_pattern(raw)] = vid
|
|
except MigrationError:
|
|
pass
|
|
return result
|
|
|
|
|
|
def warnings_for(item: NpmProxyHost, tls_verify: bool) -> list[str]:
|
|
warnings: list[str] = []
|
|
if item.access_list_id:
|
|
warnings.append(f"NPM access_list_id={item.access_list_id} was not imported; attach an ACL Profile manually")
|
|
if item.advanced_config.strip():
|
|
warnings.append("NPM advanced_config was not imported")
|
|
if item.locations:
|
|
warnings.append(f"{len(item.locations)} NPM custom location(s) were not imported")
|
|
if item.caching_enabled:
|
|
warnings.append("NPM caching_enabled was not mapped; configure RustProxy cache policy separately")
|
|
if item.block_exploits:
|
|
warnings.append("NPM block_exploits was not mapped")
|
|
if item.ssl_forced:
|
|
warnings.append("NPM ssl_forced is mapped to RustProxy Force SSL (HTTP -> HTTPS 308 redirect)")
|
|
if item.forward_scheme == "https" and tls_verify:
|
|
warnings.append("HTTPS upstream verification was explicitly enabled; provide a matching DNS name/private CA when needed")
|
|
if item.meta.get("nginx_err"):
|
|
warnings.append("NPM meta.nginx_err is set; review the source host before cutover")
|
|
return warnings
|
|
|
|
|
|
def build_migration(
|
|
npm_hosts: list[NpmProxyHost],
|
|
config: dict[str, Any],
|
|
*,
|
|
http_listener_ids: list[str] | None = None,
|
|
https_listener_ids: list[str] | None = None,
|
|
certificate_id: str | None = None,
|
|
force_https: bool = False,
|
|
npm_compatible_backend_tls: bool = False,
|
|
) -> tuple[dict[str, Any], list[Decision]]:
|
|
new_config = json.loads(json.dumps(config))
|
|
new_config.setdefault("vhosts", [])
|
|
discovered_http, discovered_https = listener_inventory(new_config)
|
|
http_ids = http_listener_ids if http_listener_ids is not None else discovered_http[:1]
|
|
https_ids = https_listener_ids if https_listener_ids is not None else discovered_https[:1]
|
|
unknown_http = [value for value in http_ids if value not in discovered_http]
|
|
unknown_https = [value for value in https_ids if value not in discovered_https]
|
|
if unknown_http:
|
|
raise MigrationError("HTTP listener ID is missing, disabled or TLS-enabled: " + ", ".join(unknown_http))
|
|
if unknown_https:
|
|
raise MigrationError("HTTPS listener ID is missing, disabled or not TLS-enabled: " + ", ".join(unknown_https))
|
|
if not http_ids and not https_ids:
|
|
raise MigrationError("RustProxy config has no enabled HTTP/TLS listener to attach imported vhosts")
|
|
https_ports = selected_tls_ports(new_config, https_ids)
|
|
if certificate_id:
|
|
known_certificates = {str(item.get("id")) for item in ((new_config.get("tls") or {}).get("certificates") or []) if isinstance(item, dict)}
|
|
if certificate_id not in known_certificates:
|
|
raise MigrationError(f"RustProxy certificate ID not found: {certificate_id}")
|
|
|
|
owners = existing_host_owners(new_config)
|
|
used_ids = {str(v.get("id")) for v in new_config.get("vhosts") or [] if isinstance(v, dict)}
|
|
decisions: list[Decision] = []
|
|
|
|
for item in npm_hosts:
|
|
warnings = warnings_for(item, False)
|
|
if not item.domain_names:
|
|
decisions.append(Decision(item.id, "skip", [], warnings=warnings, reason="no domain_names"))
|
|
continue
|
|
try:
|
|
normalized = []
|
|
seen = set()
|
|
for raw in item.domain_names:
|
|
host = normalize_host_pattern(raw)
|
|
if host not in seen:
|
|
normalized.append(host)
|
|
seen.add(host)
|
|
origin = backend_url(item)
|
|
except MigrationError as error:
|
|
decisions.append(Decision(item.id, "skip", item.domain_names, warnings=warnings, reason=str(error)))
|
|
continue
|
|
|
|
conflicts = [(host, owners[host]) for host in normalized if host in owners]
|
|
if conflicts:
|
|
reason = "hostname already owned in RustProxy: " + ", ".join(f"{host} ({owner})" for host, owner in conflicts)
|
|
decisions.append(Decision(item.id, "skip", normalized, backend=origin, warnings=warnings, reason=reason))
|
|
continue
|
|
|
|
force_ssl = item.ssl_forced or force_https
|
|
source_tls = item.certificate_id > 0 or force_ssl
|
|
if force_ssl and not https_ids:
|
|
decisions.append(Decision(
|
|
item.id, "skip", normalized, backend=origin, warnings=warnings,
|
|
reason="NPM Force SSL requires an enabled RustProxy TLS listener",
|
|
))
|
|
continue
|
|
if force_ssl and len(https_ports) != 1:
|
|
decisions.append(Decision(
|
|
item.id, "skip", normalized, backend=origin, warnings=warnings,
|
|
reason="RustProxy Force SSL requires the selected TLS listeners to use one HTTPS port",
|
|
))
|
|
continue
|
|
if source_tls:
|
|
listeners = list(dict.fromkeys([*http_ids, *https_ids]))
|
|
else:
|
|
listeners = list(http_ids or https_ids)
|
|
if source_tls and not https_ids:
|
|
warnings.append("source host used TLS but RustProxy has no enabled TLS listener; imported on HTTP only")
|
|
listeners = list(http_ids)
|
|
if force_ssl and not http_ids:
|
|
warnings.append("Force SSL is preserved, but no HTTP listener is selected so there is currently no cleartext request to redirect")
|
|
if not listeners:
|
|
decisions.append(Decision(item.id, "skip", normalized, backend=origin, warnings=warnings, reason="no compatible RustProxy listener"))
|
|
continue
|
|
|
|
created_ids: list[str] = []
|
|
base = f"npm-{item.id}-{slug(normalized[0])}"
|
|
vid = unique_id(base, used_ids)
|
|
vhost: dict[str, Any] = {
|
|
"id": vid,
|
|
"enabled": item.enabled,
|
|
"force_ssl": force_ssl,
|
|
"forward_client_ip": True,
|
|
"websockets": item.allow_websocket_upgrade,
|
|
"http2_enabled": item.http2_support,
|
|
"http3_enabled": False,
|
|
"frontend": {"hosts": normalized, "listener_ids": listeners},
|
|
"backends": [{
|
|
"id": "origin",
|
|
"url": origin,
|
|
"allow_private": True,
|
|
"preserve_host": True,
|
|
"tls_verify": False,
|
|
}],
|
|
"load_balancing": {"health_check": {"mode": "off"}},
|
|
"routes": [{"id": "root", "path_prefix": "/"}],
|
|
}
|
|
if source_tls and certificate_id:
|
|
vhost["certificate_id"] = certificate_id
|
|
new_config["vhosts"].append(vhost)
|
|
created_ids.append(vid)
|
|
for host in normalized:
|
|
owners[host] = vid
|
|
decisions.append(Decision(item.id, "import", normalized, backend=origin, rustproxy_ids=created_ids, warnings=warnings))
|
|
return new_config, decisions
|
|
|
|
|