v0.25.1
This commit is contained in:
Mateusz Gruszczyński committed
1 parent
87e571c170
commit
fa7abc9e53
308 files changed
+49746
-50237
No files matched your search
Executable
+403
@@ -0,0 +1,403 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Migrate Nginx Proxy Manager proxy hosts from SQLite into RustProxy.
|
||||
|
||||
The migrator intentionally focuses on proxy-host identity and upstream routing. It
|
||||
never imports certificate/key material from NPM. NPM hosts that used TLS are attached
|
||||
to RustProxy's TLS listener and use the RustProxy default certificate unless a named
|
||||
certificate is supplied explicitly.
|
||||
|
||||
Run without --apply first. The script logs in to the RustProxy administration API,
|
||||
reads the current revision, builds a merged configuration and prints a review report.
|
||||
Only --apply performs the revisioned PUT /api/config.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass, asdict
|
||||
import getpass
|
||||
import http.cookiejar
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sqlite3
|
||||
import ssl
|
||||
import sys
|
||||
from typing import Any
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
MAX_CONFIG_BYTES = 262_144
|
||||
|
||||
|
||||
class MigrationError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NpmProxyHost:
|
||||
id: int
|
||||
domain_names: list[str]
|
||||
forward_scheme: str
|
||||
forward_host: str
|
||||
forward_port: int
|
||||
allow_websocket_upgrade: bool
|
||||
http2_support: bool
|
||||
enabled: bool
|
||||
certificate_id: int
|
||||
ssl_forced: bool
|
||||
access_list_id: int
|
||||
advanced_config: str
|
||||
locations: list[Any]
|
||||
caching_enabled: bool
|
||||
block_exploits: bool
|
||||
meta: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Decision:
|
||||
npm_id: int
|
||||
action: str
|
||||
hosts: list[str]
|
||||
backend: str | None = None
|
||||
rustproxy_ids: list[str] | None = None
|
||||
warnings: list[str] | None = None
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
def json_value(raw: Any, fallback: Any) -> Any:
|
||||
if raw is None or raw == "":
|
||||
return fallback
|
||||
if isinstance(raw, (list, dict)):
|
||||
return raw
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return fallback
|
||||
|
||||
|
||||
def bool_value(raw: Any) -> bool:
|
||||
if isinstance(raw, str):
|
||||
return raw.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(raw)
|
||||
|
||||
|
||||
def normalize_host_pattern(value: str) -> str:
|
||||
value = value.strip().lower().rstrip(".")
|
||||
wildcard = value.startswith("*.")
|
||||
base = value[2:] if wildcard else value
|
||||
if not base or len(base) > 253 or "*" in base or ":" in base or "/" in base:
|
||||
raise MigrationError(f"invalid hostname {value!r}")
|
||||
labels = base.split(".")
|
||||
if wildcard and len(labels) < 2:
|
||||
raise MigrationError(f"invalid wildcard hostname {value!r}")
|
||||
for label in labels:
|
||||
if not label or len(label) > 63 or label.startswith("-") or label.endswith("-"):
|
||||
raise MigrationError(f"invalid hostname {value!r}")
|
||||
if not re.fullmatch(r"[a-z0-9-]+", label):
|
||||
raise MigrationError(f"hostname must be ASCII/punycode: {value!r}")
|
||||
return ("*." if wildcard else "") + base
|
||||
|
||||
|
||||
def sql_columns(connection: sqlite3.Connection, table: str) -> set[str]:
|
||||
return {str(row[1]) for row in connection.execute(f"PRAGMA table_info({table})")}
|
||||
|
||||
|
||||
def load_npm_hosts(database: Path) -> list[NpmProxyHost]:
|
||||
if database.is_symlink() or not database.is_file():
|
||||
raise MigrationError(f"NPM SQLite database not found or unsafe: {database}")
|
||||
uri = f"file:{urllib.parse.quote(str(database.resolve()))}?mode=ro"
|
||||
connection = sqlite3.connect(uri, uri=True)
|
||||
connection.row_factory = sqlite3.Row
|
||||
try:
|
||||
tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
if "proxy_host" not in tables:
|
||||
raise MigrationError("NPM database does not contain the proxy_host table")
|
||||
columns = sql_columns(connection, "proxy_host")
|
||||
required = {"id", "domain_names", "forward_scheme", "forward_host", "forward_port"}
|
||||
missing = sorted(required - columns)
|
||||
if missing:
|
||||
raise MigrationError("NPM proxy_host table is missing required columns: " + ", ".join(missing))
|
||||
wanted = [
|
||||
"id", "domain_names", "forward_scheme", "forward_host", "forward_port",
|
||||
"allow_websocket_upgrade", "http2_support", "enabled", "certificate_id",
|
||||
"ssl_forced", "access_list_id", "advanced_config", "locations",
|
||||
"caching_enabled", "block_exploits", "meta", "is_deleted",
|
||||
]
|
||||
select = ", ".join(name for name in wanted if name in columns)
|
||||
where = " WHERE is_deleted = 0" if "is_deleted" in columns else ""
|
||||
rows = connection.execute(f"SELECT {select} FROM proxy_host{where} ORDER BY id").fetchall()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
result: list[NpmProxyHost] = []
|
||||
for row in rows:
|
||||
data = dict(row)
|
||||
domains_raw = json_value(data.get("domain_names"), [])
|
||||
if not isinstance(domains_raw, list):
|
||||
domains_raw = []
|
||||
domains: list[str] = []
|
||||
for item in domains_raw:
|
||||
if isinstance(item, str) and item.strip():
|
||||
domains.append(item.strip())
|
||||
locations = json_value(data.get("locations"), [])
|
||||
if not isinstance(locations, list):
|
||||
locations = []
|
||||
meta = json_value(data.get("meta"), {})
|
||||
if not isinstance(meta, dict):
|
||||
meta = {}
|
||||
try:
|
||||
port = int(data.get("forward_port"))
|
||||
except (TypeError, ValueError):
|
||||
port = 0
|
||||
result.append(NpmProxyHost(
|
||||
id=int(data["id"]),
|
||||
domain_names=domains,
|
||||
forward_scheme=str(data.get("forward_scheme") or "").strip().lower(),
|
||||
forward_host=str(data.get("forward_host") or "").strip(),
|
||||
forward_port=port,
|
||||
allow_websocket_upgrade=bool_value(data.get("allow_websocket_upgrade", False)),
|
||||
http2_support=bool_value(data.get("http2_support", True)),
|
||||
enabled=bool_value(data.get("enabled", True)),
|
||||
certificate_id=int(data.get("certificate_id") or 0),
|
||||
ssl_forced=bool_value(data.get("ssl_forced", False)),
|
||||
access_list_id=int(data.get("access_list_id") or 0),
|
||||
advanced_config=str(data.get("advanced_config") or ""),
|
||||
locations=locations,
|
||||
caching_enabled=bool_value(data.get("caching_enabled", False)),
|
||||
block_exploits=bool_value(data.get("block_exploits", False)),
|
||||
meta=meta,
|
||||
))
|
||||
return result
|
||||
|
||||
|
||||
def backend_url(item: NpmProxyHost) -> str:
|
||||
if item.forward_scheme not in {"http", "https"}:
|
||||
raise MigrationError(f"unsupported upstream scheme {item.forward_scheme!r}")
|
||||
if not 1 <= item.forward_port <= 65535:
|
||||
raise MigrationError(f"invalid upstream port {item.forward_port}")
|
||||
host = item.forward_host.strip()
|
||||
if not host or any(ch.isspace() for ch in host) or "/" in host or "@" in host:
|
||||
raise MigrationError(f"invalid upstream host {host!r}")
|
||||
if host.startswith("[") and host.endswith("]"):
|
||||
rendered = host
|
||||
else:
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
rendered = f"[{ip}]" if ip.version == 6 else str(ip)
|
||||
except ValueError:
|
||||
if ":" in host:
|
||||
raise MigrationError(f"invalid upstream host {host!r}")
|
||||
rendered = host
|
||||
return f"{item.forward_scheme}://{rendered}:{item.forward_port}"
|
||||
|
||||
|
||||
def slug(value: str) -> str:
|
||||
value = value.lower().replace("*.", "wildcard-")
|
||||
value = re.sub(r"[^a-z0-9_-]+", "-", value).strip("-_")
|
||||
return value[:40] or "host"
|
||||
|
||||
|
||||
def unique_id(base: str, used: set[str]) -> str:
|
||||
base = re.sub(r"[^A-Za-z0-9_-]+", "-", base)[:64].strip("-_") or "npm"
|
||||
candidate = base
|
||||
index = 2
|
||||
while candidate in used:
|
||||
suffix = f"-{index}"
|
||||
candidate = base[:64 - len(suffix)] + suffix
|
||||
index += 1
|
||||
used.add(candidate)
|
||||
return candidate
|
||||
|
||||
|
||||
def listener_inventory(config: dict[str, Any]) -> tuple[list[str], list[str]]:
|
||||
http_ids: list[str] = []
|
||||
https_ids: list[str] = []
|
||||
for entry in ((config.get("listeners") or {}).get("endpoints") or []):
|
||||
if not isinstance(entry, dict) or not entry.get("enabled", True):
|
||||
continue
|
||||
listener_id = str(entry.get("id") or "").strip()
|
||||
binding = entry.get("binding") or {}
|
||||
if not listener_id or not isinstance(binding, dict):
|
||||
continue
|
||||
(https_ids if binding.get("tls") else http_ids).append(listener_id)
|
||||
return http_ids, https_ids
|
||||
|
||||
|
||||
def selected_tls_ports(config: dict[str, Any], listener_ids: list[str]) -> set[int]:
|
||||
selected = set(listener_ids)
|
||||
ports: set[int] = set()
|
||||
for entry in ((config.get("listeners") or {}).get("endpoints") or []):
|
||||
if not isinstance(entry, dict) or str(entry.get("id") or "") not in selected:
|
||||
continue
|
||||
binding = entry.get("binding") or {}
|
||||
if not isinstance(binding, dict) or not binding.get("tls"):
|
||||
continue
|
||||
for address in [binding.get("address"), *(entry.get("additional_addresses") or [])]:
|
||||
if not isinstance(address, str) or ":" not in address:
|
||||
raise MigrationError(f"invalid RustProxy listener address for Force SSL: {address!r}")
|
||||
try:
|
||||
port = int(address.rsplit(":", 1)[1])
|
||||
except (TypeError, ValueError) as error:
|
||||
raise MigrationError(f"invalid RustProxy listener address for Force SSL: {address!r}") from error
|
||||
if not 1 <= port <= 65535:
|
||||
raise MigrationError(f"invalid RustProxy listener port for Force SSL: {address!r}")
|
||||
ports.add(port)
|
||||
return ports
|
||||
|
||||
|
||||
def existing_host_owners(config: dict[str, Any]) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
for vhost in config.get("vhosts") or []:
|
||||
if not isinstance(vhost, dict):
|
||||
continue
|
||||
vid = str(vhost.get("id") or "?")
|
||||
frontend = vhost.get("frontend") or {}
|
||||
for raw in frontend.get("hosts") or []:
|
||||
if isinstance(raw, str):
|
||||
try:
|
||||
result[normalize_host_pattern(raw)] = vid
|
||||
except MigrationError:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def warnings_for(item: NpmProxyHost, tls_verify: bool) -> list[str]:
|
||||
warnings: list[str] = []
|
||||
if item.access_list_id:
|
||||
warnings.append(f"NPM access_list_id={item.access_list_id} was not imported; attach an ACL Profile manually")
|
||||
if item.advanced_config.strip():
|
||||
warnings.append("NPM advanced_config was not imported")
|
||||
if item.locations:
|
||||
warnings.append(f"{len(item.locations)} NPM custom location(s) were not imported")
|
||||
if item.caching_enabled:
|
||||
warnings.append("NPM caching_enabled was not mapped; configure RustProxy cache policy separately")
|
||||
if item.block_exploits:
|
||||
warnings.append("NPM block_exploits was not mapped")
|
||||
if item.ssl_forced:
|
||||
warnings.append("NPM ssl_forced is mapped to RustProxy Force SSL (HTTP -> HTTPS 308 redirect)")
|
||||
if item.forward_scheme == "https" and tls_verify:
|
||||
warnings.append("HTTPS upstream verification was explicitly enabled; provide a matching DNS name/private CA when needed")
|
||||
if item.meta.get("nginx_err"):
|
||||
warnings.append("NPM meta.nginx_err is set; review the source host before cutover")
|
||||
return warnings
|
||||
|
||||
|
||||
def build_migration(
|
||||
npm_hosts: list[NpmProxyHost],
|
||||
config: dict[str, Any],
|
||||
*,
|
||||
http_listener_ids: list[str] | None = None,
|
||||
https_listener_ids: list[str] | None = None,
|
||||
certificate_id: str | None = None,
|
||||
force_https: bool = False,
|
||||
npm_compatible_backend_tls: bool = False,
|
||||
) -> tuple[dict[str, Any], list[Decision]]:
|
||||
new_config = json.loads(json.dumps(config))
|
||||
new_config.setdefault("vhosts", [])
|
||||
discovered_http, discovered_https = listener_inventory(new_config)
|
||||
http_ids = http_listener_ids if http_listener_ids is not None else discovered_http[:1]
|
||||
https_ids = https_listener_ids if https_listener_ids is not None else discovered_https[:1]
|
||||
unknown_http = [value for value in http_ids if value not in discovered_http]
|
||||
unknown_https = [value for value in https_ids if value not in discovered_https]
|
||||
if unknown_http:
|
||||
raise MigrationError("HTTP listener ID is missing, disabled or TLS-enabled: " + ", ".join(unknown_http))
|
||||
if unknown_https:
|
||||
raise MigrationError("HTTPS listener ID is missing, disabled or not TLS-enabled: " + ", ".join(unknown_https))
|
||||
if not http_ids and not https_ids:
|
||||
raise MigrationError("RustProxy config has no enabled HTTP/TLS listener to attach imported vhosts")
|
||||
https_ports = selected_tls_ports(new_config, https_ids)
|
||||
if certificate_id:
|
||||
known_certificates = {str(item.get("id")) for item in ((new_config.get("tls") or {}).get("certificates") or []) if isinstance(item, dict)}
|
||||
if certificate_id not in known_certificates:
|
||||
raise MigrationError(f"RustProxy certificate ID not found: {certificate_id}")
|
||||
|
||||
owners = existing_host_owners(new_config)
|
||||
used_ids = {str(v.get("id")) for v in new_config.get("vhosts") or [] if isinstance(v, dict)}
|
||||
decisions: list[Decision] = []
|
||||
|
||||
for item in npm_hosts:
|
||||
warnings = warnings_for(item, False)
|
||||
if not item.domain_names:
|
||||
decisions.append(Decision(item.id, "skip", [], warnings=warnings, reason="no domain_names"))
|
||||
continue
|
||||
try:
|
||||
normalized = []
|
||||
seen = set()
|
||||
for raw in item.domain_names:
|
||||
host = normalize_host_pattern(raw)
|
||||
if host not in seen:
|
||||
normalized.append(host)
|
||||
seen.add(host)
|
||||
origin = backend_url(item)
|
||||
except MigrationError as error:
|
||||
decisions.append(Decision(item.id, "skip", item.domain_names, warnings=warnings, reason=str(error)))
|
||||
continue
|
||||
|
||||
conflicts = [(host, owners[host]) for host in normalized if host in owners]
|
||||
if conflicts:
|
||||
reason = "hostname already owned in RustProxy: " + ", ".join(f"{host} ({owner})" for host, owner in conflicts)
|
||||
decisions.append(Decision(item.id, "skip", normalized, backend=origin, warnings=warnings, reason=reason))
|
||||
continue
|
||||
|
||||
force_ssl = item.ssl_forced or force_https
|
||||
source_tls = item.certificate_id > 0 or force_ssl
|
||||
if force_ssl and not https_ids:
|
||||
decisions.append(Decision(
|
||||
item.id, "skip", normalized, backend=origin, warnings=warnings,
|
||||
reason="NPM Force SSL requires an enabled RustProxy TLS listener",
|
||||
))
|
||||
continue
|
||||
if force_ssl and len(https_ports) != 1:
|
||||
decisions.append(Decision(
|
||||
item.id, "skip", normalized, backend=origin, warnings=warnings,
|
||||
reason="RustProxy Force SSL requires the selected TLS listeners to use one HTTPS port",
|
||||
))
|
||||
continue
|
||||
if source_tls:
|
||||
listeners = list(dict.fromkeys([*http_ids, *https_ids]))
|
||||
else:
|
||||
listeners = list(http_ids or https_ids)
|
||||
if source_tls and not https_ids:
|
||||
warnings.append("source host used TLS but RustProxy has no enabled TLS listener; imported on HTTP only")
|
||||
listeners = list(http_ids)
|
||||
if force_ssl and not http_ids:
|
||||
warnings.append("Force SSL is preserved, but no HTTP listener is selected so there is currently no cleartext request to redirect")
|
||||
if not listeners:
|
||||
decisions.append(Decision(item.id, "skip", normalized, backend=origin, warnings=warnings, reason="no compatible RustProxy listener"))
|
||||
continue
|
||||
|
||||
created_ids: list[str] = []
|
||||
base = f"npm-{item.id}-{slug(normalized[0])}"
|
||||
vid = unique_id(base, used_ids)
|
||||
vhost: dict[str, Any] = {
|
||||
"id": vid,
|
||||
"enabled": item.enabled,
|
||||
"force_ssl": force_ssl,
|
||||
"forward_client_ip": True,
|
||||
"websockets": item.allow_websocket_upgrade,
|
||||
"http2_enabled": item.http2_support,
|
||||
"http3_enabled": False,
|
||||
"frontend": {"hosts": normalized, "listener_ids": listeners},
|
||||
"backends": [{
|
||||
"id": "origin",
|
||||
"url": origin,
|
||||
"allow_private": True,
|
||||
"preserve_host": True,
|
||||
"tls_verify": False,
|
||||
}],
|
||||
"load_balancing": {"health_check": {"mode": "off"}},
|
||||
"routes": [{"id": "root", "path_prefix": "/"}],
|
||||
}
|
||||
if source_tls and certificate_id:
|
||||
vhost["certificate_id"] = certificate_id
|
||||
new_config["vhosts"].append(vhost)
|
||||
created_ids.append(vid)
|
||||
for host in normalized:
|
||||
owners[host] = vid
|
||||
decisions.append(Decision(item.id, "import", normalized, backend=origin, rustproxy_ids=created_ids, warnings=warnings))
|
||||
return new_config, decisions
|
||||
|
||||
|
||||
Reference in new issue
Block a user