This commit is contained in:
Mateusz Gruszczyński committed 2026-10-04 11:15:58 +02:00
1 parent 87e571c170
commit fa7abc9e53
308 files changed
+49746 -50237

No files matched your search

+118 -107
View File
@@ -8,15 +8,23 @@ import sqlite3
import subprocess
import tomllib
from preflight import check as preflight
from rust_source import rust_source
from web_assets import read_app_js, read_vhosts_js
root = Path(__file__).resolve().parents[1]
def read_cache_source() -> str:
return "\n".join(path.read_text() for path in sorted((root / "src/cache").glob("*.rs")))
return rust_source(root, "src/cache/mod.rs")
def read_deploy_source() -> str:
return "\n".join((root / name).read_text(encoding="utf-8") for name in ("scripts/deploy_core.py", "scripts/deploy_runtime.py", "scripts/deploy.py"))
def read_docker_agent_source() -> str:
return "\n".join(path.read_text(encoding="utf-8") for path in sorted((root / "docker-agent").glob("*.go")) if not path.name.endswith("_test.go"))
preflight(root)
manifest = tomllib.loads((root / "Cargo.toml").read_text())
assert manifest["package"]["version"] == "0.25.0"
assert manifest["package"]["version"] == "0.25.1"
assert manifest["dependencies"]["quinn"]["version"] == "0.11.12"
h3dep = manifest["dependencies"]["h3"]
assert h3dep["package"] == "hpx-h3" and h3dep["version"] == "=2.5.19" and "quinn" in h3dep["features"]
@@ -24,14 +32,14 @@ assert "h3-quinn" not in manifest["dependencies"]
lock_path = root / "Cargo.lock"
if lock_path.exists():
lock = lock_path.read_text()
assert 'name = "rust-proxy"\nversion = "0.25.0"' in lock
assert 'name = "rust-proxy"\nversion = "0.25.1"' in lock
assert 'name = "systemstat"' in lock and 'name = "metrics-process"' in lock and 'name = "sysinfo"' in lock, "Cargo.lock must be resolver-generated after adding cross-platform statistics libraries"
for release_doc, marker in {
"README.md": "# RustProxy 0.25.0",
"CHANGELOG.md": "## 0.25.0",
"ROADMAP.md": "## 0.25.0",
"AGENTS.md": "Current source version: 0.25.0.",
"docs/TEST_REPORT.md": "# Verification report - RustProxy 0.25.0",
"README.md": "# RustProxy 0.25.1",
"CHANGELOG.md": "## 0.25.1",
"ROADMAP.md": "## 0.25.1",
"AGENTS.md": "Current source version: 0.25.1.",
"docs/TEST_REPORT.md": "# Verification report - RustProxy 0.25.1",
}.items():
assert marker in (root / release_doc).read_text(), f"stale release identity in {release_doc}"
assert tomllib.loads((root / "rust-toolchain.toml").read_text())["toolchain"]["channel"] == "stable"
@@ -41,7 +49,7 @@ assert update_defaults == {
"version_url": "https://git.linuxiarz.pl/gru/RustProxy/raw/branch/master/Cargo.toml",
"interval_seconds": 3600,
}
update_check_source = (root / "src/update_check.rs").read_text()
update_check_source = rust_source(root, "src/update_check.rs")
for marker in [
'include_str!("../UPDATE_CHECK.toml")',
'pub async fn poll_loop',
@@ -82,7 +90,7 @@ for path in (root / "src").rglob("*.rs"):
for index in range(len(lines) - 1):
assert not (lines[index].lstrip().startswith("#[derive(") and lines[index + 1].lstrip().startswith("#[derive(")), f"consecutive derive attributes in {path}:{index + 1}"
html = (root / "src/admin/web/index.html").read_text()
js = "\n".join(p.read_text() for p in (root / "src/admin/web").glob("*.js") if p.name not in ("login.js", "docs.js"))
js = "\n".join(p.read_text() for p in (root / "src/admin/web").rglob("*.js") if p.name not in ("login.js", "docs.js"))
ids = re.findall(r'id="([^"]+)"', html)
assert len(ids) == len(set(ids)), "duplicate HTML IDs"
for identifier in set(re.findall(r'\$\("([^"]+)"\)', js)):
@@ -112,10 +120,12 @@ unit = (root / "deploy/rustproxy.service").read_text()
assert "LogsDirectory=rustproxy" in unit and "/var/log/rustproxy" in unit
assert "Environment=MALLOC_ARENA_MAX=2" in unit
assert "Environment=MALLOC_ARENA_MAX=2" in (root / "debian/rustproxy.service").read_text()
assert "Environment=MALLOC_ARENA_MAX=2" in (root / "scripts/deploy.py").read_text()
assert "Environment=MALLOC_ARENA_MAX=2" in read_deploy_source()
assert not re.search(r"\b20\d{2}[-/]\d{1,2}[-/]\d{1,2}\b", (root / "CHANGELOG.md").read_text()), "changelog contains a release date"
subprocess.run(["python3", str(root / "scripts/langs_check.py")], check=True)
subprocess.run(["python3", str(root / "scripts/web_server_contracts.py")], check=True)
subprocess.run(["python3", str(root / "scripts/rust_module_contracts.py")], check=True)
subprocess.run(["python3", str(root / "scripts/source_module_contracts.py")], check=True)
dockerfile = (root / "Dockerfile").read_text()
assert "cargo generate-lockfile" in dockerfile and "cargo build --locked --release" in dockerfile
assert "USER rustproxy:rustproxy" in dockerfile
@@ -131,8 +141,9 @@ assert 'data-panel="docker-vhosts"' in html and 'id="panel-docker-vhosts"' in ht
assert 'id="settings-tab-docker-agents"' in html and 'id="settings-page-docker-agents"' in html
assert '/assets/docker.js' in html and 'cache-live-toggle' in html
assert 'expose-revision' not in html and 'hideRevision' not in js and 'diagRevision' not in js
for path in ['src/settings.rs', 'src/admin/mod.rs', 'src/cache/mod.rs', 'src/headers.rs', 'src/tinyauth.rs', 'src/admin/web/vhosts.js']:
assert 'x-rust-proxy-revision' not in (root / path).read_text().lower(), f'legacy revision-header logic remains in {path}'
for path in ['src/settings.rs', 'src/admin/mod.rs', 'src/cache/mod.rs', 'src/headers.rs', 'src/tinyauth.rs', 'src/admin/web/vhosts/core.js']:
text = rust_source(root, path) if path.endswith('.rs') else (root / path).read_text()
assert 'x-rust-proxy-revision' not in text.lower(), f'legacy revision-header logic remains in {path}'
assert './dev.sh admin reset-password' not in html and re.search(r'rust-proxy\s+admin reset-password locally', html)
assert 'data-panel="redirects"' in html and 'id="panel-redirects"' in html
assert 'data-panel="configuration"' not in html and 'id="panel-configuration"' not in html
@@ -140,7 +151,7 @@ assert 'id="settings-tab-manual-config"' in html and 'id="settings-page-manual-c
assert 'id="security-audit-open"' in html and 'id="audit-dialog"' in html and 'id="audit-dialog-close"' in html
assert 'id="security-password-editor"' in html and 'id="security-policy-editor"' in html
assert 'docker-service-editor' in (root / "src/admin/web/docker.js").read_text() and 'docker-agent-details' in (root / "src/admin/web/docker.js").read_text()
balance_source = (root / "src/balance.rs").read_text()
balance_source = rust_source(root, "src/balance.rs")
assert 'mode: HealthMode::Tcp' in balance_source
for marker in [
'pub struct PassiveHealth', 'enabled: false', 'pub passive_health: PassiveHealth',
@@ -160,7 +171,7 @@ assert html.index('data-panel="vhosts"') < html.index('data-panel="redirects"')
assert 'id="stream-dialog"' in html and 'id="stream-list"' in html and 'id="stream-search"' in html
assert 'id="settings-page-streams"' not in html and 'id="streams-form"' not in html
assert '/assets/streams.js' in html and '/assets/redirects.js' in html and '/assets/host404.js' in html
vhosts_js = (root / "src/admin/web/vhosts.js").read_text()
vhosts_js = read_vhosts_js(root)
assert '"force-ssl", "vhForceSsl"' in vhosts_js and 'force_ssl: yes("force-ssl")' in vhosts_js
assert 'vhForceSslNeedsHttp' in vhosts_js and 'tlsPorts.has(443)' in vhosts_js
assert '"forward-client-ip", "vhForwardClientIp"' in vhosts_js and 'forward_client_ip: yes("forward-client-ip")' in vhosts_js
@@ -182,10 +193,10 @@ docker_js = (root / "src/admin/web/docker.js").read_text()
assert 'preserve.checked = assignment?.preserve_host !== false' in docker_js, "new Docker backends must preserve public Host by default"
for marker in ['saveSection("docker-agents"', '/api/docker/refresh', 'docker-publish', 'docker-hostname', 'host-chip-editor docker-alias-editor', 'docker-target-ip', 'default_target_ip', 'docker-agent-fingerprint', 'docker-agent-certificate', 'docker-health-mode', 'docker-passive-enabled', 'docker-passive-5xx', 'docker-search', 'docker-show-hidden', 'dockerHideVhost', 'pool_routes', 'setInterval']:
assert marker in docker_js, marker
docker_source = (root / "src/docker.rs").read_text()
docker_source = rust_source(root, "src/docker.rs")
for marker in ['pub struct Settings', 'pub struct Observation', 'pub struct HostAddress', 'pub struct Binding', 'aliases: Vec<String>', 'target_ip: Option<IpAddr>', 'tls_fingerprint_sha256', 'certificate_id: Option<String>', 'health_check: crate::balance::HealthCheck', 'passive_health: crate::balance::PassiveHealth', 'PinnedCertificateVerifier', 'fingerprint mismatch', 'effective_config', 'poll_loop', 'address_allowlist']:
assert marker in docker_source, marker
agent_source = (root / "docker-agent/main.go").read_text()
agent_source = read_docker_agent_source()
for marker in ['/v1/inventory', 'Authorization', '/containers/json?all=0', 'IsLoopback', 'localHostAddresses', 'detectedAddresses', 'Bindings', 'RUSTPROXY_DOCKER_AGENT_TOKEN', 'ensureSelfSignedCertificate', 'ListenAndServeTLS', 'RUSTPROXY_DOCKER_AGENT_TLS', 'CertificateSHA256']:
assert marker in agent_source, marker
assert 'pool-vhost-disabled' in vhosts_js and '.pool-vhost-disabled' in css
@@ -213,7 +224,7 @@ assert 'Opt-in health decisions based on ordinary user traffic. Defaults to disa
assert '"force_ssl"' in openapi and '"forward_client_ip"' in openapi
assert '"backend_header"' in openapi and '"backend_timing_header"' in openapi
assert '"StreamMetrics"' in openapi and '"streams"' in openapi
for marker in ['"version": "0.25.0"', '"/api/traffic"', '"capture_user_agent"', '"capture_country"', '"hot_auto_trim_enabled"', '"hot_auto_trim_after_seconds"', '"hot_auto_trim_target_percent"', '"/api/config/{section}"', '"ConfigSectionUpdate"', '"cold_on_idle"', '"cold_on_age"', '"cold_max_hot_seconds"', '"cold_on_lru"', '"cold_on_shutdown"', '"cold_gc_interval_seconds"', '"cold_gc_max_delete"', '"collapse_enabled"', '"collapse_wait_timeout_ms"', '"collapse_timeout_fallback"', '"stale_if_error_timeout"', '"stale_if_error_connection"', '"stale_if_error_dns"', '"stale_if_error_tls"', '"stale_if_error_statuses"', '"proactive_refresh_enabled"', '"proactive_refresh_before_expiry_seconds"', '"proactive_refresh_percent"', '"proactive_refresh_max_inflight"', '"TransformSettings"', '"TransformOverrides"', '"/api/cache/cold/cleanup"', '"/api/cache/invalidate"', '"/api/cache/bans"', '"/api/cache/bans/{id}"', '"CacheInvalidationRequest"', '"CacheInvalidationResponse"', '"ColdCleanupJob"', '"match_type"', '"regex_insensitive"', '"rewrite"', '"response"', '"covered_hosts"', '"default": true']:
for marker in ['"version": "0.25.1"', '"/api/traffic"', '"capture_user_agent"', '"capture_country"', '"hot_auto_trim_enabled"', '"hot_auto_trim_after_seconds"', '"hot_auto_trim_target_percent"', '"/api/config/{section}"', '"ConfigSectionUpdate"', '"cold_on_idle"', '"cold_on_age"', '"cold_max_hot_seconds"', '"cold_on_lru"', '"cold_on_shutdown"', '"cold_gc_interval_seconds"', '"cold_gc_max_delete"', '"collapse_enabled"', '"collapse_wait_timeout_ms"', '"collapse_timeout_fallback"', '"stale_if_error_timeout"', '"stale_if_error_connection"', '"stale_if_error_dns"', '"stale_if_error_tls"', '"stale_if_error_statuses"', '"proactive_refresh_enabled"', '"proactive_refresh_before_expiry_seconds"', '"proactive_refresh_percent"', '"proactive_refresh_max_inflight"', '"TransformSettings"', '"TransformOverrides"', '"/api/cache/cold/cleanup"', '"/api/cache/invalidate"', '"/api/cache/bans"', '"/api/cache/bans/{id}"', '"CacheInvalidationRequest"', '"CacheInvalidationResponse"', '"ColdCleanupJob"', '"match_type"', '"regex_insensitive"', '"rewrite"', '"response"', '"covered_hosts"', '"default": true']:
assert marker in openapi, marker
for marker in ['"NotFoundHost"', '"not_found_hosts"', '"PrometheusSettings"', '"allow_trusted_network_without_token"', '"include_docker_vhosts"', '"include_streams"', '"include_logging"']:
assert marker in openapi, marker
@@ -233,17 +244,17 @@ cache_source = read_cache_source()
assert not (root / "src/cache.rs").exists()
for fragment in ["mod.rs", "model.rs", "lifecycle.rs", "cold.rs", "dashboard.rs", "tickets.rs", "lookup.rs", "policy.rs", "fill.rs", "collapse.rs", "range.rs", "invalidation.rs", "tests.rs"]:
assert (root / "src/cache" / fragment).is_file(), fragment
cache_mod = (root / "src/cache/mod.rs").read_text()
cache_mod = rust_source(root, "src/cache/mod.rs")
assert 'impl Cache {\n include!(' not in cache_mod, "include! cannot be used directly as Cache impl items"
for fragment in ["lifecycle.rs", "cold.rs", "dashboard.rs", "tickets.rs", "lookup.rs"]:
assert (root / "src/cache" / fragment).read_text().lstrip().startswith("impl Cache {"), fragment
assert rust_source(root, f"src/cache/{fragment}").lstrip().startswith("impl Cache {"), fragment
assert 'disabled_cache_strips_diagnostics_without_exposing_bypass_headers' in cache_source
for marker in ['pub enum MemoryPreset', 'pub enum ColdPreset', 'BACKGROUND_REVALIDATION_LIMIT', 'cleanup_slots']:
assert marker in cache_source, marker
for marker in ['selectedCacheMemoryPreset', 'selectedCacheColdPreset', 'applyRecommendedColdCachePreset', 'data-cache-cold-preset="recommended"']:
assert marker in js or marker in html, marker
assert 'if !self.enabled' in cache_source
proxy_source = (root / "src/proxy/mod.rs").read_text()
proxy_source = rust_source(root, "src/proxy/mod.rs")
assert 'response.extensions().get::<LocalHostResponse>().is_none()' in proxy_source
assert '"cdn-cache-control"' in proxy_source and '"x-cache-resident-age"' in proxy_source
for marker in [
@@ -290,7 +301,7 @@ for marker in [
'selective_purge_removes_matching_cold_object',
]:
assert marker in cache_source, marker
transform_source = (root / "src/transform/mod.rs").read_text()
transform_source = rust_source(root, "src/transform/mod.rs")
for marker in [
'pub struct Settings',
'pub struct Overrides',
@@ -312,9 +323,9 @@ for marker in [
assert marker in transform_source, marker
assert 'collect bounded static response for transformation: {error}' in transform_source
assert '.context("collect bounded static response for transformation")' not in transform_source
lookup_source = (root / 'src/cache/lookup.rs').read_text()
model_source = (root / 'src/cache/model.rs').read_text()
css_minifier_source = (root / 'src/transform/css.rs').read_text()
lookup_source = rust_source(root, 'src/cache/lookup.rs')
model_source = rust_source(root, 'src/cache/model.rs')
css_minifier_source = rust_source(root, 'src/transform/css.rs')
assert 'let Some(_read_lease) = self.try_reserve_transient' in lookup_source
assert 'struct TransientLease {\n _inner: Arc<TransientLeaseInner>,' in model_source
assert 'struct TransientLease(Arc<TransientLeaseInner>);' not in model_source
@@ -326,7 +337,7 @@ for name in ['js.rs', 'css.rs']:
minifier_source = (root / 'src/transform' / name).read_text()
assert 'Vec<char>' not in minifier_source and '.chars().collect' not in minifier_source, name
assert 'into_bytes()' in minifier_source, name
assert 'pub mod transform;' in (root / 'src/lib.rs').read_text()
assert 'pub mod transform;' in rust_source(root, 'src/lib.rs')
for marker in ['cache-cold-gc-interval', 'cache-cold-gc-max-delete', 'settings-tab-transform', 'transform-form', 'transform-memory-budget', 'transform-max-inflight', 'data-transform-preset="uber"', 'cache-cleanup-dialog']:
assert marker in html, marker
for marker in ['cache-transient-memory', 'cache-cold-queue-bytes', 'cache-eviction-limit', 'data-cache-memory-preset="recommended"', 'cache-memory-warning']:
@@ -355,7 +366,7 @@ for marker in ['liveChartSamples = { hits: [], misses: [], bypasses: [] }', 'LIV
assert marker in cache_panel_js, marker
for marker in ['liveChartSamples = { applied: [], cache_hits: [], skipped: [], errors: [] }', 'LIVE_CHART_LIMITS = new Set([60, 120, 300, 600])', 'rustproxy.transform.liveChartSamples', 'performance.now()', 'sparklinePoints(liveChartSamples.applied, 900, 180, 10, peak)', 'slice(-liveChartLimit)']:
assert marker in transform_panel_js, marker
admin_cache_source = (root / 'src/admin/mod.rs').read_text()
admin_cache_source = rust_source(root, 'src/admin/mod.rs')
for marker in ['"transform" => {', 'transform_status', 'cache_cold_cleanup', 'cache_cold_cleanup_job', 'cache_invalidate', 'cache_bans', 'cache_ban_delete', 'selective_cache_invalidation_api_manages_bans_and_rejects_wide_rules_without_confirmation']:
assert marker in admin_cache_source, marker
for marker in [
@@ -378,10 +389,10 @@ for marker in [
]:
assert marker in cache_source, marker
assert 'impl Drop for Candidate' not in cache_source, 'fill accounting must not block moving Candidate fields during finish'
redirect_source = (root / "src/redirect.rs").read_text()
redirect_source = rust_source(root, "src/redirect.rs")
for marker in ["pub struct Host", "pub struct Rule", "strip_prefix", "cache_seconds", "unconditional self-redirect"]:
assert marker in redirect_source, marker
proxy_source = (root / "src/proxy/mod.rs").read_text()
proxy_source = rust_source(root, "src/proxy/mod.rs")
assert 'state.cache.purge(None)' not in proxy_source, 'proxy requests must not globally purge cache'
assert 'request.method() == Method::HEAD && cache_ticket.is_some()' in proxy_source
assert 'RangeSelection::Single' in proxy_source and 'request_range(request.method(), request.headers())' in proxy_source
@@ -389,7 +400,7 @@ assert 'claim_fill(ticket, cache_settings)' in proxy_source and 'cache_collapse_
assert 'schedule_proactive_refresh' in proxy_source and 'cache_proactive_refreshed' in proxy_source
assert 'classify_stale_error' in proxy_source and 'allows_stale_error_status' in proxy_source
assert 'Varnish built-in behavior treats non-GET/HEAD methods as pass' in proxy_source
config_source = (root / "src/config.rs").read_text()
config_source = rust_source(root, "src/config.rs")
assert 'plain HTTP listener' in config_source and 'tls_ports.contains(&443)' in config_source
assert '#[serde(default)]\n pub tls_verify: bool' in config_source
assert 'backend_tls_verification_defaults_off_and_can_be_enabled_explicitly' in config_source
@@ -407,7 +418,7 @@ for marker in [
'backend_timing_header: host.backend_timing_header',
]:
assert marker in config_source, marker
vhost_source = (root / "src/vhost.rs").read_text()
vhost_source = rust_source(root, "src/vhost.rs")
assert 'is_legacy_main_fallback' in vhost_source
assert 'values.length ? values : [{ id: "root"' not in vhosts_js
for marker in [
@@ -423,29 +434,29 @@ for marker in [
'websocket_phase("tunnel_ready")',
]:
assert marker in proxy_source, marker
generic_headers_source = (root / "src/headers.rs").read_text()
generic_headers_source = rust_source(root, "src/headers.rs")
assert '"x-rust-proxy-backend"' in generic_headers_source and '"x-rustproxy-backend-time"' in generic_headers_source
headers_source = (root / "src/proxy/headers.rs").read_text()
headers_source = rust_source(root, "src/proxy/headers.rs")
for marker in ['pub fn restore_admin_cookie', 'admin_cookie_can_be_restored_only_from_captured_fragment']:
assert marker in headers_source, marker
settings_source = (root / "src/settings.rs").read_text()
settings_source = rust_source(root, "src/settings.rs")
for marker in ['pub session_days: u16', 'session_days: 30', '(7..=30).contains(&self.session_days)', 'administration_session_lifetime_is_7_to_30_days_and_hot_apply_only']:
assert marker in settings_source, marker
auth_source = (root / "src/auth/mod.rs").read_text()
auth_store = (root / "src/auth/store.rs").read_text()
auth_source = rust_source(root, "src/auth/mod.rs")
auth_store = rust_source(root, "src/auth/store.rs")
for marker in ['30 * 24 * 60 * 60', 'session_cookie(secret: &str, secure: bool, max_age_seconds: i64)', 'browser_session_lifetime_is_configurable_without_sliding_expiry', 'session_cookie_uses_configured_remaining_lifetime']:
assert marker in auth_source, marker
for marker in ['create_browser_session', 'session_at_with_browser_lifetime', 'browser session lifetime must be 7..30 days', "created_at + ?"]:
assert marker in auth_store, marker
upstream_source = (root / "src/upstream.rs").read_text()
upstream_source = rust_source(root, "src/upstream.rs")
for marker in ['pub accept_local_listener_risk: bool', '!self_target || self.accept_local_listener_risk', 'explicit_risk_acceptance_allows_local_listener_but_not_private_policy_bypass']:
assert marker in upstream_source, marker
transport_tests = (root / "tests/transport_config.rs").read_text()
assert 'local_listener_backend_requires_explicit_risk_acceptance' in transport_tests and '127.0.0.1:9090' in transport_tests
runtime_source = (root / "src/runtime.rs").read_text()
runtime_source = rust_source(root, "src/runtime.rs")
for marker in ['fn cache_revision', 'CacheInvalidation::None', 'operational_saves_keep_cache_identity_but_cache_affecting_saves_rotate_it']:
assert marker in runtime_source, marker
admin = (root / "src/admin/mod.rs").read_text()
admin = rust_source(root, "src/admin/mod.rs")
for marker in [
"fn admin_request_authority",
"Version::HTTP_2 | Version::HTTP_3",
@@ -458,7 +469,7 @@ for marker in ['"/api/config/{section}"', 'async fn update_section', 'section_up
assert marker in admin, marker
for route in ['/overview', '/vhosts', '/redirects', '/404-hosts', '/streams', '/acl-profiles', '/configuration', '/tls', '/cache', '/transform', '/logging', '/settings']:
assert f'.route("{route}", get(public::index))' in admin, route
app_js = (root / "src/admin/web/app.js").read_text()
app_js = read_app_js(root)
for marker in ['const PANEL_PATHS', 'history.pushState', 'window.addEventListener("popstate"', 'async function saveSection', 'async function revalidateSession', 'Object.values(PANEL_PATHS), "/configuration"', 'certificateCovers', 'covered_hosts', 'refreshCertificateInventory', 'certificateDetails', 'certificate-domain-pills', 'certificateIssuerName', 'certificateDaysRemaining', 'certificateRenewalStatus']:
assert marker in app_js, marker
for marker in ['.certificate-details', '.certificate-domain-pills', '.certificate-card .pill', '.certificate-operation-card', '.renewal-meta', '#panel-tls .certificate-tabs']:
@@ -495,13 +506,13 @@ for marker in ['function setOptionalMetric', 'setOptionalMetric("process-rss-ano
assert marker in app_js, marker
for marker in ['/api/certificate-api', 'certificate-api-form', 'certificate-api-revoke-all', 'certificate-api-credentials', 'data-cache-settings-tab']:
assert marker in settings_js or marker in html, marker
db_source = (root / "src/db.rs").read_text()
acme_source = (root / "src/acme.rs").read_text()
main_source = (root / "src/main.rs").read_text()
logging_source = (root / "src/logging.rs").read_text()
secret_store_source = (root / "src/secret_store.rs").read_text()
backup_source = (root / "src/backup.rs").read_text()
config_source = (root / "src/config.rs").read_text()
db_source = rust_source(root, "src/db.rs")
acme_source = rust_source(root, "src/acme.rs")
main_source = rust_source(root, "src/main.rs")
logging_source = rust_source(root, "src/logging.rs")
secret_store_source = rust_source(root, "src/secret_store.rs")
backup_source = rust_source(root, "src/backup.rs")
config_source = rust_source(root, "src/config.rs")
acme_worker_source = (root / "scripts/acme_provider.py").read_text()
for feature_marker in [
"AES_256_GCM", "master.key", "O_NOFOLLOW", "migrate_acme_credentials",
@@ -545,7 +556,7 @@ for marker in [
'LocalAdminCertificateVerifier', 'detect_local_admin_scheme',
'local_admin_scheme_from_probe', 'admin.uri("/api/live-logs")',
]:
assert marker in main_source or marker in (root / 'src/auth/store.rs').read_text(), marker
assert marker in main_source or marker in rust_source(root, 'src/auth/store.rs'), marker
assert '.uri(format!("http://{addr}/api/live-logs"))' not in main_source
assert 'http://{addr}/healthz' in main_source and 'StatusCode::PERMANENT_REDIRECT' in main_source
assert 'Ok(line) if command_tx.send(line).is_ok()' not in main_source, 'do not move String from a match guard in --livelogs stdin handling'
@@ -567,29 +578,29 @@ for marker in [
'state.start_update_check(update_check);',
]:
assert marker in main_source, marker
runtime_source = (root / "src/runtime.rs").read_text()
runtime_source = rust_source(root, "src/runtime.rs")
assert 'pub software_update: Arc<crate::update_check::SharedStatus>' in runtime_source
assert 'pub fn start_update_check(self: &Arc<Self>, config: crate::update_check::Config)' in runtime_source
assert 'payload["software_update"] = json!(state.software_update.snapshot());' in admin
settings_rust = (root / 'src/settings.rs').read_text()
settings_rust = rust_source(root, 'src/settings.rs')
assert 'pub struct StartupLocks' in settings_rust and 'fixed by systemd/startup' in settings_rust
logging_rust = (root / 'src/logging.rs').read_text()
logging_rust = rust_source(root, 'src/logging.rs')
assert 'pub per_vhost: bool' in logging_rust and 'default_vhost_log_directory' in logging_rust and '_unmatched' in logging_rust
pages_rust = (root / 'src/pages.rs').read_text()
pages_rust = rust_source(root, 'src/pages.rs')
assert 'validate_local_response' in pages_rust and '(200..=599).contains(&status)' in pages_rust and '204 | 205 | 304' in pages_rust
assert 'store.load_certificate_renewals().await?' in main_source and 'state.acme.restore_renewals(renewal_observations)' in main_source
assert 'renewals.json' not in main_source
upgrade_source = (root / 'scripts/upgrade.py').read_text()
assert 'legacy = certs_dir / "renewals.json"' in upgrade_source and 'removed_obsolete_runtime_state' in upgrade_source and 'legacy-data' not in upgrade_source and 'certs_dir.is_symlink()' in upgrade_source
assert '.managed-update' in main_source and 'flush_hot_to_cold_for_update' in main_source
tls_source = (root / "src/tls.rs").read_text()
tls_source = rust_source(root, "src/tls.rs")
assert 'pub fn validate_certificate_removals' in tls_source and 'administration panel' in tls_source
assert 'pub fn effective_cipher_suites' in tls_source
vhost_source = (root / 'src/vhost.rs').read_text()
vhost_source = rust_source(root, 'src/vhost.rs')
assert 'pub websocket_rfc9220: bool' in vhost_source and 'RFC 9220 requires WebSocket support to be enabled' in vhost_source
for marker in ['pub struct ProxyTimeouts', 'connect_timeout_seconds', 'response_header_timeout_seconds', 'read_timeout_seconds', 'send_timeout_seconds']:
assert marker in vhost_source, marker
docker_source = (root / 'src/docker.rs').read_text()
docker_source = rust_source(root, 'src/docker.rs')
for marker in ['pub poll_interval_seconds: u64', 'pub acl_profile_id: Option<String>', 'pub proxy_timeouts: crate::vhost::ProxyTimeouts', 'refresh(&self, settings: &Settings, force: bool)', 'poll_interval_seconds.saturating_mul(1000)']:
assert marker in docker_source, marker
docker_js = (root / 'src/admin/web/docker.js').read_text()
@@ -599,33 +610,33 @@ agent_installer = (root / 'docker-agent/install.sh').read_text()
assert 'Bearer token:' in agent_installer and 'RUSTPROXY_DOCKER_AGENT_TOKEN=' in agent_installer
assert 'Kind::Builtin' in pages_rust and 'builtin_application_page' in pages_rust
assert 'pub covered_hosts: Vec<String>' in tls_source and 'fn certificate_dns_names' in tls_source and 'pub fn certificate_covered_hosts' in tls_source and 'legacy_certificate_coverage_can_be_read_from_existing_pem' in tls_source and 'assert_eq!(a.covered_hosts' in tls_source
acl_source = (root / 'src/acl.rs').read_text()
acl_source = rust_source(root, 'src/acl.rs')
assert '/Ubuntu%2014.04%20LTS/' in acl_source and 'repeated_slashes_are_normalized_for_routing' in acl_source and 'merge_slashes' in acl_source
policies_source = (root / 'tests/policies.rs').read_text()
assert 'percent_encoded_spaces_are_valid_request_paths' in policies_source and 'repeated_slashes_route_normally_but_reach_upstream_unchanged' in policies_source and 'local_location_response_uses_response_rules_without_opening_backend' in policies_source and 'local_location_response_still_respects_route_acl' in policies_source
http3_source = (root / "src/http3.rs").read_text()
http3_source = rust_source(root, "src/http3.rs")
drop_impl = http3_source.split('impl Drop for ReceiveBody', 1)[1].split('#[cfg(test)]', 1)[0]
assert 'self.stream.stop_sending(' not in drop_impl
for marker in ['h3::quinn::RecvStream', 'websocket_rfc9220_on(local)', 'run_h3', 'H3_REQUEST_CANCELLED']:
assert marker in http3_source or marker in (root / 'src/proxy/websocket.rs').read_text(), marker
assert marker in http3_source or marker in rust_source(root, 'src/proxy/websocket.rs'), marker
assert 'h3_quinn' not in http3_source and 'h3_quinn' not in (root / 'tests/http3.rs').read_text()
h3_test = (root / 'tests/http3.rs').read_text()
assert 'RFC 9220: Extended CONNECT negotiates WebSocket on the same H3 stream.' in h3_test and '"websocket_rfc9220":true' in h3_test
assert 'Method::CONNECT' in h3_test and 'h3::ext::Protocol' in h3_test and '.send_data(' in h3_test
admin_source = (root / 'src/admin/mod.rs').read_text()
admin_source = rust_source(root, 'src/admin/mod.rs')
for marker in ['external_admin_authority', 'external_admin_scheme', 'trusted_admin_forwarded_proto', 'reverse_proxy_dns_allowlist_uses_external_port_and_keeps_direct_port_available', 'trusted_https_reverse_proxy_accepts_external_origin_and_refreshes_secure_cookie']:
assert marker in admin_source, marker
ws_source = (root / 'src/proxy/websocket.rs').read_text()
ws_source = rust_source(root, 'src/proxy/websocket.rs')
assert 'upstream: Upgraded' in ws_source
assert 'timeout(Duration::from_secs(10), upstream)' not in ws_source
for marker in ['websocket_phase("tunnel_started")', 'websocket_phase("tunnel_closed")', 'h3_reset("H3_REQUEST_CANCELLED")']:
assert marker in ws_source, marker
for marker in ['http_version', 'transport', 'websocket_bootstrap', 'websocket_phase', 'websocket_error', 'h3_reset_code', 'upstream_status']:
assert marker in (root / 'src/logging.rs').read_text(), marker
diagnostics_source = (root / 'src/diagnostics.rs').read_text() + (root / 'src/system_stats/mod.rs').read_text() + (root / 'src/system_stats/linux.rs').read_text()
assert marker in rust_source(root, 'src/logging.rs'), marker
diagnostics_source = rust_source(root, 'src/diagnostics.rs') + rust_source(root, 'src/system_stats/mod.rs') + rust_source(root, 'src/system_stats/linux.rs')
for marker in ['process_rss_anon_bytes', 'process_rss_file_bytes', 'process_rss_shmem_bytes', 'process_data_bytes', 'process_swap_bytes', 'RssAnon', 'RssFile', 'RssShmem', 'VmData', 'VmSwap']:
assert marker in diagnostics_source, marker
prometheus_source = (root / 'src/prometheus.rs').read_text()
prometheus_source = rust_source(root, 'src/prometheus.rs')
for marker in ['allow_trusted_network_without_token', 'include_docker_vhosts', 'include_streams', 'include_logging', 'rust_proxy_docker_vhost_{name}', 'rust_proxy_docker_backend_{name}', 'rust_proxy_geoip_loaded', 'not_after_timestamp_seconds', 'ewma_latency_seconds', 'passive_failures', 'active_health_failures_total', 'passive_health_failures_total', 'passive_http_5xx_total', 'passive_timeouts_total', 'passive_connect_errors_total', 'passive_io_errors_total', 'passive_slow_total', 'http_5xx_total', 'timeouts_total', 'connect_errors_total', 'io_errors_total', 'slow_responses_total', 'rust_proxy_log_sink_queue_dropped_total', 'rust_proxy_not_found_hosts_configured']:
assert marker in prometheus_source, marker
assert 'if docker_targets.contains_key(&pool.id)' in prometheus_source
@@ -638,7 +649,7 @@ for marker in ['cache-memory-breakdown', 'cache-transient-breakdown', 'cache-col
for marker in ['validate_h3', 'H3Tunnel', 'h3_client_to_backend', 'backend_to_h3', 'rfc9220_bootstrap_uses_extended_connect_without_h1_proof_headers']:
assert marker in ws_source, marker
for marker in ['PrefixPriority', 'RegexInsensitive', 'LocationResponse', 'pub rewrite: PathRewrite']:
assert marker in (root / 'src/vhost.rs').read_text(), marker
assert marker in rust_source(root, 'src/vhost.rs'), marker
for marker in ['rewritten_path', 'nginx_style_location_precedence_exact_priority_regex_prefix', 'location_rewrites_are_normalized_and_regex_captures_expand', 'custom_location_response_is_validated']:
assert marker in config_source, marker
for marker in ['local_location_response', 'route.rewritten_path(&path)', 'encoded_origin_path', 'client_ip.to_string()']:
@@ -650,11 +661,11 @@ for marker in ['locationMatchRegex', 'headersVisualBuilder', 'headersAdvancedJso
assert marker in vhosts_js, marker
for marker in ['lastApiLatencyMs', 'cacheMemorySamples', 'cacheTransientSamples', 'cacheColdStorageSamples', 'cacheHitSamples', 'processCpuSamples', 'processRssSamples', 'networkRxSamples', 'responseLatencySamples', 'client_ip']:
assert marker in app_js, marker
logging_source = (root / 'src/logging.rs').read_text()
logging_source = rust_source(root, 'src/logging.rs')
assert 'pub client_ip: Option<String>' in logging_source and '"client_ip" => entry.client_ip' in logging_source
for marker in ['check_rustproxy_api', 'response time', 'backend']:
assert marker in (root / 'nagios/README.md').read_text(), marker
stream_source = (root / "src/stream.rs").read_text()
stream_source = rust_source(root, "src/stream.rs")
for marker in ["pub struct Metrics", "pub struct MetricsView", "struct ActiveSession", "sessions_total", "sessions_completed", "packets_in", "packets_out", "metrics_view_reports_l4_counters"]:
assert marker in stream_source, marker
transport_doc = (root / "docs/TRANSPORT.md").read_text()
@@ -663,10 +674,10 @@ api_access_doc = (root / "docs/API_ACCESS.md").read_text()
assert 'accept_local_listener_risk: true' in transport_doc and '127.0.0.1:9090' in transport_doc
assert 'accept_local_listener_risk=true' in security_doc and 'including port 9090' in security_doc
assert '7–30' in api_access_doc and 'default 30' in api_access_doc
not_found_source = (root / 'src/not_found.rs').read_text()
not_found_source = rust_source(root, 'src/not_found.rs')
for marker in ['pub struct Host', 'pub struct CompiledHost', 'default_status', 'crate::pages::Page']:
assert marker in not_found_source, marker
pages_source = (root / 'src/pages.rs').read_text()
pages_source = rust_source(root, 'src/pages.rs')
assert 'pub struct MinimalWelcome' in pages_source and 'HeaderValue::from_static("no-store")' in pages_source
assert 'builtin_welcome_is_minimal_and_never_cacheable' in pages_source
assert 'minimal_welcome' in proxy_source and 'cache-control' in proxy_source
@@ -694,15 +705,15 @@ expressions = '\n'.join(target.get('expr','') for panel in grafana['panels'] for
for marker in ['rust_proxy_process_rss_bytes', 'rust_proxy_cache_transient_bytes', 'rust_proxy_backend_latency_seconds', 'rust_proxy_stream_active', 'rust_proxy_log_sink_queue_dropped_total']:
assert marker in expressions, marker
# 0.19.0 compression/protocol/dashboard UX regressions.
compression_source = (root / "src/compression.rs").read_text()
metrics_source = (root / "src/metrics.rs").read_text()
admin_source = (root / "src/admin/mod.rs").read_text()
compression_source = rust_source(root, "src/compression.rs")
metrics_source = rust_source(root, "src/metrics.rs")
admin_source = rust_source(root, "src/admin/mod.rs")
style_source = (root / "src/admin/web/style.css").read_text()
docker_ui = (root / "src/admin/web/docker.js").read_text()
for marker in ["pub enum AppliedEncoding", "AppliedEncoding::Gzip", "AppliedEncoding::Brotli", "AppliedEncoding::Zstd", "AppliedEncoding::Deflate"]:
assert marker in compression_source, marker
assert "quality_weights_can_select_each_enabled_codec" in compression_source
diagnostics_source = (root / "src/diagnostics.rs").read_text() + (root / "src/system_stats/mod.rs").read_text() + (root / "src/system_stats/linux.rs").read_text()
diagnostics_source = rust_source(root, "src/diagnostics.rs") + rust_source(root, "src/system_stats/mod.rs") + rust_source(root, "src/system_stats/linux.rs")
for marker in ["pub struct NetworkInterfaceView", "pub network_interfaces: Vec<NetworkInterfaceView>", "sysinfo::Networks::new_with_refreshed_list", "system.network_stats", "collect_network_systemstat", "normalize_network_interfaces", "merge_network_interfaces"]:
assert marker in diagnostics_source, marker
assert "interfaces.truncate(64)" not in diagnostics_source, "network inventory must not silently truncate interfaces"
@@ -717,7 +728,7 @@ assert 'id="network-interface-charts"' in html
assert 'compressionCountersHelp' not in html
assert "dockerOpenApp" in docker_ui and "proxiedUrl(agent, hostname)" in docker_ui
# 0.19.0 Docker-vhost controls: chip aliases, default agent ACL, redirect/WebSockets/cache.
docker_source = (root / "src/docker.rs").read_text()
docker_source = rust_source(root, "src/docker.rs")
management_ui = (root / "src/admin/web/management.js").read_text()
for marker in ["function aliasEditor(", "host-chip-editor docker-alias-editor", ".docker-force-ssl", ".docker-websockets", ".docker-cache-enabled", ".docker-agent-acl-profile"]:
assert marker in docker_ui, marker
@@ -727,7 +738,7 @@ assert "agent.acl_profile_id === profile.id" in management_ui
assert 'payload["backend_pool_domains"]' in admin_source and "status.backend_pool_domains" in vhosts_js
assert 'body[data-density="compact"]' in style_source and '--ui-control: 22px' in style_source
assert 'rust_proxy_http1_requests_total' in expressions
vhost_source = (root / "src/vhost.rs").read_text()
vhost_source = rust_source(root, "src/vhost.rs")
for marker in ["DEFAULT_PROXY_CONNECT_TIMEOUT_SECONDS: u64 = 10", "DEFAULT_PROXY_RESPONSE_HEADER_TIMEOUT_SECONDS: u64 = 120", "DEFAULT_PROXY_READ_TIMEOUT_SECONDS: u64 = 60", "DEFAULT_PROXY_SEND_TIMEOUT_SECONDS: u64 = 60", "proxy_timeouts_have_explicit_safe_defaults_but_keep_null_compatibility"]:
assert marker in vhost_source, marker
assert 'defaultProxyTimeouts = () => ({ connect_timeout_seconds: 10, response_header_timeout_seconds: 120, read_timeout_seconds: 60, send_timeout_seconds: 60 })' in vhosts_js
@@ -764,7 +775,7 @@ assert 'pub transformed: bool' in cache_source and 'annotate_transformed' in cac
assert 'x-cache-status' in cache_source and 'Cache-Status (RFC 9211)' in html
assert 'generated_vhost_inherits_global_server_header_policy' in docker_source and 'effective.vhosts[0].server_header.is_none()' in docker_source
assert '.insert("x-rustproxy-backend-time", value)' in proxy_source and '.insert("x-rust-proxy-backend-time", value)' not in proxy_source
assert 'x-rustproxy-backend-time' in (root / "src/headers.rs").read_text()
assert 'x-rustproxy-backend-time' in rust_source(root, "src/headers.rs")
# Expanded upstream connection diagnostics must retain nested Hyper/rustls/io causes.
for marker in [
"fn format_error_chain(",
@@ -802,7 +813,7 @@ assert '#[serde(default = "default_admin_theme")]' in admin_source and 'matches!
assert 'bindSettingsForm("pages-form", next => { next.pages = global.read(); }, "pages")' in management_ui
assert '"pages" => {' in admin_source and 'config.pages =' in admin_source
assert 'default_pages_section_accepts_the_standard_value_payload' in admin_source
logging_source = (root / "src/logging.rs").read_text()
logging_source = rust_source(root, "src/logging.rs")
assert 'pub const LIVE_WINDOW: usize = 256;' in logging_source
assert 'pub const HISTORY_CAPACITY: usize = 4096;' in logging_source
assert 'retained_history_is_larger_than_the_live_snapshot_but_still_bounded' in logging_source
@@ -819,18 +830,18 @@ traffic_panel_js = (root / "src/admin/web/traffic-panel.js").read_text()
assert 'id="panel-traffic"' in html and 'data-panel="traffic"' in html
for marker in ['/api/traffic', 'trafficTopCountries', 'trafficTopUserAgents', 'trafficTopClientIps', 'traffic-donut', 'rustProxyTrafficPanel']:
assert marker in traffic_panel_js or marker in html, marker
traffic_source = (root / "src/traffic.rs").read_text()
traffic_source = rust_source(root, "src/traffic.rs")
for marker in ['QUEUE_CAPACITY: usize = 2048', 'TOP_CAPACITY: usize = 32', 'MAX_VHOSTS: usize = 64', 'self.tx.try_send(sample)', 'client_ips: TopTable']:
assert marker in traffic_source, marker
assert 'if (envelope) render();' in (root / "src/admin/web/web-server.js").read_text()
assert 'pub capture_user_agent: bool' in logging_source and 'pub capture_country: bool' in logging_source
assert 'spawn_blocking(crate::diagnostics::trim_allocator)' in (root / 'src/runtime.rs').read_text()
assert 'spawn_blocking(crate::diagnostics::trim_allocator)' in rust_source(root, 'src/runtime.rs')
assert 'fn trim_hot_idle(&self)' in cache_source and 'hot_auto_trim_target_percent' in cache_source
assert 'id="cache-domain-filter"' in html and 'cacheDomainFilterEnabled' in html and 'cacheDomainFilterDisabled' in html
assert 'const visibleDomains = (data, groups) =>' in cache_panel_js and 'scoped?.settings?.enabled ?? data.settings?.enabled ?? false' in cache_panel_js
assert 'groups = d.domains || [], shownGroups = visibleDomains(data, groups)' in cache_panel_js
assert 'replaceChildren(...shownGroups.map(row =>' in cache_panel_js
balance_source = (root / 'src/balance.rs').read_text()
balance_source = rust_source(root, 'src/balance.rs')
assert 'traffic_rate_drops_to_zero_after_an_idle_sample_window' in balance_source and 'if delta == 0 {' in balance_source and 'sample.rate = 0.0;' in balance_source
all_admin_js = "\n".join(path.read_text() for path in (root / "src/admin/web").glob("*.js"))
assert not re.search(r'(?<![A-Za-z])(?:window\.)?(?:alert|confirm)\s*\(', all_admin_js), "native browser alert/confirm returned"
@@ -854,7 +865,7 @@ assert 'form.reset(); window.rustProxyCertificates?.close();' in integrations_ui
assert '$("cert-import-pem").value = ""' not in integrations_ui_0194 and '$("cert-import-key").value = ""' not in integrations_ui_0194
assert 'throw fieldError(row.querySelector(".docker-agent-listener-choices")' in docker_ui_0194
assert '#docker-agents-form>.form-actions {' in style_source and 'justify-content: flex-start' in style_source
deploy_source = (root / "scripts/deploy.py").read_text()
deploy_source = read_deploy_source()
assert "SHELL_PATH_PROFILE = Path('/etc/profile.d/rustproxy-path.sh')" in deploy_source
assert 'install_shell_path(prefix)' in deploy_source and 'check_shell_path(prefix)' in deploy_source and "'shell_path_action':'install' if args.mode == 'install' else 'check-only'" in deploy_source
assert 'grid-template-columns: minmax(0, 1fr)' in style_source and '.log-column-option:hover' in style_source
@@ -874,7 +885,7 @@ assert '<span class="version">' not in html, "duplicate top-right version label
assert '<div class="edition">INDEPENDENT BY DESIGN <span>' not in html, "release version returned beside the dashboard brand"
assert '<div class="edition">INDEPENDENT BY DESIGN</div>' in html
# tls_verify=false must not feed legacy appliance certificates back through webpki.
upstream_source = (root / "src/upstream.rs").read_text(encoding="utf-8")
upstream_source = rust_source(root, "src/upstream.rs")
assert "BackendCertificateVerificationDisabled { supported_schemes }" in upstream_source
disabled_verifier = upstream_source.split("impl ServerCertVerifier for BackendCertificateVerificationDisabled", 1)[1].split("pub fn client(", 1)[0]
assert disabled_verifier.count("Ok(HandshakeSignatureValid::assertion())") == 2
@@ -882,9 +893,9 @@ assert "WebPkiServerVerifier" not in disabled_verifier
assert '.docker-route-grid {' in style_source and 'align-items: start' in style_source and '.docker-acl-grid {' in style_source
# 0.22.3 cache-modal, RFC8441/h2c and active-tunnel contracts.
server_source = (root / "src/server.rs").read_text()
websocket_source = (root / "src/proxy/websocket.rs").read_text()
vhost_source = (root / "src/vhost.rs").read_text()
server_source = rust_source(root, "src/server.rs")
websocket_source = rust_source(root, "src/proxy/websocket.rs")
vhost_source = rust_source(root, "src/vhost.rs")
integrations_0221 = (root / "src/admin/web/integrations.js").read_text()
http2_test = (root / "tests/http2.rs").read_text()
transport_test = (root / "tests/transport_config.rs").read_text()
@@ -940,7 +951,7 @@ for memory_marker_0230 in [
'http2_active_connections',
]:
assert memory_marker_0230 in server_source or memory_marker_0230 in metrics_source, memory_marker_0230
cache_fill_0230 = (root / 'src/cache/fill.rs').read_text()
cache_fill_0230 = rust_source(root, 'src/cache/fill.rs')
assert 'let body_allocation = self.data.capacity().max(self.data.len());' in cache_fill_0230
assert 'let cost = hot_entry_cost(' in cache_fill_0230
assert 'fn hot_entry_cost(' in cache_source
@@ -1008,7 +1019,7 @@ assert 'prior knowledge' in transport_doc and 'Upgrade: h2c' in transport_doc an
assert (root / "docs/V0.24.1_AUDIT.md").is_file() and (root / "docs/V0.24.0_AUDIT.md").is_file() and (root / "docs/V0.23.2_AUDIT.md").is_file() and (root / "docs/V0.23.1_AUDIT.md").is_file() and (root / "docs/V0.23.0_AUDIT.md").is_file() and (root / "docs/V0.22.3_AUDIT.md").is_file()
# 0.22.3 replay-safe retry and ordinary H2 upstream contracts.
balance_0222 = (root / "src/balance.rs").read_text()
balance_0222 = rust_source(root, "src/balance.rs")
for marker_0222 in [
"pub struct RetryPolicy", "pub max_retries: u8", "pub retry_statuses: Vec<u16>",
"pub fn select_retry", "retry_attempts_total", "retry_successes_total",
@@ -1022,9 +1033,9 @@ for marker_0222 in [
"explicit_http2_backend_uses_http2_request_version",
]:
assert marker_0222 in proxy_source, marker_0222
upstream_0222 = (root / "src/upstream.rs").read_text()
upstream_0222 = rust_source(root, "src/upstream.rs")
assert "explicit_h2c_backend_uses_http2_prior_knowledge" in upstream_0222
vhosts_0222 = (root / "src/admin/web/vhosts.js").read_text()
vhosts_0222 = read_vhosts_js(root)
for marker_0222 in ["retry-max", "retry-connect", "retry-timeouts", "retry-statuses"]:
assert marker_0222 in vhosts_0222, marker_0222
for marker_0222 in [
@@ -1042,7 +1053,7 @@ assert 'json!({"id":m.definition.id, "url":m.definition.url' not in balance_0222
for marker_0222 in ["member-max-concurrent", "member-queue-timeout", "vhLastFailure", "vhLastRecovery"]:
assert marker_0222 in vhosts_0222 or marker_0222 in html, marker_0222
for marker_0222 in ["CertificateMetadata", "certificate_public_metadata", "x509_time_parser_handles_utc_and_generalized_time_with_offsets"]:
assert marker_0222 in (root / "src/tls.rs").read_text(), marker_0222
assert marker_0222 in rust_source(root, "src/tls.rs"), marker_0222
for marker_0222 in [
'"max_concurrent_requests"', '"queue_timeout_ms"', '"certificate_issuer"',
'"days_remaining"', '"expiry_status"', '"last_renewal_error"'
@@ -1072,13 +1083,13 @@ for marker_0223 in [
'"operation": "unchanged"', 'config.tls.certificates.push(crate::tls::Certificate',
]:
assert marker_0223 in admin, marker_0223
runtime_0223 = (root / 'src/runtime.rs').read_text()
runtime_0223 = rust_source(root, 'src/runtime.rs')
for marker_0223 in ['pub async fn reload_certificates', 'configured certificates reloaded from disk']:
assert marker_0223 in runtime_0223, marker_0223
main_0223 = (root / 'src/main.rs').read_text()
main_0223 = rust_source(root, 'src/main.rs')
for marker_0223 in ['TlsCommand::Deploy', '/api/certificates/deploy', '/api/certificates/reload', 'read_external_pem']:
assert marker_0223 in main_0223, marker_0223
tls_0223 = (root / 'src/tls.rs').read_text()
tls_0223 = rust_source(root, 'src/tls.rs')
for marker_0223 in ['pub fn stage_external_pair', 'pub fn reconcile_external_pair', 'ExternalPairDeployment::Unchanged', 'external-', 'remove_external_pair']:
assert marker_0223 in tls_0223, marker_0223
for marker_0223 in ['"/api/certificates/reload"', '"CertificateReloadResult"', '"/api/certificates/deploy"', '"CertificateDeploy"', '"CertificateDeployResult"']:
@@ -1087,9 +1098,9 @@ assert 'api("/api/certificates/reload"' in app_js
assert 'certificateAdminRestartRequired' in app_js
# 0.24.0 H2 lifecycle contracts retained by 0.24.1.
server_0240 = (root / "src/server.rs").read_text()
settings_0240 = (root / "src/settings.rs").read_text()
config_0240 = (root / "src/config.rs").read_text()
server_0240 = rust_source(root, "src/server.rs")
settings_0240 = rust_source(root, "src/settings.rs")
config_0240 = rust_source(root, "src/config.rs")
for marker_0240 in [
"H2ConnectionState", "H2StreamLease", "H2TrackedBody",
"wait_for_h2_idle", "graceful_shutdown()", "H2ShutdownCause::Lifetime",
@@ -1101,16 +1112,16 @@ for marker_0240 in [
for marker_0240 in ["Http2Connections", "idle_timeout_seconds", "max_connection_lifetime_seconds", "drain_timeout_seconds"]:
assert marker_0240 in settings_0240, marker_0240
assert "pub http2: crate::settings::Http2Connections" in config_0240
assert "pub http2_settings: Option<crate::settings::Http2Connections>" in (root / "src/vhost.rs").read_text()
assert "pub http2_settings: Option<crate::settings::Http2Connections>" in rust_source(root, "src/vhost.rs")
assert "serve_frontend_with_http2" in main_source and "binding.http2_settings.unwrap_or_default()" in main_source
assert 'renderHttp2Settings' in integrations_js and 'collectHttp2Listeners' in integrations_js
assert '"Http2ConnectionSettings"' in openapi and '"max_connection_lifetime_seconds"' in openapi
# 0.24.1 shared statistics/runtime-info/settings-navigation contracts.
system_stats_0241 = (root / "src/system_stats/mod.rs").read_text()
linux_stats_0241 = (root / "src/system_stats/linux.rs").read_text()
main_0241 = (root / "src/main.rs").read_text()
admin_0241 = (root / "src/admin/mod.rs").read_text()
system_stats_0241 = rust_source(root, "src/system_stats/mod.rs")
linux_stats_0241 = rust_source(root, "src/system_stats/linux.rs")
main_0241 = rust_source(root, "src/main.rs")
admin_0241 = rust_source(root, "src/admin/mod.rs")
settings_js_0241 = (root / "src/admin/web/settings.js").read_text()
assert not (root / "src/system_stats/bsd.rs").exists(), "per-BSD parser backend must not return"
assert manifest["target"]['cfg(any(target_os = "linux", target_os = "freebsd", target_os = "openbsd", target_os = "netbsd", target_os = "dragonfly", target_os = "macos", target_os = "windows"))']["dependencies"]["systemstat"] == "=0.2.7"
@@ -1145,14 +1156,14 @@ assert '"error_log" => {' not in admin_0241 and 'request("/api/config/error_log"
assert '"error_log"' in openapi and 'logging, error_log, forwarding' not in openapi
assert 'logging_section_persists_error_log_settings_atomically' in admin_0241 and 'error_log_is_not_a_standalone_configuration_section' in admin_0241
assert 'location.pathname === "/logging"' in settings_js_0241
assert '"RuntimeInformation"' in openapi and '"version": "0.25.0"' in openapi
assert '"RuntimeInformation"' in openapi and '"version": "0.25.1"' in openapi
# 0.24.1 compiler and hostname-cardinality regressions.
tls_0241 = (root / "src/tls.rs").read_text()
vhost_0241 = (root / "src/vhost.rs").read_text()
redirect_0241 = (root / "src/redirect.rs").read_text()
not_found_0241 = (root / "src/not_found.rs").read_text()
docker_0241 = (root / "src/docker.rs").read_text()
tls_0241 = rust_source(root, "src/tls.rs")
vhost_0241 = rust_source(root, "src/vhost.rs")
redirect_0241 = rust_source(root, "src/redirect.rs")
not_found_0241 = rust_source(root, "src/not_found.rs")
docker_0241 = rust_source(root, "src/docker.rs")
integrations_0241 = (root / "src/admin/web/integrations.js").read_text()
docker_js_0241 = (root / "src/admin/web/docker.js").read_text()
assert "future::Future" not in server_0240
@@ -1190,5 +1201,5 @@ for schema_name in ["RedirectHost", "NotFoundHost"]:
node = openapi_0241["components"]["schemas"][schema_name]["properties"]["frontend"]["properties"]["hosts"]
assert "maxItems" not in node, schema_name
print("PASS: 0.25.0 manifest/package, shared system-statistics/runtime metadata, Settings logging, retained H2, Debian/Grafana/Prometheus/404-host/custom-location/path/cache/dashboard/Nagios UI/source contracts, stopped legacy-renewal removal, session/admin/transport contracts, deterministic package source and embedded SQLite schema-v7 migrations.")
print("PASS: 0.25.1 manifest/package, shared system-statistics/runtime metadata, Settings logging, retained H2, Debian/Grafana/Prometheus/404-host/custom-location/path/cache/dashboard/Nagios UI/source contracts, stopped legacy-renewal removal, session/admin/transport contracts, deterministic package source and embedded SQLite schema-v7 migrations.")
print("NOT VERIFIED: Rust parsing, types, compilation, linking or runtime behavior. Run ./dev.sh test.")
+4 -699
View File
@@ -1,705 +1,9 @@
#!/usr/bin/env python3
"""Install/update a local release binary on Linux/systemd. No builds.
First install downloads the PDDL user-country GeoIP CSV files from the
sapics/ip-location-db GitHub releases and verifies publisher SHA-256 files.
Updates compare the installed GeoIP files with current publisher checksums and
stage only stale assets before service downtime. Updates stop the service before
copying all private data, refresh the managed systemd unit, switch an immutable
release symlink, check health/version and restore code, data and unit on failure.
Use --plan to inspect operations without changing the host.
"""
"""Install/update a local release binary on Linux/systemd. No builds."""
from __future__ import annotations
import argparse
import fcntl
import hashlib
import http.client
import ipaddress
import json
import os
from pathlib import Path
import pwd
import re
import shutil
import sqlite3
import ssl
import stat
import subprocess
import sys
import time
import uuid
import urllib.request
import urllib.parse
ROOT = Path(__file__).resolve().parents[1]
UNIT_PATH = Path('/etc/systemd/system/rustproxy.service')
SHELL_PATH_PROFILE = Path('/etc/profile.d/rustproxy-path.sh')
SERVICE = 'rustproxy.service'
GEOIP_ASSETS = (
'user-country-ipv4.csv',
'user-country-ipv6.csv',
)
GEOIP_RELEASE = 'https://github.com/sapics/ip-location-db/releases/download/latest/'
GEOIP_CHECKSUM = 'https://github.com/sapics/ip-location-db/releases/download/checksum/'
GEOIP_MAX_BYTES = 256 * 1024 * 1024
def run(args: list[str], **kwargs) -> subprocess.CompletedProcess:
return subprocess.run(args, check=True, stdin=subprocess.DEVNULL, **kwargs)
def safe_path(raw: str) -> Path:
if not re.fullmatch(r'/[A-Za-z0-9_./-]+', raw) or '..' in raw.split('/') or raw in ('/', '/etc', '/var', '/opt', '/usr', '/var/lib'):
raise ValueError('Use a dedicated absolute path with no spaces, traversal or shell characters.')
p = Path(raw)
for part in (p, *p.parents):
if part.is_symlink():
raise ValueError(f'Symlinked installation/data path is not allowed: {part}')
return p
def address(raw: str, loopback: bool = False, allow_remote: bool = False) -> str:
if raw.startswith('['):
host, port = raw[1:].split(']:', 1)
else:
host, port = raw.rsplit(':', 1)
ip = ipaddress.ip_address(host)
if not port.isdecimal() or not 1 <= int(port) <= 65535:
raise ValueError('Invalid listener IP/port.')
if loopback and not ip.is_loopback and not allow_remote:
raise ValueError('Administration must use loopback unless remote administration is explicitly accepted.')
return f'[{ip}]:{int(port)}' if ip.version == 6 else f'{ip}:{int(port)}'
def listener_ip(raw: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address:
host = raw[1:].split(']:', 1)[0] if raw.startswith('[') else raw.rsplit(':', 1)[0]
return ipaddress.ip_address(host)
def admin_address(raw: str, allow_remote: bool = False) -> tuple[str, bool]:
value = address(raw)
ip = listener_ip(value)
remote = not ip.is_loopback
if remote and not allow_remote:
raise ValueError('Remote administration requires --allow-remote-admin.')
return value, remote
def digest(path: Path) -> str:
with path.open('rb') as stream:
result = hashlib.sha256()
for block in iter(lambda: stream.read(1024 * 1024), b''):
result.update(block)
return result.hexdigest()
def fetch_https(url: str, max_bytes: int) -> bytes:
if urllib.parse.urlparse(url).scheme != 'https':
raise ValueError('GeoIP downloads require HTTPS.')
request = urllib.request.Request(url, headers={'User-Agent': 'RustProxy-installer/0.25.0'})
with urllib.request.urlopen(request, timeout=30) as response:
if urllib.parse.urlparse(response.geturl()).scheme != 'https':
raise ValueError('GeoIP download redirected away from HTTPS.')
length = response.headers.get('Content-Length')
if length and int(length) > max_bytes:
raise ValueError('GeoIP download is larger than the configured safety limit.')
result = bytearray()
while True:
block = response.read(min(1024 * 1024, max_bytes + 1 - len(result)))
if not block:
break
result.extend(block)
if len(result) > max_bytes:
raise ValueError('GeoIP download exceeded the configured safety limit.')
return bytes(result)
def publisher_geoip_checksums() -> dict[str, str]:
expected: dict[str, str] = {}
for name in GEOIP_ASSETS:
checksum = fetch_https(GEOIP_CHECKSUM + name + '.sha256', 4096).decode('ascii', 'strict').strip()
matched = re.search(r'\b([0-9a-fA-F]{64})\b', checksum)
if not matched:
raise ValueError(f'Invalid publisher checksum for {name}.')
expected[name] = matched.group(1).lower()
return expected
def geoip_file_digest(path: Path) -> str | None:
try:
st = path.lstat()
except FileNotFoundError:
return None
if not stat.S_ISREG(st.st_mode) or path.is_symlink():
return None
return digest(path)
def prepare_geoip_update(data: Path, prefix: Path) -> tuple[Path | None, list[str]]:
"""Verify local GeoIP files against current publisher checksums.
Network/download work happens before the service is stopped. Changed files are
staged under the code prefix and are copied into the private data transaction
only after its backup exists.
"""
expected = publisher_geoip_checksums()
directory = data / 'geoip'
if directory.exists() and directory.is_symlink():
raise ValueError('GeoIP directory cannot be a symlink.')
stale = [name for name in GEOIP_ASSETS if geoip_file_digest(directory / name) != expected[name]]
if not stale:
print('GeoIP database is current (publisher SHA-256 matches).')
return None, []
staging = prefix / ('.geoip-stage-' + uuid.uuid4().hex)
staging.mkdir(mode=0o700)
try:
for name in stale:
payload = fetch_https(GEOIP_RELEASE + name, GEOIP_MAX_BYTES)
actual = hashlib.sha256(payload).hexdigest()
if actual != expected[name]:
raise ValueError(f'GeoIP SHA-256 mismatch for {name}.')
write_private(staging / name, payload, 0o600)
print('GeoIP update prepared: ' + ', '.join(stale))
return staging, stale
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
def copy_private(source: Path, target: Path, mode: int = 0o600) -> None:
temporary = target.with_name(target.name + '.' + uuid.uuid4().hex + '.tmp')
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
try:
with source.open('rb') as input_stream, os.fdopen(fd, 'wb') as output_stream:
shutil.copyfileobj(input_stream, output_stream, 1024 * 1024)
output_stream.flush(); os.fsync(output_stream.fileno())
os.replace(temporary, target)
directory = os.open(target.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
temporary.unlink(missing_ok=True)
def apply_geoip_update(data: Path, staging: Path | None, names: list[str], uid: int, gid: int) -> None:
if not names:
return
directory = data / 'geoip'
if directory.exists() and directory.is_symlink():
raise ValueError('GeoIP directory cannot be a symlink.')
directory.mkdir(mode=0o700, exist_ok=True)
os.chmod(directory, 0o700)
os.chown(directory, uid, gid)
if staging is None:
raise ValueError('GeoIP staging directory is missing.')
for name in names:
source = staging / name
target = directory / name
copy_private(source, target, 0o600)
os.chown(target, uid, gid)
print('GeoIP database updated: ' + ', '.join(names))
def download_geoip(data: Path, uid: int, gid: int) -> None:
# First install has no code-prefix staging transaction, so download and verify
# each asset directly before publishing it atomically into the private data dir.
expected = publisher_geoip_checksums()
directory = data / 'geoip'
if directory.exists() and directory.is_symlink():
raise ValueError('GeoIP directory cannot be a symlink.')
directory.mkdir(mode=0o700, exist_ok=True)
os.chmod(directory, 0o700)
os.chown(directory, uid, gid)
for name in GEOIP_ASSETS:
payload = fetch_https(GEOIP_RELEASE + name, GEOIP_MAX_BYTES)
actual = hashlib.sha256(payload).hexdigest()
if actual != expected[name]:
raise ValueError(f'GeoIP SHA-256 mismatch for {name}.')
target = directory / name
write_private(target, payload, 0o600)
os.chown(target, uid, gid)
def write_private(path: Path, value: bytes, mode: int = 0o600) -> None:
temporary = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
try:
with os.fdopen(fd, 'wb') as stream:
stream.write(value); stream.flush(); os.fsync(stream.fileno())
os.replace(temporary, path)
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
temporary.unlink(missing_ok=True)
def shell_path_profile(prefix: Path) -> bytes:
binary_dir = prefix / 'current'
return (
'# Managed by the RustProxy installer. New login shells can invoke rust-proxy directly.\n'
'case ":${PATH:-}:" in\n'
f' *":{binary_dir}:"*) ;;\n'
f' *) export PATH="{binary_dir}${{PATH:+:$PATH}}" ;;\n'
'esac\n'
).encode()
def install_shell_path(prefix: Path) -> None:
write_private(SHELL_PATH_PROFILE, shell_path_profile(prefix), 0o644)
def check_shell_path(prefix: Path) -> bool:
expected = shell_path_profile(prefix)
try:
st = SHELL_PATH_PROFILE.lstat()
valid = stat.S_ISREG(st.st_mode) and st.st_uid == 0 and not (st.st_mode & 0o022) and SHELL_PATH_PROFILE.read_bytes() == expected
except OSError:
valid = False
if not valid:
print(
f'WARNING: shell PATH setup is missing or outdated: {SHELL_PATH_PROFILE}. '
f'Expected {prefix}/current in PATH; update mode only checks this file and does not modify it.',
file=sys.stderr,
)
return valid
def read_record(prefix: Path) -> dict:
path = prefix / 'install.json'
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
with os.fdopen(fd, 'rb') as stream:
st = os.fstat(stream.fileno())
if not stat.S_ISREG(st.st_mode) or st.st_uid != 0 or st.st_mode & 0o077 or st.st_size > 16384:
raise ValueError('Unsafe or invalid root-owned installation record.')
value = json.load(stream)
if value.get('schema') != 1 or value.get('prefix') != str(prefix):
raise ValueError('Installation record mismatch.')
safe_path(value['data_dir']); validate_user(value['user'])
remote_admin = bool(value.get('remote_admin', False))
admin, checked_remote = admin_address(value['admin'], remote_admin)
if checked_remote != remote_admin:
raise ValueError('Installation record remote administration flag does not match its address.')
value['admin'] = admin
value['remote_admin'] = remote_admin
return value
def validate_user(name: str) -> None:
if not re.fullmatch(r'[a-z_][a-z0-9_-]{0,30}', name) or name in ('root', 'nobody'):
raise ValueError('Use a dedicated non-root service account.')
def service_unit(record: dict) -> str:
prefix, data, user = record['prefix'], record['data_dir'], record['user']
args = f'{prefix}/current/rust-proxy --data-dir {data} --admin-listen {record["admin"]} serve'
args += f' --proxy-listen {record["http"]}'
if record.get('https'):
args += f' --tls-listen {record["https"]}'
if record.get('remote_admin'):
args += ' --allow-remote-admin'
return f'''[Unit]
Description=RustProxy reverse proxy
Wants=network-online.target
After=network-online.target
[Service]
Type=simple
User={user}
Group={record['group']}
WorkingDirectory={data}
UMask=0077
Environment=RUST_LOG=rust_proxy=info,warn
Environment=MALLOC_ARENA_MAX=2
Environment=PATH=/opt/rustproxy-certbot/bin:/usr/local/bin:/usr/bin:/bin
EnvironmentFile=-/etc/rustproxy/environment
ExecStart={args}
Restart=on-failure
RestartSec=3
KillSignal=SIGINT
KillMode=control-group
TimeoutStopSec=120
AmbientCapabilities=CAP_NET_BIND_SERVICE
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectSystem=strict
LogsDirectory=rustproxy
LogsDirectoryMode=0770
ReadWritePaths={data} /var/log/rustproxy
ProtectHome=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectKernelLogs=true
ProtectControlGroups=true
RestrictSUIDSGID=true
RestrictNamespaces=true
RestrictRealtime=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
SystemCallArchitectures=native
LockPersonality=true
[Install]
WantedBy=multi-user.target
'''
def checked_tree(source: Path) -> None:
"""Accept only regular files/directories and relative Certbot-internal symlinks."""
boundary = source.resolve(strict=True)
if source.is_symlink():
raise ValueError('Data root cannot be a symlink.')
for p in source.rglob('*'):
st = p.lstat()
if stat.S_ISLNK(st.st_mode):
target = os.readlink(p)
if os.path.isabs(target) or not p.resolve(strict=True).is_relative_to(boundary):
raise ValueError('Data backup contains an external or absolute symlink.')
elif not (stat.S_ISDIR(st.st_mode) or stat.S_ISREG(st.st_mode)):
raise ValueError('Data backup contains a special file.')
def copy_data(source: Path, target: Path) -> None:
checked_tree(source)
shutil.copytree(source, target, symlinks=True)
os.chmod(target, 0o700)
def chown_tree(path: Path, uid: int, gid: int) -> None:
os.chown(path, uid, gid)
for child in path.rglob('*'):
os.chown(child, uid, gid, follow_symlinks=False)
def ensure_service_data_access(data: Path, account) -> None:
"""Repair the managed data-root metadata and verify service-user traversal.
systemd applies WorkingDirectory after dropping to User=/Group=. A stale
root-owned 0700 data directory therefore fails as status=200/CHDIR before
RustProxy itself can emit diagnostics. Repair only the managed data root;
never chmod/chown parent directories. Then fail before service downtime if a
parent directory still prevents traversal.
"""
try:
st = data.lstat()
except FileNotFoundError as error:
raise ValueError(f'Data directory is missing: {data}') from error
if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode):
raise ValueError(f'Data directory must be a real directory: {data}')
os.chown(data, account.pw_uid, account.pw_gid)
os.chmod(data, 0o700)
probe = subprocess.run(
['runuser', '-u', account.pw_name, '--', 'test', '-x', str(data)],
check=False, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
if probe.returncode != 0:
raise ValueError(
f'Service user {account.pw_name} cannot traverse data directory {data}; '
'check execute/search permissions on its parent directories (for example with namei -l).'
)
def switch(prefix: Path, release: str) -> None:
if not re.fullmatch(r'[A-Za-z0-9_.-]+', release) or '..' in release:
raise ValueError('Invalid release name.')
temporary = prefix / ('current-' + uuid.uuid4().hex)
os.symlink('releases/' + release, temporary)
os.replace(temporary, prefix / 'current')
def running_release_matches(record: dict) -> bool:
"""Check the actual systemd process executable without publishing a version API.
This installer targets Linux/systemd. Fail closed if /proc or MainPID cannot
establish that the service runs the exact staged, version-checked binary.
"""
try:
result = subprocess.run(['systemctl', 'show', '--property=MainPID', '--value', SERVICE],
check=False, capture_output=True, text=True, timeout=3)
if result.returncode != 0 or not result.stdout.strip().isdigit():
return False
pid = int(result.stdout.strip())
if pid <= 1:
return False
expected = Path(record['prefix']) / 'releases' / record['release'] / 'rust-proxy'
return Path(f'/proc/{pid}/exe').samefile(expected)
except (OSError, ValueError, KeyError, subprocess.SubprocessError):
return False
def admin_health_endpoint(record: dict) -> tuple[str, bool, str]:
"""Resolve the active persisted administration listener and scheme.
The installer record contains only the startup CLI address. Since 0.9.x the
dashboard can persist both a different admin bind and HTTPS in config.sqlite3.
Update health checks must follow that active intent or a healthy HTTPS panel is
mistaken for a failed release and rollback can fail for the same reason.
"""
raw = record['admin']
tls = False
allowed_hosts: list[str] = []
database = Path(record['data_dir']) / 'config.sqlite3'
try:
connection = sqlite3.connect(f'file:{database}?mode=ro', uri=True, timeout=2)
try:
row = connection.execute('SELECT document FROM configuration WHERE id = 1').fetchone()
finally:
connection.close()
if row:
document = json.loads(row[0])
administration = document.get('administration') or {}
if isinstance(administration, dict):
if administration.get('listen'):
raw = address(str(administration['listen']))
tls = bool(administration.get('tls', False))
values = administration.get('allowed_hosts') or []
if isinstance(values, list):
allowed_hosts = [str(value).strip() for value in values if str(value).strip()]
except (OSError, sqlite3.Error, ValueError, TypeError, json.JSONDecodeError):
# Old releases and damaged/uninitialized databases used the startup HTTP
# listener. The real startup diagnostics remain authoritative if this
# compatibility read cannot resolve persisted settings.
pass
host, port = (raw[1:].split(']:') if raw.startswith('[') else raw.rsplit(':', 1))
ip = ipaddress.ip_address(host)
connect_host = ('::1' if ip.version == 6 else '127.0.0.1') if ip.is_unspecified else host
authority_host = allowed_hosts[0] if allowed_hosts else connect_host
try:
authority_ip = ipaddress.ip_address(authority_host)
authority = f'[{authority_ip}]:{port}' if authority_ip.version == 6 else f'{authority_ip}:{port}'
except ValueError:
authority = f'{authority_host}:{port}'
connect_raw = f'[{connect_host}]:{port}' if ':' in connect_host else f'{connect_host}:{port}'
return connect_raw, tls, authority
def healthy(record: dict, version: str, attempts: int = 30) -> bool:
raw, tls, health_host = admin_health_endpoint(record)
host, port = (raw[1:].split(']:') if raw.startswith('[') else raw.rsplit(':', 1))
for _ in range(attempts):
if tls:
# Local deployment health is process identity/liveness, not PKI validation.
# The configured admin certificate may intentionally be self-signed or for
# a DNS name while we connect to the local socket.
conn = http.client.HTTPSConnection(host, int(port), timeout=2, context=ssl._create_unverified_context())
else:
conn = http.client.HTTPConnection(host, int(port), timeout=2)
try:
conn.request('GET', '/healthz', headers={'Host': health_host})
response = conn.getresponse(); content = response.read(4097)
active = subprocess.run(['systemctl', 'is-active', '--quiet', SERVICE], check=False, stdout=subprocess.DEVNULL).returncode == 0
if (active and response.status == 200 and len(content) <= 4096
and json.loads(content).get('status') == 'ok' and record.get('version') == version
and running_release_matches(record)):
return True
except (OSError, ValueError, http.client.HTTPException, ssl.SSLError):
pass
finally:
conn.close()
time.sleep(1)
return False
def stage(binary: Path, prefix: Path, user: str, sha: str) -> tuple[str, str]:
release = 'build-' + sha[:24]
releases = prefix / 'releases'; releases.mkdir(mode=0o755, exist_ok=True); os.chmod(releases, 0o755)
directory = releases / release
if not directory.exists():
directory.mkdir(mode=0o755); os.chmod(directory, 0o755)
try:
shutil.copyfile(binary, directory / 'rust-proxy')
os.chmod(directory / 'rust-proxy', 0o755)
except BaseException:
shutil.rmtree(directory); raise
if directory.is_symlink() or (directory / 'rust-proxy').is_symlink() or digest(directory / 'rust-proxy') != sha:
raise ValueError('Staged binary checksum mismatch.')
# Never execute the supplied binary as root, even for its version banner.
result = run(['runuser', '-u', user, '--', str(directory / 'rust-proxy'), '--version'], capture_output=True, text=True, timeout=15)
matched = re.fullmatch(r'(?:rust-proxy|RustProxy) (\d+\.\d+\.\d+(?:[-+][\w.-]+)?)\s*', result.stdout)
if not matched:
raise ValueError('Binary did not return a RustProxy version.')
return release, matched[1]
def backup_stopped(record: dict, prefix: Path) -> Path:
# Service stop must happen before this call: ACME files and SQLite form one backup.
data = Path(record['data_dir'])
lock_fd = os.open(data / 'process.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
try:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
account = pwd.getpwnam(record['user']); os.fchown(lock_fd, account.pw_uid, account.pw_gid)
backups = prefix / 'backups'; backups.mkdir(mode=0o700, exist_ok=True)
backup = backups / ('before-' + record['version'] + '-' + uuid.uuid4().hex[:12]); backup.mkdir(mode=0o700)
copy_data(data, backup / 'data')
write_private(backup / 'install.json', json.dumps(record, indent=2).encode())
if UNIT_PATH.exists():
shutil.copyfile(UNIT_PATH, backup / 'rustproxy.service'); os.chmod(backup / 'rustproxy.service', 0o600)
return backup
finally:
os.close(lock_fd)
def restore_service_unit(backup: Path) -> None:
saved = backup / 'rustproxy.service'
if saved.exists():
copy_private(saved, UNIT_PATH, 0o644)
else:
UNIT_PATH.unlink(missing_ok=True)
run(['systemctl', 'daemon-reload'])
def reset_saved_config(data: Path) -> None:
"""Development-only reset of desired config while preserving auth/certs/audit data."""
database = data / 'config.sqlite3'
if database.is_symlink() or not database.is_file():
raise ValueError('Cannot reset configuration: config.sqlite3 is missing or unsafe.')
connection = sqlite3.connect(f'file:{database}?mode=rw', uri=True, timeout=5)
try:
row = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='configuration'").fetchone()
if row is None:
raise ValueError('Cannot reset configuration: configuration table is missing.')
connection.execute('DELETE FROM configuration WHERE id = 1')
connection.commit()
finally:
connection.close()
print('Stored vhost/settings configuration cleared; account, certificates and other private data were preserved.')
def service_failure_details() -> str:
chunks: list[str] = []
commands = (
['systemctl', 'show', SERVICE, '--property=ActiveState,SubState,ExecMainStatus', '--no-pager'],
['journalctl', '-u', SERVICE, '-n', '25', '--no-pager', '--output=cat'],
)
for command in commands:
try:
result = subprocess.run(command, check=False, capture_output=True, text=True, timeout=5)
output = ((result.stdout or '') + (result.stderr or '')).strip()
if output:
chunks.append(output)
except (OSError, subprocess.SubprocessError):
pass
return '\n'.join(chunks)[-8000:]
def stop_for_managed_update(data: Path) -> None:
"""Ask RustProxy 0.14.0+ to force hot cache into the enabled cold tier before exit.
Older installed versions ignore the marker and still follow their ordinary graceful
shutdown policy, so the first upgrade to 0.14.0 remains backward compatible.
The marker is removed before backup so it never becomes persisted application data.
"""
marker = data / '.managed-update'
marker.unlink(missing_ok=True)
write_private(marker, b'1\n', 0o644)
try:
run(['systemctl', 'stop', SERVICE])
finally:
marker.unlink(missing_ok=True)
def update(record: dict, binary: Path, sha: str, start: bool, refresh_geoip: bool = False, reset_config: bool = False) -> dict:
prefix = Path(record['prefix']); data = Path(record['data_dir']); account = pwd.getpwnam(record['user'])
# Updates deliberately do not rewrite shell environment. Check the installer-
# managed PATH profile early and report the exact remediation target instead.
check_shell_path(prefix)
# Do this before staging/network work and, critically, before stopping the
# currently healthy service. It also repairs stale root:root/0700 metadata
# left by manual maintenance while preserving private 0700 semantics.
ensure_service_data_access(data, account)
release, version = stage(binary, prefix, record['user'], sha)
geoip_stage: Path | None = None
geoip_names: list[str] = []
if refresh_geoip:
geoip_stage, geoip_names = prepare_geoip_update(data, prefix)
active = subprocess.run(['systemctl', 'is-active', '--quiet', SERVICE], check=False, stdout=subprocess.DEVNULL).returncode == 0
stop_for_managed_update(data)
try:
backup = backup_stopped(record, prefix)
except BaseException:
if active: run(['systemctl', 'start', SERVICE])
if geoip_stage is not None:
shutil.rmtree(geoip_stage, ignore_errors=True)
raise
replacement = dict(record, release=release, version=version, sha256=sha)
try:
if reset_config:
reset_saved_config(data)
apply_geoip_update(data, geoip_stage, geoip_names, account.pw_uid, account.pw_gid)
switch(prefix, release)
write_private(UNIT_PATH, service_unit(replacement).encode(), 0o644)
run(['systemctl', 'daemon-reload'])
if start:
run(['systemctl', 'start', SERVICE])
if not healthy(replacement, version):
raise RuntimeError('New release failed the local health/version check.')
write_private(prefix / 'install.json', json.dumps(replacement, indent=2).encode())
except BaseException as error:
details = service_failure_details()
if details:
print('New release diagnostics:\n' + details, file=sys.stderr)
stop_for_managed_update(data)
failed = data.with_name(data.name + '.failed-' + uuid.uuid4().hex[:12])
data.rename(failed) # keep the failed generation for diagnosis; do not delete it
copy_data(backup / 'data', data); chown_tree(data, account.pw_uid, account.pw_gid)
switch(prefix, record['release'])
restore_service_unit(backup)
write_private(prefix / 'install.json', json.dumps(record, indent=2).encode())
if active:
run(['systemctl', 'start', SERVICE])
if not healthy(record, record['version']):
raise RuntimeError(f'Update failed: {error}. Rollback restored code/data/unit but previous service health also failed. Backup: {backup}; failed state: {failed}') from error
raise RuntimeError(f'Update failed: {error}. Previous code, private data and service unit were restored. Installation is still present; do not run install.sh. Fix the cause and rerun update.sh. Backup: {backup}; failed state: {failed}') from error
finally:
if geoip_stage is not None:
shutil.rmtree(geoip_stage, ignore_errors=True)
print(f'Backup (contains private keys): {backup}')
return replacement
def install(record: dict, binary: Path, sha: str, start: bool, install_geoip: bool = False) -> dict:
import grp
prefix = Path(record['prefix']); data = Path(record['data_dir'])
if (prefix / 'install.json').exists() or (prefix / 'current').exists() or UNIT_PATH.exists():
raise ValueError('Installation/service already exists. A failed update rolls back to the previous installed release, so rerun update.sh after fixing the reported cause. install.sh never overwrites an existing installation.')
if data.exists() and any(data.iterdir()):
raise ValueError('First installation requires an empty data directory; existing data will never be reset.')
try:
account = pwd.getpwnam(record['user'])
if account.pw_uid == 0 or account.pw_shell not in ('/usr/sbin/nologin', '/sbin/nologin', '/bin/false'):
raise ValueError('Existing service account must be non-root with a non-login shell.')
except KeyError:
run(['useradd', '--system', '--user-group', '--home-dir', str(data), '--no-create-home', '--shell', '/usr/sbin/nologin', record['user']])
account = pwd.getpwnam(record['user'])
record['group'] = grp.getgrgid(account.pw_gid).gr_name
prefix.mkdir(parents=True, mode=0o755, exist_ok=True); os.chmod(prefix, 0o755)
data.mkdir(parents=True, mode=0o700, exist_ok=True); os.chmod(data, 0o700); os.chown(data, account.pw_uid, account.pw_gid)
if install_geoip:
download_geoip(data, account.pw_uid, account.pw_gid)
release, version = stage(binary, prefix, record['user'], sha)
record.update(release=release, version=version, sha256=sha)
switch(prefix, release)
install_shell_path(prefix)
# Record is written even if initial startup fails, so an update can repair it.
write_private(prefix / 'install.json', json.dumps(record, indent=2).encode())
write_private(UNIT_PATH, service_unit(record).encode(), 0o644)
run(['systemctl', 'daemon-reload']); run(['systemctl', 'enable', SERVICE])
if start:
run(['systemctl', 'start', SERVICE])
if not healthy(record, version):
run(['systemctl', 'stop', SERVICE])
raise RuntimeError('Installed but startup failed. Data was retained; inspect journalctl -u rustproxy.')
return record
from deploy_core import * # stable public helper surface used by release checks
from deploy_runtime import * # stable install/update helper surface
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
@@ -771,5 +75,6 @@ def main() -> None:
parser.exit(1, f'Deployment failed: {error}\n')
if __name__ == '__main__':
main()
+352
View File
@@ -0,0 +1,352 @@
#!/usr/bin/env python3
"""Install/update a local release binary on Linux/systemd. No builds.
First install downloads the PDDL user-country GeoIP CSV files from the
sapics/ip-location-db GitHub releases and verifies publisher SHA-256 files.
Updates compare the installed GeoIP files with current publisher checksums and
stage only stale assets before service downtime. Updates stop the service before
copying all private data, refresh the managed systemd unit, switch an immutable
release symlink, check health/version and restore code, data and unit on failure.
Use --plan to inspect operations without changing the host.
"""
from __future__ import annotations
import argparse
import fcntl
import hashlib
import http.client
import ipaddress
import json
import os
from pathlib import Path
import pwd
import re
import shutil
import sqlite3
import ssl
import stat
import subprocess
import sys
import time
import uuid
import urllib.request
import urllib.parse
ROOT = Path(__file__).resolve().parents[1]
UNIT_PATH = Path('/etc/systemd/system/rustproxy.service')
SHELL_PATH_PROFILE = Path('/etc/profile.d/rustproxy-path.sh')
SERVICE = 'rustproxy.service'
GEOIP_ASSETS = (
'user-country-ipv4.csv',
'user-country-ipv6.csv',
)
GEOIP_RELEASE = 'https://github.com/sapics/ip-location-db/releases/download/latest/'
GEOIP_CHECKSUM = 'https://github.com/sapics/ip-location-db/releases/download/checksum/'
GEOIP_MAX_BYTES = 256 * 1024 * 1024
def run(args: list[str], **kwargs) -> subprocess.CompletedProcess:
return subprocess.run(args, check=True, stdin=subprocess.DEVNULL, **kwargs)
def safe_path(raw: str) -> Path:
if not re.fullmatch(r'/[A-Za-z0-9_./-]+', raw) or '..' in raw.split('/') or raw in ('/', '/etc', '/var', '/opt', '/usr', '/var/lib'):
raise ValueError('Use a dedicated absolute path with no spaces, traversal or shell characters.')
p = Path(raw)
for part in (p, *p.parents):
if part.is_symlink():
raise ValueError(f'Symlinked installation/data path is not allowed: {part}')
return p
def address(raw: str, loopback: bool = False, allow_remote: bool = False) -> str:
if raw.startswith('['):
host, port = raw[1:].split(']:', 1)
else:
host, port = raw.rsplit(':', 1)
ip = ipaddress.ip_address(host)
if not port.isdecimal() or not 1 <= int(port) <= 65535:
raise ValueError('Invalid listener IP/port.')
if loopback and not ip.is_loopback and not allow_remote:
raise ValueError('Administration must use loopback unless remote administration is explicitly accepted.')
return f'[{ip}]:{int(port)}' if ip.version == 6 else f'{ip}:{int(port)}'
def listener_ip(raw: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address:
host = raw[1:].split(']:', 1)[0] if raw.startswith('[') else raw.rsplit(':', 1)[0]
return ipaddress.ip_address(host)
def admin_address(raw: str, allow_remote: bool = False) -> tuple[str, bool]:
value = address(raw)
ip = listener_ip(value)
remote = not ip.is_loopback
if remote and not allow_remote:
raise ValueError('Remote administration requires --allow-remote-admin.')
return value, remote
def digest(path: Path) -> str:
with path.open('rb') as stream:
result = hashlib.sha256()
for block in iter(lambda: stream.read(1024 * 1024), b''):
result.update(block)
return result.hexdigest()
def fetch_https(url: str, max_bytes: int) -> bytes:
if urllib.parse.urlparse(url).scheme != 'https':
raise ValueError('GeoIP downloads require HTTPS.')
request = urllib.request.Request(url, headers={'User-Agent': 'RustProxy-installer/0.25.1'})
with urllib.request.urlopen(request, timeout=30) as response:
if urllib.parse.urlparse(response.geturl()).scheme != 'https':
raise ValueError('GeoIP download redirected away from HTTPS.')
length = response.headers.get('Content-Length')
if length and int(length) > max_bytes:
raise ValueError('GeoIP download is larger than the configured safety limit.')
result = bytearray()
while True:
block = response.read(min(1024 * 1024, max_bytes + 1 - len(result)))
if not block:
break
result.extend(block)
if len(result) > max_bytes:
raise ValueError('GeoIP download exceeded the configured safety limit.')
return bytes(result)
def publisher_geoip_checksums() -> dict[str, str]:
expected: dict[str, str] = {}
for name in GEOIP_ASSETS:
checksum = fetch_https(GEOIP_CHECKSUM + name + '.sha256', 4096).decode('ascii', 'strict').strip()
matched = re.search(r'\b([0-9a-fA-F]{64})\b', checksum)
if not matched:
raise ValueError(f'Invalid publisher checksum for {name}.')
expected[name] = matched.group(1).lower()
return expected
def geoip_file_digest(path: Path) -> str | None:
try:
st = path.lstat()
except FileNotFoundError:
return None
if not stat.S_ISREG(st.st_mode) or path.is_symlink():
return None
return digest(path)
def prepare_geoip_update(data: Path, prefix: Path) -> tuple[Path | None, list[str]]:
"""Verify local GeoIP files against current publisher checksums.
Network/download work happens before the service is stopped. Changed files are
staged under the code prefix and are copied into the private data transaction
only after its backup exists.
"""
expected = publisher_geoip_checksums()
directory = data / 'geoip'
if directory.exists() and directory.is_symlink():
raise ValueError('GeoIP directory cannot be a symlink.')
stale = [name for name in GEOIP_ASSETS if geoip_file_digest(directory / name) != expected[name]]
if not stale:
print('GeoIP database is current (publisher SHA-256 matches).')
return None, []
staging = prefix / ('.geoip-stage-' + uuid.uuid4().hex)
staging.mkdir(mode=0o700)
try:
for name in stale:
payload = fetch_https(GEOIP_RELEASE + name, GEOIP_MAX_BYTES)
actual = hashlib.sha256(payload).hexdigest()
if actual != expected[name]:
raise ValueError(f'GeoIP SHA-256 mismatch for {name}.')
write_private(staging / name, payload, 0o600)
print('GeoIP update prepared: ' + ', '.join(stale))
return staging, stale
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
def copy_private(source: Path, target: Path, mode: int = 0o600) -> None:
temporary = target.with_name(target.name + '.' + uuid.uuid4().hex + '.tmp')
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
try:
with source.open('rb') as input_stream, os.fdopen(fd, 'wb') as output_stream:
shutil.copyfileobj(input_stream, output_stream, 1024 * 1024)
output_stream.flush(); os.fsync(output_stream.fileno())
os.replace(temporary, target)
directory = os.open(target.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
temporary.unlink(missing_ok=True)
def apply_geoip_update(data: Path, staging: Path | None, names: list[str], uid: int, gid: int) -> None:
if not names:
return
directory = data / 'geoip'
if directory.exists() and directory.is_symlink():
raise ValueError('GeoIP directory cannot be a symlink.')
directory.mkdir(mode=0o700, exist_ok=True)
os.chmod(directory, 0o700)
os.chown(directory, uid, gid)
if staging is None:
raise ValueError('GeoIP staging directory is missing.')
for name in names:
source = staging / name
target = directory / name
copy_private(source, target, 0o600)
os.chown(target, uid, gid)
print('GeoIP database updated: ' + ', '.join(names))
def download_geoip(data: Path, uid: int, gid: int) -> None:
# First install has no code-prefix staging transaction, so download and verify
# each asset directly before publishing it atomically into the private data dir.
expected = publisher_geoip_checksums()
directory = data / 'geoip'
if directory.exists() and directory.is_symlink():
raise ValueError('GeoIP directory cannot be a symlink.')
directory.mkdir(mode=0o700, exist_ok=True)
os.chmod(directory, 0o700)
os.chown(directory, uid, gid)
for name in GEOIP_ASSETS:
payload = fetch_https(GEOIP_RELEASE + name, GEOIP_MAX_BYTES)
actual = hashlib.sha256(payload).hexdigest()
if actual != expected[name]:
raise ValueError(f'GeoIP SHA-256 mismatch for {name}.')
target = directory / name
write_private(target, payload, 0o600)
os.chown(target, uid, gid)
def write_private(path: Path, value: bytes, mode: int = 0o600) -> None:
temporary = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
try:
with os.fdopen(fd, 'wb') as stream:
stream.write(value); stream.flush(); os.fsync(stream.fileno())
os.replace(temporary, path)
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
temporary.unlink(missing_ok=True)
def shell_path_profile(prefix: Path) -> bytes:
binary_dir = prefix / 'current'
return (
'# Managed by the RustProxy installer. New login shells can invoke rust-proxy directly.\n'
'case ":${PATH:-}:" in\n'
f' *":{binary_dir}:"*) ;;\n'
f' *) export PATH="{binary_dir}${{PATH:+:$PATH}}" ;;\n'
'esac\n'
).encode()
def install_shell_path(prefix: Path) -> None:
write_private(SHELL_PATH_PROFILE, shell_path_profile(prefix), 0o644)
def check_shell_path(prefix: Path) -> bool:
expected = shell_path_profile(prefix)
try:
st = SHELL_PATH_PROFILE.lstat()
valid = stat.S_ISREG(st.st_mode) and st.st_uid == 0 and not (st.st_mode & 0o022) and SHELL_PATH_PROFILE.read_bytes() == expected
except OSError:
valid = False
if not valid:
print(
f'WARNING: shell PATH setup is missing or outdated: {SHELL_PATH_PROFILE}. '
f'Expected {prefix}/current in PATH; update mode only checks this file and does not modify it.',
file=sys.stderr,
)
return valid
def read_record(prefix: Path) -> dict:
path = prefix / 'install.json'
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
with os.fdopen(fd, 'rb') as stream:
st = os.fstat(stream.fileno())
if not stat.S_ISREG(st.st_mode) or st.st_uid != 0 or st.st_mode & 0o077 or st.st_size > 16384:
raise ValueError('Unsafe or invalid root-owned installation record.')
value = json.load(stream)
if value.get('schema') != 1 or value.get('prefix') != str(prefix):
raise ValueError('Installation record mismatch.')
safe_path(value['data_dir']); validate_user(value['user'])
remote_admin = bool(value.get('remote_admin', False))
admin, checked_remote = admin_address(value['admin'], remote_admin)
if checked_remote != remote_admin:
raise ValueError('Installation record remote administration flag does not match its address.')
value['admin'] = admin
value['remote_admin'] = remote_admin
return value
def validate_user(name: str) -> None:
if not re.fullmatch(r'[a-z_][a-z0-9_-]{0,30}', name) or name in ('root', 'nobody'):
raise ValueError('Use a dedicated non-root service account.')
def service_unit(record: dict) -> str:
prefix, data, user = record['prefix'], record['data_dir'], record['user']
args = f'{prefix}/current/rust-proxy --data-dir {data} --admin-listen {record["admin"]} serve'
args += f' --proxy-listen {record["http"]}'
if record.get('https'):
args += f' --tls-listen {record["https"]}'
if record.get('remote_admin'):
args += ' --allow-remote-admin'
return f'''[Unit]
Description=RustProxy reverse proxy
Wants=network-online.target
After=network-online.target
[Service]
Type=simple
User={user}
Group={record['group']}
WorkingDirectory={data}
UMask=0077
Environment=RUST_LOG=rust_proxy=info,warn
Environment=MALLOC_ARENA_MAX=2
Environment=PATH=/opt/rustproxy-certbot/bin:/usr/local/bin:/usr/bin:/bin
EnvironmentFile=-/etc/rustproxy/environment
ExecStart={args}
Restart=on-failure
RestartSec=3
KillSignal=SIGINT
KillMode=control-group
TimeoutStopSec=120
AmbientCapabilities=CAP_NET_BIND_SERVICE
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectSystem=strict
LogsDirectory=rustproxy
LogsDirectoryMode=0770
ReadWritePaths={data} /var/log/rustproxy
ProtectHome=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectKernelLogs=true
ProtectControlGroups=true
RestrictSUIDSGID=true
RestrictNamespaces=true
RestrictRealtime=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
SystemCallArchitectures=native
LockPersonality=true
[Install]
WantedBy=multi-user.target
'''
+356
View File
@@ -0,0 +1,356 @@
"""Runtime install/update operations for deploy.py."""
from __future__ import annotations
from deploy_core import * # re-exported internal deployment primitives
def checked_tree(source: Path) -> None:
"""Accept only regular files/directories and relative Certbot-internal symlinks."""
boundary = source.resolve(strict=True)
if source.is_symlink():
raise ValueError('Data root cannot be a symlink.')
for p in source.rglob('*'):
st = p.lstat()
if stat.S_ISLNK(st.st_mode):
target = os.readlink(p)
if os.path.isabs(target) or not p.resolve(strict=True).is_relative_to(boundary):
raise ValueError('Data backup contains an external or absolute symlink.')
elif not (stat.S_ISDIR(st.st_mode) or stat.S_ISREG(st.st_mode)):
raise ValueError('Data backup contains a special file.')
def copy_data(source: Path, target: Path) -> None:
checked_tree(source)
shutil.copytree(source, target, symlinks=True)
os.chmod(target, 0o700)
def chown_tree(path: Path, uid: int, gid: int) -> None:
os.chown(path, uid, gid)
for child in path.rglob('*'):
os.chown(child, uid, gid, follow_symlinks=False)
def ensure_service_data_access(data: Path, account) -> None:
"""Repair the managed data-root metadata and verify service-user traversal.
systemd applies WorkingDirectory after dropping to User=/Group=. A stale
root-owned 0700 data directory therefore fails as status=200/CHDIR before
RustProxy itself can emit diagnostics. Repair only the managed data root;
never chmod/chown parent directories. Then fail before service downtime if a
parent directory still prevents traversal.
"""
try:
st = data.lstat()
except FileNotFoundError as error:
raise ValueError(f'Data directory is missing: {data}') from error
if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode):
raise ValueError(f'Data directory must be a real directory: {data}')
os.chown(data, account.pw_uid, account.pw_gid)
os.chmod(data, 0o700)
probe = subprocess.run(
['runuser', '-u', account.pw_name, '--', 'test', '-x', str(data)],
check=False, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
if probe.returncode != 0:
raise ValueError(
f'Service user {account.pw_name} cannot traverse data directory {data}; '
'check execute/search permissions on its parent directories (for example with namei -l).'
)
def switch(prefix: Path, release: str) -> None:
if not re.fullmatch(r'[A-Za-z0-9_.-]+', release) or '..' in release:
raise ValueError('Invalid release name.')
temporary = prefix / ('current-' + uuid.uuid4().hex)
os.symlink('releases/' + release, temporary)
os.replace(temporary, prefix / 'current')
def running_release_matches(record: dict) -> bool:
"""Check the actual systemd process executable without publishing a version API.
This installer targets Linux/systemd. Fail closed if /proc or MainPID cannot
establish that the service runs the exact staged, version-checked binary.
"""
try:
result = subprocess.run(['systemctl', 'show', '--property=MainPID', '--value', SERVICE],
check=False, capture_output=True, text=True, timeout=3)
if result.returncode != 0 or not result.stdout.strip().isdigit():
return False
pid = int(result.stdout.strip())
if pid <= 1:
return False
expected = Path(record['prefix']) / 'releases' / record['release'] / 'rust-proxy'
return Path(f'/proc/{pid}/exe').samefile(expected)
except (OSError, ValueError, KeyError, subprocess.SubprocessError):
return False
def admin_health_endpoint(record: dict) -> tuple[str, bool, str]:
"""Resolve the active persisted administration listener and scheme.
The installer record contains only the startup CLI address. Since 0.9.x the
dashboard can persist both a different admin bind and HTTPS in config.sqlite3.
Update health checks must follow that active intent or a healthy HTTPS panel is
mistaken for a failed release and rollback can fail for the same reason.
"""
raw = record['admin']
tls = False
allowed_hosts: list[str] = []
database = Path(record['data_dir']) / 'config.sqlite3'
try:
connection = sqlite3.connect(f'file:{database}?mode=ro', uri=True, timeout=2)
try:
row = connection.execute('SELECT document FROM configuration WHERE id = 1').fetchone()
finally:
connection.close()
if row:
document = json.loads(row[0])
administration = document.get('administration') or {}
if isinstance(administration, dict):
if administration.get('listen'):
raw = address(str(administration['listen']))
tls = bool(administration.get('tls', False))
values = administration.get('allowed_hosts') or []
if isinstance(values, list):
allowed_hosts = [str(value).strip() for value in values if str(value).strip()]
except (OSError, sqlite3.Error, ValueError, TypeError, json.JSONDecodeError):
# Old releases and damaged/uninitialized databases used the startup HTTP
# listener. The real startup diagnostics remain authoritative if this
# compatibility read cannot resolve persisted settings.
pass
host, port = (raw[1:].split(']:') if raw.startswith('[') else raw.rsplit(':', 1))
ip = ipaddress.ip_address(host)
connect_host = ('::1' if ip.version == 6 else '127.0.0.1') if ip.is_unspecified else host
authority_host = allowed_hosts[0] if allowed_hosts else connect_host
try:
authority_ip = ipaddress.ip_address(authority_host)
authority = f'[{authority_ip}]:{port}' if authority_ip.version == 6 else f'{authority_ip}:{port}'
except ValueError:
authority = f'{authority_host}:{port}'
connect_raw = f'[{connect_host}]:{port}' if ':' in connect_host else f'{connect_host}:{port}'
return connect_raw, tls, authority
def healthy(record: dict, version: str, attempts: int = 30) -> bool:
raw, tls, health_host = admin_health_endpoint(record)
host, port = (raw[1:].split(']:') if raw.startswith('[') else raw.rsplit(':', 1))
for _ in range(attempts):
if tls:
# Local deployment health is process identity/liveness, not PKI validation.
# The configured admin certificate may intentionally be self-signed or for
# a DNS name while we connect to the local socket.
conn = http.client.HTTPSConnection(host, int(port), timeout=2, context=ssl._create_unverified_context())
else:
conn = http.client.HTTPConnection(host, int(port), timeout=2)
try:
conn.request('GET', '/healthz', headers={'Host': health_host})
response = conn.getresponse(); content = response.read(4097)
active = subprocess.run(['systemctl', 'is-active', '--quiet', SERVICE], check=False, stdout=subprocess.DEVNULL).returncode == 0
if (active and response.status == 200 and len(content) <= 4096
and json.loads(content).get('status') == 'ok' and record.get('version') == version
and running_release_matches(record)):
return True
except (OSError, ValueError, http.client.HTTPException, ssl.SSLError):
pass
finally:
conn.close()
time.sleep(1)
return False
def stage(binary: Path, prefix: Path, user: str, sha: str) -> tuple[str, str]:
release = 'build-' + sha[:24]
releases = prefix / 'releases'; releases.mkdir(mode=0o755, exist_ok=True); os.chmod(releases, 0o755)
directory = releases / release
if not directory.exists():
directory.mkdir(mode=0o755); os.chmod(directory, 0o755)
try:
shutil.copyfile(binary, directory / 'rust-proxy')
os.chmod(directory / 'rust-proxy', 0o755)
except BaseException:
shutil.rmtree(directory); raise
if directory.is_symlink() or (directory / 'rust-proxy').is_symlink() or digest(directory / 'rust-proxy') != sha:
raise ValueError('Staged binary checksum mismatch.')
# Never execute the supplied binary as root, even for its version banner.
result = run(['runuser', '-u', user, '--', str(directory / 'rust-proxy'), '--version'], capture_output=True, text=True, timeout=15)
matched = re.fullmatch(r'(?:rust-proxy|RustProxy) (\d+\.\d+\.\d+(?:[-+][\w.-]+)?)\s*', result.stdout)
if not matched:
raise ValueError('Binary did not return a RustProxy version.')
return release, matched[1]
def backup_stopped(record: dict, prefix: Path) -> Path:
# Service stop must happen before this call: ACME files and SQLite form one backup.
data = Path(record['data_dir'])
lock_fd = os.open(data / 'process.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
try:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
account = pwd.getpwnam(record['user']); os.fchown(lock_fd, account.pw_uid, account.pw_gid)
backups = prefix / 'backups'; backups.mkdir(mode=0o700, exist_ok=True)
backup = backups / ('before-' + record['version'] + '-' + uuid.uuid4().hex[:12]); backup.mkdir(mode=0o700)
copy_data(data, backup / 'data')
write_private(backup / 'install.json', json.dumps(record, indent=2).encode())
if UNIT_PATH.exists():
shutil.copyfile(UNIT_PATH, backup / 'rustproxy.service'); os.chmod(backup / 'rustproxy.service', 0o600)
return backup
finally:
os.close(lock_fd)
def restore_service_unit(backup: Path) -> None:
saved = backup / 'rustproxy.service'
if saved.exists():
copy_private(saved, UNIT_PATH, 0o644)
else:
UNIT_PATH.unlink(missing_ok=True)
run(['systemctl', 'daemon-reload'])
def reset_saved_config(data: Path) -> None:
"""Development-only reset of desired config while preserving auth/certs/audit data."""
database = data / 'config.sqlite3'
if database.is_symlink() or not database.is_file():
raise ValueError('Cannot reset configuration: config.sqlite3 is missing or unsafe.')
connection = sqlite3.connect(f'file:{database}?mode=rw', uri=True, timeout=5)
try:
row = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='configuration'").fetchone()
if row is None:
raise ValueError('Cannot reset configuration: configuration table is missing.')
connection.execute('DELETE FROM configuration WHERE id = 1')
connection.commit()
finally:
connection.close()
print('Stored vhost/settings configuration cleared; account, certificates and other private data were preserved.')
def service_failure_details() -> str:
chunks: list[str] = []
commands = (
['systemctl', 'show', SERVICE, '--property=ActiveState,SubState,ExecMainStatus', '--no-pager'],
['journalctl', '-u', SERVICE, '-n', '25', '--no-pager', '--output=cat'],
)
for command in commands:
try:
result = subprocess.run(command, check=False, capture_output=True, text=True, timeout=5)
output = ((result.stdout or '') + (result.stderr or '')).strip()
if output:
chunks.append(output)
except (OSError, subprocess.SubprocessError):
pass
return '\n'.join(chunks)[-8000:]
def stop_for_managed_update(data: Path) -> None:
"""Ask RustProxy 0.14.0+ to force hot cache into the enabled cold tier before exit.
Older installed versions ignore the marker and still follow their ordinary graceful
shutdown policy, so the first upgrade to 0.14.0 remains backward compatible.
The marker is removed before backup so it never becomes persisted application data.
"""
marker = data / '.managed-update'
marker.unlink(missing_ok=True)
write_private(marker, b'1\n', 0o644)
try:
run(['systemctl', 'stop', SERVICE])
finally:
marker.unlink(missing_ok=True)
def update(record: dict, binary: Path, sha: str, start: bool, refresh_geoip: bool = False, reset_config: bool = False) -> dict:
prefix = Path(record['prefix']); data = Path(record['data_dir']); account = pwd.getpwnam(record['user'])
# Updates deliberately do not rewrite shell environment. Check the installer-
# managed PATH profile early and report the exact remediation target instead.
check_shell_path(prefix)
# Do this before staging/network work and, critically, before stopping the
# currently healthy service. It also repairs stale root:root/0700 metadata
# left by manual maintenance while preserving private 0700 semantics.
ensure_service_data_access(data, account)
release, version = stage(binary, prefix, record['user'], sha)
geoip_stage: Path | None = None
geoip_names: list[str] = []
if refresh_geoip:
geoip_stage, geoip_names = prepare_geoip_update(data, prefix)
active = subprocess.run(['systemctl', 'is-active', '--quiet', SERVICE], check=False, stdout=subprocess.DEVNULL).returncode == 0
stop_for_managed_update(data)
try:
backup = backup_stopped(record, prefix)
except BaseException:
if active: run(['systemctl', 'start', SERVICE])
if geoip_stage is not None:
shutil.rmtree(geoip_stage, ignore_errors=True)
raise
replacement = dict(record, release=release, version=version, sha256=sha)
try:
if reset_config:
reset_saved_config(data)
apply_geoip_update(data, geoip_stage, geoip_names, account.pw_uid, account.pw_gid)
switch(prefix, release)
write_private(UNIT_PATH, service_unit(replacement).encode(), 0o644)
run(['systemctl', 'daemon-reload'])
if start:
run(['systemctl', 'start', SERVICE])
if not healthy(replacement, version):
raise RuntimeError('New release failed the local health/version check.')
write_private(prefix / 'install.json', json.dumps(replacement, indent=2).encode())
except BaseException as error:
details = service_failure_details()
if details:
print('New release diagnostics:\n' + details, file=sys.stderr)
stop_for_managed_update(data)
failed = data.with_name(data.name + '.failed-' + uuid.uuid4().hex[:12])
data.rename(failed) # keep the failed generation for diagnosis; do not delete it
copy_data(backup / 'data', data); chown_tree(data, account.pw_uid, account.pw_gid)
switch(prefix, record['release'])
restore_service_unit(backup)
write_private(prefix / 'install.json', json.dumps(record, indent=2).encode())
if active:
run(['systemctl', 'start', SERVICE])
if not healthy(record, record['version']):
raise RuntimeError(f'Update failed: {error}. Rollback restored code/data/unit but previous service health also failed. Backup: {backup}; failed state: {failed}') from error
raise RuntimeError(f'Update failed: {error}. Previous code, private data and service unit were restored. Installation is still present; do not run install.sh. Fix the cause and rerun update.sh. Backup: {backup}; failed state: {failed}') from error
finally:
if geoip_stage is not None:
shutil.rmtree(geoip_stage, ignore_errors=True)
print(f'Backup (contains private keys): {backup}')
return replacement
def install(record: dict, binary: Path, sha: str, start: bool, install_geoip: bool = False) -> dict:
import grp
prefix = Path(record['prefix']); data = Path(record['data_dir'])
if (prefix / 'install.json').exists() or (prefix / 'current').exists() or UNIT_PATH.exists():
raise ValueError('Installation/service already exists. A failed update rolls back to the previous installed release, so rerun update.sh after fixing the reported cause. install.sh never overwrites an existing installation.')
if data.exists() and any(data.iterdir()):
raise ValueError('First installation requires an empty data directory; existing data will never be reset.')
try:
account = pwd.getpwnam(record['user'])
if account.pw_uid == 0 or account.pw_shell not in ('/usr/sbin/nologin', '/sbin/nologin', '/bin/false'):
raise ValueError('Existing service account must be non-root with a non-login shell.')
except KeyError:
run(['useradd', '--system', '--user-group', '--home-dir', str(data), '--no-create-home', '--shell', '/usr/sbin/nologin', record['user']])
account = pwd.getpwnam(record['user'])
record['group'] = grp.getgrgid(account.pw_gid).gr_name
prefix.mkdir(parents=True, mode=0o755, exist_ok=True); os.chmod(prefix, 0o755)
data.mkdir(parents=True, mode=0o700, exist_ok=True); os.chmod(data, 0o700); os.chown(data, account.pw_uid, account.pw_gid)
if install_geoip:
download_geoip(data, account.pw_uid, account.pw_gid)
release, version = stage(binary, prefix, record['user'], sha)
record.update(release=release, version=version, sha256=sha)
switch(prefix, release)
install_shell_path(prefix)
# Record is written even if initial startup fails, so an update can repair it.
write_private(prefix / 'install.json', json.dumps(record, indent=2).encode())
write_private(UNIT_PATH, service_unit(record).encode(), 0o644)
run(['systemctl', 'daemon-reload']); run(['systemctl', 'enable', SERVICE])
if start:
run(['systemctl', 'start', SERVICE])
if not healthy(record, version):
run(['systemctl', 'stop', SERVICE])
raise RuntimeError('Installed but startup failed. Data was retained; inspect journalctl -u rustproxy.')
return record
+8 -6
View File
@@ -1,7 +1,8 @@
#!/usr/bin/env python3
"""Rust-independent documentation/Reader consistency checks for the 0.25 release."""
"""Rust-independent documentation/Reader consistency checks for the 0.25.1 release."""
from pathlib import Path
import re
from rust_source import rust_source
ROOT = Path(__file__).resolve().parents[1]
DOCS = ROOT / "docs"
@@ -13,6 +14,7 @@ required = [
DOCS / "README.md",
DOCS / "WEB_SERVER.md",
DOCS / "TODO.md",
DOCS / "V0.25.1_AUDIT.md",
DOCS / "V0.25.0_AUDIT.md",
DOCS / "TEST_REPORT.md",
DOCS / "CHANGELOG_TECHNICAL_ARCHIVE.md",
@@ -22,7 +24,7 @@ for path in required:
assert path.is_file(), f"missing required documentation: {path.relative_to(ROOT)}"
index = (DOCS / "README.md").read_text(encoding="utf-8")
for marker in ["Functional documentation", "Development and testing", "Technical documentation", "Historical or version-labelled context", "TODO.md", "V0.25.0_AUDIT.md", "CHANGELOG_TECHNICAL_ARCHIVE.md", "ROADMAP_HISTORY.md", "/docs"]:
for marker in ["Functional documentation", "Development and testing", "Technical documentation", "Historical or version-labelled context", "TODO.md", "V0.25.1_AUDIT.md", "V0.25.0_AUDIT.md", "CHANGELOG_TECHNICAL_ARCHIVE.md", "ROADMAP_HISTORY.md", "/docs"]:
assert marker in index, marker
for document in sorted(DOCS.glob("*.md")):
if document.name == "README.md" or (document.name.startswith("V") and document.name.endswith("_AUDIT.md")):
@@ -32,7 +34,7 @@ for document in sorted(DOCS.glob("*.md")):
reader_html = (ROOT / "src/admin/web/docs.html").read_text(encoding="utf-8")
reader_js = (ROOT / "src/admin/web/docs.js").read_text(encoding="utf-8")
build = (ROOT / "build.rs").read_text(encoding="utf-8")
admin = (ROOT / "src/admin/mod.rs").read_text(encoding="utf-8")
admin = rust_source(ROOT, "src/admin/mod.rs")
for marker in ["docs-tab-reader", "docs-reader-list", "docs-reader-content", "docs-tab-api"]:
assert marker in reader_html, marker
for marker in ["renderMarkdown", "selectDocument", 'get("/docs/content.json")', "docsByPath", "docsCategoryHistorical"]:
@@ -49,13 +51,13 @@ assert '"/docs/content.json"' in admin
readme = (ROOT / "README.md").read_text(encoding="utf-8")
roadmap = (ROOT / "ROADMAP.md").read_text(encoding="utf-8")
changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8")
assert readme.startswith("# RustProxy 0.25.0")
assert readme.startswith("# RustProxy 0.25.1")
assert "docs/WEB_SERVER.md" in readme and "docs/README.md" in readme
assert readme.count("## What changed in ") == 1, "README duplicates historical release notes; use CHANGELOG/archive"
assert "## Release history" in readme and "docs/ROADMAP_HISTORY.md" in readme
assert roadmap.startswith("# RustProxy roadmap") and "## 0.25.0" in roadmap
assert roadmap.startswith("# RustProxy roadmap") and "## 0.25.1" in roadmap and "## 0.25.0" in roadmap
assert "## 0.24." not in roadmap and "## Web Server follow-up" in roadmap and "docs/ROADMAP_HISTORY.md" in roadmap
assert changelog.startswith("# Changelog\n") and "## 0.25.0" in changelog[:800]
assert changelog.startswith("# Changelog\n") and "## 0.25.1" in changelog[:800]
assert "CHANGELOG_TECHNICAL_ARCHIVE.md" in changelog
for developer_only in ["Clean the remaining compiler warnings", "Update package/source/UI/OpenAPI/release identity", "source regressions and update release identity"]:
assert developer_only not in changelog, f"developer-only CHANGELOG detail: {developer_only}"
+3 -1
View File
@@ -8,12 +8,14 @@ silently disappearing before compiler-enabled acceptance runs.
from __future__ import annotations
from pathlib import Path
from rust_source import rust_source
ROOT = Path(__file__).resolve().parents[1]
def read(path: str) -> str:
return (ROOT / path).read_text(encoding="utf-8")
target = ROOT / path
return rust_source(ROOT, path) if target.suffix == ".rs" else target.read_text(encoding="utf-8")
def require(path: str, *markers: str) -> None:
+4 -518
View File
@@ -1,524 +1,9 @@
#!/usr/bin/env python3
"""Migrate Nginx Proxy Manager proxy hosts from SQLite into RustProxy.
The migrator intentionally focuses on proxy-host identity and upstream routing. It
never imports certificate/key material from NPM. NPM hosts that used TLS are attached
to RustProxy's TLS listener and use the RustProxy default certificate unless a named
certificate is supplied explicitly.
Run without --apply first. The script logs in to the RustProxy administration API,
reads the current revision, builds a merged configuration and prints a review report.
Only --apply performs the revisioned PUT /api/config.
"""
"""Migrate Nginx Proxy Manager proxy hosts from SQLite into RustProxy."""
from __future__ import annotations
import argparse
from dataclasses import dataclass, asdict
import getpass
import http.cookiejar
import ipaddress
import json
import os
from pathlib import Path
import re
import sqlite3
import ssl
import sys
from typing import Any
import urllib.error
import urllib.parse
import urllib.request
MAX_CONFIG_BYTES = 262_144
class MigrationError(RuntimeError):
pass
@dataclass(frozen=True)
class NpmProxyHost:
id: int
domain_names: list[str]
forward_scheme: str
forward_host: str
forward_port: int
allow_websocket_upgrade: bool
http2_support: bool
enabled: bool
certificate_id: int
ssl_forced: bool
access_list_id: int
advanced_config: str
locations: list[Any]
caching_enabled: bool
block_exploits: bool
meta: dict[str, Any]
@dataclass
class Decision:
npm_id: int
action: str
hosts: list[str]
backend: str | None = None
rustproxy_ids: list[str] | None = None
warnings: list[str] | None = None
reason: str | None = None
def json_value(raw: Any, fallback: Any) -> Any:
if raw is None or raw == "":
return fallback
if isinstance(raw, (list, dict)):
return raw
try:
return json.loads(raw)
except (TypeError, json.JSONDecodeError):
return fallback
def bool_value(raw: Any) -> bool:
if isinstance(raw, str):
return raw.strip().lower() in {"1", "true", "yes", "on"}
return bool(raw)
def normalize_host_pattern(value: str) -> str:
value = value.strip().lower().rstrip(".")
wildcard = value.startswith("*.")
base = value[2:] if wildcard else value
if not base or len(base) > 253 or "*" in base or ":" in base or "/" in base:
raise MigrationError(f"invalid hostname {value!r}")
labels = base.split(".")
if wildcard and len(labels) < 2:
raise MigrationError(f"invalid wildcard hostname {value!r}")
for label in labels:
if not label or len(label) > 63 or label.startswith("-") or label.endswith("-"):
raise MigrationError(f"invalid hostname {value!r}")
if not re.fullmatch(r"[a-z0-9-]+", label):
raise MigrationError(f"hostname must be ASCII/punycode: {value!r}")
return ("*." if wildcard else "") + base
def sql_columns(connection: sqlite3.Connection, table: str) -> set[str]:
return {str(row[1]) for row in connection.execute(f"PRAGMA table_info({table})")}
def load_npm_hosts(database: Path) -> list[NpmProxyHost]:
if database.is_symlink() or not database.is_file():
raise MigrationError(f"NPM SQLite database not found or unsafe: {database}")
uri = f"file:{urllib.parse.quote(str(database.resolve()))}?mode=ro"
connection = sqlite3.connect(uri, uri=True)
connection.row_factory = sqlite3.Row
try:
tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")}
if "proxy_host" not in tables:
raise MigrationError("NPM database does not contain the proxy_host table")
columns = sql_columns(connection, "proxy_host")
required = {"id", "domain_names", "forward_scheme", "forward_host", "forward_port"}
missing = sorted(required - columns)
if missing:
raise MigrationError("NPM proxy_host table is missing required columns: " + ", ".join(missing))
wanted = [
"id", "domain_names", "forward_scheme", "forward_host", "forward_port",
"allow_websocket_upgrade", "http2_support", "enabled", "certificate_id",
"ssl_forced", "access_list_id", "advanced_config", "locations",
"caching_enabled", "block_exploits", "meta", "is_deleted",
]
select = ", ".join(name for name in wanted if name in columns)
where = " WHERE is_deleted = 0" if "is_deleted" in columns else ""
rows = connection.execute(f"SELECT {select} FROM proxy_host{where} ORDER BY id").fetchall()
finally:
connection.close()
result: list[NpmProxyHost] = []
for row in rows:
data = dict(row)
domains_raw = json_value(data.get("domain_names"), [])
if not isinstance(domains_raw, list):
domains_raw = []
domains: list[str] = []
for item in domains_raw:
if isinstance(item, str) and item.strip():
domains.append(item.strip())
locations = json_value(data.get("locations"), [])
if not isinstance(locations, list):
locations = []
meta = json_value(data.get("meta"), {})
if not isinstance(meta, dict):
meta = {}
try:
port = int(data.get("forward_port"))
except (TypeError, ValueError):
port = 0
result.append(NpmProxyHost(
id=int(data["id"]),
domain_names=domains,
forward_scheme=str(data.get("forward_scheme") or "").strip().lower(),
forward_host=str(data.get("forward_host") or "").strip(),
forward_port=port,
allow_websocket_upgrade=bool_value(data.get("allow_websocket_upgrade", False)),
http2_support=bool_value(data.get("http2_support", True)),
enabled=bool_value(data.get("enabled", True)),
certificate_id=int(data.get("certificate_id") or 0),
ssl_forced=bool_value(data.get("ssl_forced", False)),
access_list_id=int(data.get("access_list_id") or 0),
advanced_config=str(data.get("advanced_config") or ""),
locations=locations,
caching_enabled=bool_value(data.get("caching_enabled", False)),
block_exploits=bool_value(data.get("block_exploits", False)),
meta=meta,
))
return result
def backend_url(item: NpmProxyHost) -> str:
if item.forward_scheme not in {"http", "https"}:
raise MigrationError(f"unsupported upstream scheme {item.forward_scheme!r}")
if not 1 <= item.forward_port <= 65535:
raise MigrationError(f"invalid upstream port {item.forward_port}")
host = item.forward_host.strip()
if not host or any(ch.isspace() for ch in host) or "/" in host or "@" in host:
raise MigrationError(f"invalid upstream host {host!r}")
if host.startswith("[") and host.endswith("]"):
rendered = host
else:
try:
ip = ipaddress.ip_address(host)
rendered = f"[{ip}]" if ip.version == 6 else str(ip)
except ValueError:
if ":" in host:
raise MigrationError(f"invalid upstream host {host!r}")
rendered = host
return f"{item.forward_scheme}://{rendered}:{item.forward_port}"
def slug(value: str) -> str:
value = value.lower().replace("*.", "wildcard-")
value = re.sub(r"[^a-z0-9_-]+", "-", value).strip("-_")
return value[:40] or "host"
def unique_id(base: str, used: set[str]) -> str:
base = re.sub(r"[^A-Za-z0-9_-]+", "-", base)[:64].strip("-_") or "npm"
candidate = base
index = 2
while candidate in used:
suffix = f"-{index}"
candidate = base[:64 - len(suffix)] + suffix
index += 1
used.add(candidate)
return candidate
def listener_inventory(config: dict[str, Any]) -> tuple[list[str], list[str]]:
http_ids: list[str] = []
https_ids: list[str] = []
for entry in ((config.get("listeners") or {}).get("endpoints") or []):
if not isinstance(entry, dict) or not entry.get("enabled", True):
continue
listener_id = str(entry.get("id") or "").strip()
binding = entry.get("binding") or {}
if not listener_id or not isinstance(binding, dict):
continue
(https_ids if binding.get("tls") else http_ids).append(listener_id)
return http_ids, https_ids
def selected_tls_ports(config: dict[str, Any], listener_ids: list[str]) -> set[int]:
selected = set(listener_ids)
ports: set[int] = set()
for entry in ((config.get("listeners") or {}).get("endpoints") or []):
if not isinstance(entry, dict) or str(entry.get("id") or "") not in selected:
continue
binding = entry.get("binding") or {}
if not isinstance(binding, dict) or not binding.get("tls"):
continue
for address in [binding.get("address"), *(entry.get("additional_addresses") or [])]:
if not isinstance(address, str) or ":" not in address:
raise MigrationError(f"invalid RustProxy listener address for Force SSL: {address!r}")
try:
port = int(address.rsplit(":", 1)[1])
except (TypeError, ValueError) as error:
raise MigrationError(f"invalid RustProxy listener address for Force SSL: {address!r}") from error
if not 1 <= port <= 65535:
raise MigrationError(f"invalid RustProxy listener port for Force SSL: {address!r}")
ports.add(port)
return ports
def existing_host_owners(config: dict[str, Any]) -> dict[str, str]:
result: dict[str, str] = {}
for vhost in config.get("vhosts") or []:
if not isinstance(vhost, dict):
continue
vid = str(vhost.get("id") or "?")
frontend = vhost.get("frontend") or {}
for raw in frontend.get("hosts") or []:
if isinstance(raw, str):
try:
result[normalize_host_pattern(raw)] = vid
except MigrationError:
pass
return result
def warnings_for(item: NpmProxyHost, tls_verify: bool) -> list[str]:
warnings: list[str] = []
if item.access_list_id:
warnings.append(f"NPM access_list_id={item.access_list_id} was not imported; attach an ACL Profile manually")
if item.advanced_config.strip():
warnings.append("NPM advanced_config was not imported")
if item.locations:
warnings.append(f"{len(item.locations)} NPM custom location(s) were not imported")
if item.caching_enabled:
warnings.append("NPM caching_enabled was not mapped; configure RustProxy cache policy separately")
if item.block_exploits:
warnings.append("NPM block_exploits was not mapped")
if item.ssl_forced:
warnings.append("NPM ssl_forced is mapped to RustProxy Force SSL (HTTP -> HTTPS 308 redirect)")
if item.forward_scheme == "https" and tls_verify:
warnings.append("HTTPS upstream verification was explicitly enabled; provide a matching DNS name/private CA when needed")
if item.meta.get("nginx_err"):
warnings.append("NPM meta.nginx_err is set; review the source host before cutover")
return warnings
def build_migration(
npm_hosts: list[NpmProxyHost],
config: dict[str, Any],
*,
http_listener_ids: list[str] | None = None,
https_listener_ids: list[str] | None = None,
certificate_id: str | None = None,
force_https: bool = False,
npm_compatible_backend_tls: bool = False,
) -> tuple[dict[str, Any], list[Decision]]:
new_config = json.loads(json.dumps(config))
new_config.setdefault("vhosts", [])
discovered_http, discovered_https = listener_inventory(new_config)
http_ids = http_listener_ids if http_listener_ids is not None else discovered_http[:1]
https_ids = https_listener_ids if https_listener_ids is not None else discovered_https[:1]
unknown_http = [value for value in http_ids if value not in discovered_http]
unknown_https = [value for value in https_ids if value not in discovered_https]
if unknown_http:
raise MigrationError("HTTP listener ID is missing, disabled or TLS-enabled: " + ", ".join(unknown_http))
if unknown_https:
raise MigrationError("HTTPS listener ID is missing, disabled or not TLS-enabled: " + ", ".join(unknown_https))
if not http_ids and not https_ids:
raise MigrationError("RustProxy config has no enabled HTTP/TLS listener to attach imported vhosts")
https_ports = selected_tls_ports(new_config, https_ids)
if certificate_id:
known_certificates = {str(item.get("id")) for item in ((new_config.get("tls") or {}).get("certificates") or []) if isinstance(item, dict)}
if certificate_id not in known_certificates:
raise MigrationError(f"RustProxy certificate ID not found: {certificate_id}")
owners = existing_host_owners(new_config)
used_ids = {str(v.get("id")) for v in new_config.get("vhosts") or [] if isinstance(v, dict)}
decisions: list[Decision] = []
for item in npm_hosts:
warnings = warnings_for(item, False)
if not item.domain_names:
decisions.append(Decision(item.id, "skip", [], warnings=warnings, reason="no domain_names"))
continue
try:
normalized = []
seen = set()
for raw in item.domain_names:
host = normalize_host_pattern(raw)
if host not in seen:
normalized.append(host)
seen.add(host)
origin = backend_url(item)
except MigrationError as error:
decisions.append(Decision(item.id, "skip", item.domain_names, warnings=warnings, reason=str(error)))
continue
conflicts = [(host, owners[host]) for host in normalized if host in owners]
if conflicts:
reason = "hostname already owned in RustProxy: " + ", ".join(f"{host} ({owner})" for host, owner in conflicts)
decisions.append(Decision(item.id, "skip", normalized, backend=origin, warnings=warnings, reason=reason))
continue
force_ssl = item.ssl_forced or force_https
source_tls = item.certificate_id > 0 or force_ssl
if force_ssl and not https_ids:
decisions.append(Decision(
item.id, "skip", normalized, backend=origin, warnings=warnings,
reason="NPM Force SSL requires an enabled RustProxy TLS listener",
))
continue
if force_ssl and len(https_ports) != 1:
decisions.append(Decision(
item.id, "skip", normalized, backend=origin, warnings=warnings,
reason="RustProxy Force SSL requires the selected TLS listeners to use one HTTPS port",
))
continue
if source_tls:
listeners = list(dict.fromkeys([*http_ids, *https_ids]))
else:
listeners = list(http_ids or https_ids)
if source_tls and not https_ids:
warnings.append("source host used TLS but RustProxy has no enabled TLS listener; imported on HTTP only")
listeners = list(http_ids)
if force_ssl and not http_ids:
warnings.append("Force SSL is preserved, but no HTTP listener is selected so there is currently no cleartext request to redirect")
if not listeners:
decisions.append(Decision(item.id, "skip", normalized, backend=origin, warnings=warnings, reason="no compatible RustProxy listener"))
continue
created_ids: list[str] = []
base = f"npm-{item.id}-{slug(normalized[0])}"
vid = unique_id(base, used_ids)
vhost: dict[str, Any] = {
"id": vid,
"enabled": item.enabled,
"force_ssl": force_ssl,
"forward_client_ip": True,
"websockets": item.allow_websocket_upgrade,
"http2_enabled": item.http2_support,
"http3_enabled": False,
"frontend": {"hosts": normalized, "listener_ids": listeners},
"backends": [{
"id": "origin",
"url": origin,
"allow_private": True,
"preserve_host": True,
"tls_verify": False,
}],
"load_balancing": {"health_check": {"mode": "off"}},
"routes": [{"id": "root", "path_prefix": "/"}],
}
if source_tls and certificate_id:
vhost["certificate_id"] = certificate_id
new_config["vhosts"].append(vhost)
created_ids.append(vid)
for host in normalized:
owners[host] = vid
decisions.append(Decision(item.id, "import", normalized, backend=origin, rustproxy_ids=created_ids, warnings=warnings))
return new_config, decisions
class RustProxyClient:
def __init__(self, base_url: str, *, ca_file: Path | None = None, insecure_tls: bool = False):
parsed = urllib.parse.urlsplit(base_url.rstrip("/"))
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.path not in {"", "/"}:
raise MigrationError("--rustproxy must be an http(s) origin, e.g. https://rustproxy.example:9090")
if parsed.query or parsed.fragment or parsed.username or parsed.password:
raise MigrationError("--rustproxy must not contain credentials, query or fragment")
self.base_url = urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, "", "", ""))
jar = http.cookiejar.CookieJar()
handlers: list[Any] = [urllib.request.HTTPCookieProcessor(jar)]
if parsed.scheme == "https":
if insecure_tls:
context = ssl._create_unverified_context() # Explicit operator opt-in for local/self-signed admin TLS.
else:
context = ssl.create_default_context(cafile=str(ca_file) if ca_file else None)
handlers.append(urllib.request.HTTPSHandler(context=context))
self.opener = urllib.request.build_opener(*handlers)
self.csrf = ""
def request(self, method: str, path: str, body: dict[str, Any] | None = None, *, authenticated_write: bool = False) -> Any:
data = None if body is None else json.dumps(body, separators=(",", ":")).encode()
headers = {"Accept": "application/json", "X-RustProxy-Request": "1"}
if data is not None:
headers["Content-Type"] = "application/json"
if method != "GET":
headers["Origin"] = self.base_url
if authenticated_write:
headers["X-CSRF-Token"] = self.csrf
request = urllib.request.Request(self.base_url + path, data=data, headers=headers, method=method)
try:
with self.opener.open(request, timeout=10) as response:
payload = response.read(1024 * 1024)
except urllib.error.HTTPError as error:
payload = error.read(65536)
try:
detail = json.loads(payload).get("error", payload.decode("utf-8", "replace"))
except Exception:
detail = payload.decode("utf-8", "replace")
raise MigrationError(f"RustProxy API {method} {path} failed with HTTP {error.code}: {detail}") from error
except (OSError, urllib.error.URLError) as error:
raise MigrationError(f"RustProxy API connection failed: {error}") from error
try:
return json.loads(payload)
except json.JSONDecodeError as error:
raise MigrationError(f"RustProxy API returned invalid JSON for {path}") from error
def login(self, username: str, password: str) -> None:
info = self.request("POST", "/api/auth/login", {"username": username, "password": password})
token = info.get("csrf_token") if isinstance(info, dict) else None
if not isinstance(token, str) or not token:
session = self.request("GET", "/api/auth/session")
token = session.get("csrf_token") if isinstance(session, dict) else None
if not isinstance(token, str) or not token:
raise MigrationError("RustProxy login succeeded but no CSRF token was returned")
self.csrf = token
def config(self) -> dict[str, Any]:
value = self.request("GET", "/api/config")
if not isinstance(value, dict) or not isinstance(value.get("revision"), int) or not isinstance(value.get("config"), dict):
raise MigrationError("unexpected RustProxy /api/config response")
return value
def apply(self, revision: int, config: dict[str, Any]) -> dict[str, Any]:
value = self.request("PUT", "/api/config", {"expected_revision": revision, "config": config}, authenticated_write=True)
if not isinstance(value, dict):
raise MigrationError("unexpected RustProxy PUT /api/config response")
return value
def write_private_json(path: Path, value: dict[str, Any]) -> None:
path = path.expanduser()
path.parent.mkdir(parents=True, exist_ok=True)
if path.is_symlink():
raise MigrationError(f"refusing to write migration plan through symlink: {path}")
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
if hasattr(os, "O_NOFOLLOW"):
flags |= os.O_NOFOLLOW
fd = os.open(path, flags, 0o600)
try:
os.fchmod(fd, 0o600)
payload = (json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode("utf-8")
with os.fdopen(fd, "wb", closefd=False) as stream:
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
finally:
os.close(fd)
def split_ids(values: list[str] | None) -> list[str] | None:
if values is None:
return None
out: list[str] = []
for value in values:
for item in value.split(","):
item = item.strip()
if item and item not in out:
out.append(item)
return out
def print_report(decisions: list[Decision]) -> None:
imported = sum(1 for d in decisions if d.action == "import")
skipped = sum(1 for d in decisions if d.action == "skip")
print(f"NPM proxy hosts: {len(decisions)}; import: {imported}; skip: {skipped}")
for decision in decisions:
print(f"\n[{decision.action.upper()}] NPM #{decision.npm_id}")
if decision.hosts:
print(" hosts: " + ", ".join(decision.hosts))
if decision.backend:
print(" backend: " + decision.backend)
if decision.rustproxy_ids:
print(" ids: " + ", ".join(decision.rustproxy_ids))
if decision.reason:
print(" reason: " + decision.reason)
for warning in decision.warnings or []:
print(" warning: " + warning)
from migrate_npm_core import * # stable migration/model API
from migrate_npm_api import * # stable administration API helpers
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
@@ -585,5 +70,6 @@ def main() -> None:
parser.exit(1, f"Migration failed: {error}\n")
if __name__ == "__main__":
main()
+124
View File
@@ -0,0 +1,124 @@
"""RustProxy administration API helpers for NPM migration."""
from __future__ import annotations
from migrate_npm_core import *
class RustProxyClient:
def __init__(self, base_url: str, *, ca_file: Path | None = None, insecure_tls: bool = False):
parsed = urllib.parse.urlsplit(base_url.rstrip("/"))
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.path not in {"", "/"}:
raise MigrationError("--rustproxy must be an http(s) origin, e.g. https://rustproxy.example:9090")
if parsed.query or parsed.fragment or parsed.username or parsed.password:
raise MigrationError("--rustproxy must not contain credentials, query or fragment")
self.base_url = urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, "", "", ""))
jar = http.cookiejar.CookieJar()
handlers: list[Any] = [urllib.request.HTTPCookieProcessor(jar)]
if parsed.scheme == "https":
if insecure_tls:
context = ssl._create_unverified_context() # Explicit operator opt-in for local/self-signed admin TLS.
else:
context = ssl.create_default_context(cafile=str(ca_file) if ca_file else None)
handlers.append(urllib.request.HTTPSHandler(context=context))
self.opener = urllib.request.build_opener(*handlers)
self.csrf = ""
def request(self, method: str, path: str, body: dict[str, Any] | None = None, *, authenticated_write: bool = False) -> Any:
data = None if body is None else json.dumps(body, separators=(",", ":")).encode()
headers = {"Accept": "application/json", "X-RustProxy-Request": "1"}
if data is not None:
headers["Content-Type"] = "application/json"
if method != "GET":
headers["Origin"] = self.base_url
if authenticated_write:
headers["X-CSRF-Token"] = self.csrf
request = urllib.request.Request(self.base_url + path, data=data, headers=headers, method=method)
try:
with self.opener.open(request, timeout=10) as response:
payload = response.read(1024 * 1024)
except urllib.error.HTTPError as error:
payload = error.read(65536)
try:
detail = json.loads(payload).get("error", payload.decode("utf-8", "replace"))
except Exception:
detail = payload.decode("utf-8", "replace")
raise MigrationError(f"RustProxy API {method} {path} failed with HTTP {error.code}: {detail}") from error
except (OSError, urllib.error.URLError) as error:
raise MigrationError(f"RustProxy API connection failed: {error}") from error
try:
return json.loads(payload)
except json.JSONDecodeError as error:
raise MigrationError(f"RustProxy API returned invalid JSON for {path}") from error
def login(self, username: str, password: str) -> None:
info = self.request("POST", "/api/auth/login", {"username": username, "password": password})
token = info.get("csrf_token") if isinstance(info, dict) else None
if not isinstance(token, str) or not token:
session = self.request("GET", "/api/auth/session")
token = session.get("csrf_token") if isinstance(session, dict) else None
if not isinstance(token, str) or not token:
raise MigrationError("RustProxy login succeeded but no CSRF token was returned")
self.csrf = token
def config(self) -> dict[str, Any]:
value = self.request("GET", "/api/config")
if not isinstance(value, dict) or not isinstance(value.get("revision"), int) or not isinstance(value.get("config"), dict):
raise MigrationError("unexpected RustProxy /api/config response")
return value
def apply(self, revision: int, config: dict[str, Any]) -> dict[str, Any]:
value = self.request("PUT", "/api/config", {"expected_revision": revision, "config": config}, authenticated_write=True)
if not isinstance(value, dict):
raise MigrationError("unexpected RustProxy PUT /api/config response")
return value
def write_private_json(path: Path, value: dict[str, Any]) -> None:
path = path.expanduser()
path.parent.mkdir(parents=True, exist_ok=True)
if path.is_symlink():
raise MigrationError(f"refusing to write migration plan through symlink: {path}")
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
if hasattr(os, "O_NOFOLLOW"):
flags |= os.O_NOFOLLOW
fd = os.open(path, flags, 0o600)
try:
os.fchmod(fd, 0o600)
payload = (json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode("utf-8")
with os.fdopen(fd, "wb", closefd=False) as stream:
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
finally:
os.close(fd)
def split_ids(values: list[str] | None) -> list[str] | None:
if values is None:
return None
out: list[str] = []
for value in values:
for item in value.split(","):
item = item.strip()
if item and item not in out:
out.append(item)
return out
def print_report(decisions: list[Decision]) -> None:
imported = sum(1 for d in decisions if d.action == "import")
skipped = sum(1 for d in decisions if d.action == "skip")
print(f"NPM proxy hosts: {len(decisions)}; import: {imported}; skip: {skipped}")
for decision in decisions:
print(f"\n[{decision.action.upper()}] NPM #{decision.npm_id}")
if decision.hosts:
print(" hosts: " + ", ".join(decision.hosts))
if decision.backend:
print(" backend: " + decision.backend)
if decision.rustproxy_ids:
print(" ids: " + ", ".join(decision.rustproxy_ids))
if decision.reason:
print(" reason: " + decision.reason)
for warning in decision.warnings or []:
print(" warning: " + warning)
+403
View File
@@ -0,0 +1,403 @@
#!/usr/bin/env python3
"""Migrate Nginx Proxy Manager proxy hosts from SQLite into RustProxy.
The migrator intentionally focuses on proxy-host identity and upstream routing. It
never imports certificate/key material from NPM. NPM hosts that used TLS are attached
to RustProxy's TLS listener and use the RustProxy default certificate unless a named
certificate is supplied explicitly.
Run without --apply first. The script logs in to the RustProxy administration API,
reads the current revision, builds a merged configuration and prints a review report.
Only --apply performs the revisioned PUT /api/config.
"""
from __future__ import annotations
import argparse
from dataclasses import dataclass, asdict
import getpass
import http.cookiejar
import ipaddress
import json
import os
from pathlib import Path
import re
import sqlite3
import ssl
import sys
from typing import Any
import urllib.error
import urllib.parse
import urllib.request
MAX_CONFIG_BYTES = 262_144
class MigrationError(RuntimeError):
pass
@dataclass(frozen=True)
class NpmProxyHost:
id: int
domain_names: list[str]
forward_scheme: str
forward_host: str
forward_port: int
allow_websocket_upgrade: bool
http2_support: bool
enabled: bool
certificate_id: int
ssl_forced: bool
access_list_id: int
advanced_config: str
locations: list[Any]
caching_enabled: bool
block_exploits: bool
meta: dict[str, Any]
@dataclass
class Decision:
npm_id: int
action: str
hosts: list[str]
backend: str | None = None
rustproxy_ids: list[str] | None = None
warnings: list[str] | None = None
reason: str | None = None
def json_value(raw: Any, fallback: Any) -> Any:
if raw is None or raw == "":
return fallback
if isinstance(raw, (list, dict)):
return raw
try:
return json.loads(raw)
except (TypeError, json.JSONDecodeError):
return fallback
def bool_value(raw: Any) -> bool:
if isinstance(raw, str):
return raw.strip().lower() in {"1", "true", "yes", "on"}
return bool(raw)
def normalize_host_pattern(value: str) -> str:
value = value.strip().lower().rstrip(".")
wildcard = value.startswith("*.")
base = value[2:] if wildcard else value
if not base or len(base) > 253 or "*" in base or ":" in base or "/" in base:
raise MigrationError(f"invalid hostname {value!r}")
labels = base.split(".")
if wildcard and len(labels) < 2:
raise MigrationError(f"invalid wildcard hostname {value!r}")
for label in labels:
if not label or len(label) > 63 or label.startswith("-") or label.endswith("-"):
raise MigrationError(f"invalid hostname {value!r}")
if not re.fullmatch(r"[a-z0-9-]+", label):
raise MigrationError(f"hostname must be ASCII/punycode: {value!r}")
return ("*." if wildcard else "") + base
def sql_columns(connection: sqlite3.Connection, table: str) -> set[str]:
return {str(row[1]) for row in connection.execute(f"PRAGMA table_info({table})")}
def load_npm_hosts(database: Path) -> list[NpmProxyHost]:
if database.is_symlink() or not database.is_file():
raise MigrationError(f"NPM SQLite database not found or unsafe: {database}")
uri = f"file:{urllib.parse.quote(str(database.resolve()))}?mode=ro"
connection = sqlite3.connect(uri, uri=True)
connection.row_factory = sqlite3.Row
try:
tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")}
if "proxy_host" not in tables:
raise MigrationError("NPM database does not contain the proxy_host table")
columns = sql_columns(connection, "proxy_host")
required = {"id", "domain_names", "forward_scheme", "forward_host", "forward_port"}
missing = sorted(required - columns)
if missing:
raise MigrationError("NPM proxy_host table is missing required columns: " + ", ".join(missing))
wanted = [
"id", "domain_names", "forward_scheme", "forward_host", "forward_port",
"allow_websocket_upgrade", "http2_support", "enabled", "certificate_id",
"ssl_forced", "access_list_id", "advanced_config", "locations",
"caching_enabled", "block_exploits", "meta", "is_deleted",
]
select = ", ".join(name for name in wanted if name in columns)
where = " WHERE is_deleted = 0" if "is_deleted" in columns else ""
rows = connection.execute(f"SELECT {select} FROM proxy_host{where} ORDER BY id").fetchall()
finally:
connection.close()
result: list[NpmProxyHost] = []
for row in rows:
data = dict(row)
domains_raw = json_value(data.get("domain_names"), [])
if not isinstance(domains_raw, list):
domains_raw = []
domains: list[str] = []
for item in domains_raw:
if isinstance(item, str) and item.strip():
domains.append(item.strip())
locations = json_value(data.get("locations"), [])
if not isinstance(locations, list):
locations = []
meta = json_value(data.get("meta"), {})
if not isinstance(meta, dict):
meta = {}
try:
port = int(data.get("forward_port"))
except (TypeError, ValueError):
port = 0
result.append(NpmProxyHost(
id=int(data["id"]),
domain_names=domains,
forward_scheme=str(data.get("forward_scheme") or "").strip().lower(),
forward_host=str(data.get("forward_host") or "").strip(),
forward_port=port,
allow_websocket_upgrade=bool_value(data.get("allow_websocket_upgrade", False)),
http2_support=bool_value(data.get("http2_support", True)),
enabled=bool_value(data.get("enabled", True)),
certificate_id=int(data.get("certificate_id") or 0),
ssl_forced=bool_value(data.get("ssl_forced", False)),
access_list_id=int(data.get("access_list_id") or 0),
advanced_config=str(data.get("advanced_config") or ""),
locations=locations,
caching_enabled=bool_value(data.get("caching_enabled", False)),
block_exploits=bool_value(data.get("block_exploits", False)),
meta=meta,
))
return result
def backend_url(item: NpmProxyHost) -> str:
if item.forward_scheme not in {"http", "https"}:
raise MigrationError(f"unsupported upstream scheme {item.forward_scheme!r}")
if not 1 <= item.forward_port <= 65535:
raise MigrationError(f"invalid upstream port {item.forward_port}")
host = item.forward_host.strip()
if not host or any(ch.isspace() for ch in host) or "/" in host or "@" in host:
raise MigrationError(f"invalid upstream host {host!r}")
if host.startswith("[") and host.endswith("]"):
rendered = host
else:
try:
ip = ipaddress.ip_address(host)
rendered = f"[{ip}]" if ip.version == 6 else str(ip)
except ValueError:
if ":" in host:
raise MigrationError(f"invalid upstream host {host!r}")
rendered = host
return f"{item.forward_scheme}://{rendered}:{item.forward_port}"
def slug(value: str) -> str:
value = value.lower().replace("*.", "wildcard-")
value = re.sub(r"[^a-z0-9_-]+", "-", value).strip("-_")
return value[:40] or "host"
def unique_id(base: str, used: set[str]) -> str:
base = re.sub(r"[^A-Za-z0-9_-]+", "-", base)[:64].strip("-_") or "npm"
candidate = base
index = 2
while candidate in used:
suffix = f"-{index}"
candidate = base[:64 - len(suffix)] + suffix
index += 1
used.add(candidate)
return candidate
def listener_inventory(config: dict[str, Any]) -> tuple[list[str], list[str]]:
http_ids: list[str] = []
https_ids: list[str] = []
for entry in ((config.get("listeners") or {}).get("endpoints") or []):
if not isinstance(entry, dict) or not entry.get("enabled", True):
continue
listener_id = str(entry.get("id") or "").strip()
binding = entry.get("binding") or {}
if not listener_id or not isinstance(binding, dict):
continue
(https_ids if binding.get("tls") else http_ids).append(listener_id)
return http_ids, https_ids
def selected_tls_ports(config: dict[str, Any], listener_ids: list[str]) -> set[int]:
selected = set(listener_ids)
ports: set[int] = set()
for entry in ((config.get("listeners") or {}).get("endpoints") or []):
if not isinstance(entry, dict) or str(entry.get("id") or "") not in selected:
continue
binding = entry.get("binding") or {}
if not isinstance(binding, dict) or not binding.get("tls"):
continue
for address in [binding.get("address"), *(entry.get("additional_addresses") or [])]:
if not isinstance(address, str) or ":" not in address:
raise MigrationError(f"invalid RustProxy listener address for Force SSL: {address!r}")
try:
port = int(address.rsplit(":", 1)[1])
except (TypeError, ValueError) as error:
raise MigrationError(f"invalid RustProxy listener address for Force SSL: {address!r}") from error
if not 1 <= port <= 65535:
raise MigrationError(f"invalid RustProxy listener port for Force SSL: {address!r}")
ports.add(port)
return ports
def existing_host_owners(config: dict[str, Any]) -> dict[str, str]:
result: dict[str, str] = {}
for vhost in config.get("vhosts") or []:
if not isinstance(vhost, dict):
continue
vid = str(vhost.get("id") or "?")
frontend = vhost.get("frontend") or {}
for raw in frontend.get("hosts") or []:
if isinstance(raw, str):
try:
result[normalize_host_pattern(raw)] = vid
except MigrationError:
pass
return result
def warnings_for(item: NpmProxyHost, tls_verify: bool) -> list[str]:
warnings: list[str] = []
if item.access_list_id:
warnings.append(f"NPM access_list_id={item.access_list_id} was not imported; attach an ACL Profile manually")
if item.advanced_config.strip():
warnings.append("NPM advanced_config was not imported")
if item.locations:
warnings.append(f"{len(item.locations)} NPM custom location(s) were not imported")
if item.caching_enabled:
warnings.append("NPM caching_enabled was not mapped; configure RustProxy cache policy separately")
if item.block_exploits:
warnings.append("NPM block_exploits was not mapped")
if item.ssl_forced:
warnings.append("NPM ssl_forced is mapped to RustProxy Force SSL (HTTP -> HTTPS 308 redirect)")
if item.forward_scheme == "https" and tls_verify:
warnings.append("HTTPS upstream verification was explicitly enabled; provide a matching DNS name/private CA when needed")
if item.meta.get("nginx_err"):
warnings.append("NPM meta.nginx_err is set; review the source host before cutover")
return warnings
def build_migration(
npm_hosts: list[NpmProxyHost],
config: dict[str, Any],
*,
http_listener_ids: list[str] | None = None,
https_listener_ids: list[str] | None = None,
certificate_id: str | None = None,
force_https: bool = False,
npm_compatible_backend_tls: bool = False,
) -> tuple[dict[str, Any], list[Decision]]:
new_config = json.loads(json.dumps(config))
new_config.setdefault("vhosts", [])
discovered_http, discovered_https = listener_inventory(new_config)
http_ids = http_listener_ids if http_listener_ids is not None else discovered_http[:1]
https_ids = https_listener_ids if https_listener_ids is not None else discovered_https[:1]
unknown_http = [value for value in http_ids if value not in discovered_http]
unknown_https = [value for value in https_ids if value not in discovered_https]
if unknown_http:
raise MigrationError("HTTP listener ID is missing, disabled or TLS-enabled: " + ", ".join(unknown_http))
if unknown_https:
raise MigrationError("HTTPS listener ID is missing, disabled or not TLS-enabled: " + ", ".join(unknown_https))
if not http_ids and not https_ids:
raise MigrationError("RustProxy config has no enabled HTTP/TLS listener to attach imported vhosts")
https_ports = selected_tls_ports(new_config, https_ids)
if certificate_id:
known_certificates = {str(item.get("id")) for item in ((new_config.get("tls") or {}).get("certificates") or []) if isinstance(item, dict)}
if certificate_id not in known_certificates:
raise MigrationError(f"RustProxy certificate ID not found: {certificate_id}")
owners = existing_host_owners(new_config)
used_ids = {str(v.get("id")) for v in new_config.get("vhosts") or [] if isinstance(v, dict)}
decisions: list[Decision] = []
for item in npm_hosts:
warnings = warnings_for(item, False)
if not item.domain_names:
decisions.append(Decision(item.id, "skip", [], warnings=warnings, reason="no domain_names"))
continue
try:
normalized = []
seen = set()
for raw in item.domain_names:
host = normalize_host_pattern(raw)
if host not in seen:
normalized.append(host)
seen.add(host)
origin = backend_url(item)
except MigrationError as error:
decisions.append(Decision(item.id, "skip", item.domain_names, warnings=warnings, reason=str(error)))
continue
conflicts = [(host, owners[host]) for host in normalized if host in owners]
if conflicts:
reason = "hostname already owned in RustProxy: " + ", ".join(f"{host} ({owner})" for host, owner in conflicts)
decisions.append(Decision(item.id, "skip", normalized, backend=origin, warnings=warnings, reason=reason))
continue
force_ssl = item.ssl_forced or force_https
source_tls = item.certificate_id > 0 or force_ssl
if force_ssl and not https_ids:
decisions.append(Decision(
item.id, "skip", normalized, backend=origin, warnings=warnings,
reason="NPM Force SSL requires an enabled RustProxy TLS listener",
))
continue
if force_ssl and len(https_ports) != 1:
decisions.append(Decision(
item.id, "skip", normalized, backend=origin, warnings=warnings,
reason="RustProxy Force SSL requires the selected TLS listeners to use one HTTPS port",
))
continue
if source_tls:
listeners = list(dict.fromkeys([*http_ids, *https_ids]))
else:
listeners = list(http_ids or https_ids)
if source_tls and not https_ids:
warnings.append("source host used TLS but RustProxy has no enabled TLS listener; imported on HTTP only")
listeners = list(http_ids)
if force_ssl and not http_ids:
warnings.append("Force SSL is preserved, but no HTTP listener is selected so there is currently no cleartext request to redirect")
if not listeners:
decisions.append(Decision(item.id, "skip", normalized, backend=origin, warnings=warnings, reason="no compatible RustProxy listener"))
continue
created_ids: list[str] = []
base = f"npm-{item.id}-{slug(normalized[0])}"
vid = unique_id(base, used_ids)
vhost: dict[str, Any] = {
"id": vid,
"enabled": item.enabled,
"force_ssl": force_ssl,
"forward_client_ip": True,
"websockets": item.allow_websocket_upgrade,
"http2_enabled": item.http2_support,
"http3_enabled": False,
"frontend": {"hosts": normalized, "listener_ids": listeners},
"backends": [{
"id": "origin",
"url": origin,
"allow_private": True,
"preserve_host": True,
"tls_verify": False,
}],
"load_balancing": {"health_check": {"mode": "off"}},
"routes": [{"id": "root", "path_prefix": "/"}],
}
if source_tls and certificate_id:
vhost["certificate_id"] = certificate_id
new_config["vhosts"].append(vhost)
created_ids.append(vid)
for host in normalized:
owners[host] = vid
decisions.append(Decision(item.id, "import", normalized, backend=origin, rustproxy_ids=created_ids, warnings=warnings))
return new_config, decisions
+45 -3
View File
@@ -11,6 +11,48 @@ LEGACY_AUTH = "4df9d473c9214365d91c83072f97aafab6ede4edcf4e1369b5c3679ded5cff85"
def check(root: Path, repair_known: bool = False) -> list[str]:
messages: list[str] = []
# 0.25.1 split-source migration: these monoliths are obsolete when the new
# bounded module tree is present. Preserve any local edits instead of silently
# deleting them when a source archive is extracted over an older checkout.
obsolete_split_files = (
"src/admin/web/app.js",
"src/admin/web/vhosts.js",
"src/web_server.rs",
)
split_markers = {
"src/admin/web/app.js": "src/admin/web/app/core.js",
"src/admin/web/vhosts.js": "src/admin/web/vhosts/core.js",
"src/web_server.rs": "src/web_server/mod.rs",
}
for relative in obsolete_split_files:
obsolete = root / relative
marker = root / split_markers[relative]
if not obsolete.exists() or not marker.is_file():
continue
if not repair_known:
raise ValueError(
f"Obsolete 0.25.0 source monolith {relative} remains beside the 0.25.1 split tree. "
"Run ./dev.sh repair (or any normal ./dev.sh command) to preserve it under .upgrade-backup/."
)
backup_root = root / ".upgrade-backup" / "0.25.1-split"
target = backup_root / relative
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists():
# Never overwrite a prior backup; if the bytes match, the current stale
# copy can simply be removed. Otherwise force the operator to reconcile it.
if target.read_bytes() != obsolete.read_bytes():
raise ValueError(
f"Backup target {target.relative_to(root)} already contains different data; "
f"move {relative} outside the source tree manually before rebuilding."
)
obsolete.unlink()
else:
obsolete.rename(target)
messages.append(
f"Preserved obsolete {relative} as {target.relative_to(root)}"
)
legacy = root / "src/auth.rs"
if legacy.exists() and (root / "src/auth/mod.rs").is_file():
if legacy.is_symlink() or hashlib.sha256(legacy.read_bytes()).hexdigest() != LEGACY_AUTH:
@@ -31,12 +73,12 @@ def check(root: Path, repair_known: bool = False) -> list[str]:
flat = module.parent.with_suffix(".rs")
if flat.is_file():
raise ValueError(f"Conflicting Rust modules: {flat.relative_to(root)} and {module.relative_to(root)}")
for path in ["src/auth/mod.rs", "src/auth/store.rs", "src/upstream.rs", "src/tls.rs", "src/proxy/websocket.rs", "src/cache/mod.rs", "src/cache/model.rs", "src/cache/lifecycle.rs", "src/cache/cold.rs", "src/cache/dashboard.rs", "src/cache/tickets.rs", "src/cache/lookup.rs", "src/cache/policy.rs", "src/cache/fill.rs", "src/cache/tests.rs", "src/settings.rs", "src/compression.rs", "src/diagnostics.rs", "src/traffic.rs", "src/system_stats/mod.rs", "src/system_stats/linux.rs", "src/vhost.rs", "src/web_server.rs", "src/balance.rs", "src/security.rs", "src/admin/web/vhosts.js", "src/admin/web/web-server.js", "src/admin/web/streams.js", "src/admin/web/host404.js", "src/admin/web/management.js", "src/pages.rs", "scripts/mock_server.py", "scripts/mock_auth.py", "src/forwarding.rs", "src/tinyauth.rs", "src/http3.rs", "src/stream.rs", "src/admin/web/integrations.js", "src/admin/web/traffic-panel.js", "src/prometheus.rs", "src/admin/web/settings.js", "src/admin/web/docker.js", "src/docker.rs", "docker-agent/main.go", "docker-agent/main_test.go", "docker-agent/rustproxy-docker-agent.service", "docker-agent/install.sh", "docs/DOCKER_DISCOVERY.md", "docs/WEB_SERVER.md", "docs/PHP_FPM.md", "docs/README.md", "docs/TODO.md", "docs/V0.25.0_AUDIT.md", "docs/CHANGELOG_TECHNICAL_ARCHIVE.md", "docs/ROADMAP_HISTORY.md", "scripts/docs_check.py", "scripts/web_server_contracts.py", "scripts/acme_provider.py", "src/basic_auth.rs", "scripts/deploy.py", "scripts/migrate_npm.py", "docs/NPM_MIGRATION.md", "docs/STREAMS.md", "install.sh", "install-debian13.sh", "update.sh", "build.rs", "scripts/swagger_assets.py", "src/admin/documentation.rs", "src/admin/openapi.json", "langs/en_US.json", "langs/pl_pl.json", "Dockerfile", ".dockerignore", "docs/CONTAINER.md", "docs/PROMETHEUS.md", "docs/NOT_FOUND_HOSTS.md", "debian/control", "debian/rules", "debian/rustproxy.service", "debian/rustproxy.logrotate", "grafana/rustproxy-overview.json"]:
for path in ["src/auth/mod.rs", "src/auth/store.rs", "src/upstream.rs", "src/tls.rs", "src/proxy/websocket.rs", "src/cache/mod.rs", "src/cache/model.rs", "src/cache/lifecycle.rs", "src/cache/cold.rs", "src/cache/dashboard.rs", "src/cache/tickets.rs", "src/cache/lookup.rs", "src/cache/policy.rs", "src/cache/fill.rs", "src/cache/tests.rs", "src/settings.rs", "src/compression.rs", "src/diagnostics.rs", "src/traffic.rs", "src/system_stats/mod.rs", "src/system_stats/linux.rs", "src/vhost.rs", "src/web_server/mod.rs", "src/web_server/config.rs", "src/web_server/filesystem.rs", "src/web_server/security.rs", "src/web_server/response.rs", "src/web_server/request.rs", "src/web_server/routing.rs", "src/web_server/fastcgi.rs", "src/web_server/htaccess.rs", "src/web_server/serve.rs", "src/web_server/tests.rs", "src/balance.rs", "src/security.rs", "src/admin/web/vhosts/core.js", "src/admin/web/vhosts/editor.js", "src/admin/web/vhosts/render.js", "src/admin/web/web-server.js", "src/admin/web/streams.js", "src/admin/web/host404.js", "src/admin/web/app/core.js", "src/admin/web/app/runtime.js", "src/admin/web/app/settings.js", "src/admin/web/management.js", "src/pages.rs", "scripts/mock_server.py", "scripts/mock_auth.py", "src/forwarding.rs", "src/tinyauth.rs", "src/http3.rs", "src/stream.rs", "src/admin/web/integrations.js", "src/admin/web/traffic-panel.js", "src/prometheus.rs", "src/admin/web/settings.js", "src/admin/web/docker.js", "src/docker.rs", "docker-agent/main.go", "docker-agent/model.go", "docker-agent/docker.go", "docker-agent/network.go", "docker-agent/certificate.go", "docker-agent/server.go", "docker-agent/main_test.go", "docker-agent/rustproxy-docker-agent.service", "docker-agent/install.sh", "docs/DOCKER_DISCOVERY.md", "docs/WEB_SERVER.md", "docs/PHP_FPM.md", "docs/README.md", "docs/TODO.md", "docs/V0.25.1_AUDIT.md", "docs/CHANGELOG_TECHNICAL_ARCHIVE.md", "docs/ROADMAP_HISTORY.md", "scripts/docs_check.py", "scripts/web_server_contracts.py", "scripts/rust_source.py", "scripts/rust_module_contracts.py", "scripts/source_module_contracts.py", "scripts/acme_provider.py", "src/basic_auth.rs", "scripts/deploy.py", "scripts/deploy_core.py", "scripts/deploy_runtime.py", "scripts/migrate_npm.py", "scripts/migrate_npm_core.py", "scripts/migrate_npm_api.py", "docs/NPM_MIGRATION.md", "docs/STREAMS.md", "install.sh", "install-debian13.sh", "update.sh", "build.rs", "scripts/swagger_assets.py", "src/admin/documentation.rs", "src/admin/openapi.json", "langs/en_US.json", "langs/pl_pl.json", "Dockerfile", ".dockerignore", "docs/CONTAINER.md", "docs/PROMETHEUS.md", "docs/NOT_FOUND_HOSTS.md", "debian/control", "debian/rules", "debian/rustproxy.service", "debian/rustproxy.logrotate", "grafana/rustproxy-overview.json"]:
if not (root / path).is_file():
raise ValueError(f"Incomplete RustProxy 0.25.0 source tree: missing {path}")
raise ValueError(f"Incomplete RustProxy 0.25.1 source tree: missing {path}")
text = (root / "src/lib.rs").read_text(encoding="utf-8")
if '#[path = "auth/mod.rs"]' not in text:
raise ValueError("src/lib.rs is not the 0.25.0 entry point; extract the complete package, not individual files.")
raise ValueError("src/lib.rs is not the 0.25.1 entry point; extract the complete package, not individual files.")
return messages
+126 -82
View File
@@ -21,11 +21,19 @@ import backup as backup_module
import deploy
import migrate_npm
import upgrade as upgrade_module
from rust_source import rust_source
from web_assets import read_app_js, read_vhosts_js
ROOT = Path(__file__).resolve().parents[1]
def read_cache_source() -> str:
return "\n".join(path.read_text(encoding="utf-8") for path in sorted((ROOT / "src/cache").glob("*.rs")))
return rust_source(ROOT, "src/cache/mod.rs")
def read_deploy_source() -> str:
return "\n".join((ROOT / name).read_text(encoding="utf-8") for name in ("scripts/deploy_core.py", "scripts/deploy_runtime.py", "scripts/deploy.py"))
def read_docker_agent_source() -> str:
return "\n".join(path.read_text(encoding="utf-8") for path in sorted((ROOT / "docker-agent").glob("*.go")) if not path.name.endswith("_test.go"))
def sql_and_backup_smoke(tmp: Path) -> None:
@@ -114,13 +122,18 @@ def upgrade_smoke(tmp: Path) -> None:
assert backup.is_dir()
assert (backup / "src/obsolete.rs").is_file()
assert not (target / "src/obsolete.rs").exists()
assert 'version = "0.25.0"' in (target / "Cargo.toml").read_text(encoding="utf-8")
assert 'version = "0.25.1"' in (target / "Cargo.toml").read_text(encoding="utf-8")
assert (target / "data/keep.txt").read_text(encoding="utf-8") == "preserve\n"
assert (target / ".env").read_text(encoding="utf-8") == "RUST_PROXY_DEMO=0\n"
assert (target / "langs/site.json").is_file()
assert (target / "docker-agent/main.go").is_file()
assert (target / "docker-agent/model.go").is_file()
assert (target / "docker-agent/docker.go").is_file()
assert (target / "docker-agent/network.go").is_file()
assert (target / "docker-agent/certificate.go").is_file()
assert (target / "docker-agent/server.go").is_file()
manifest = json.loads((backup / "upgrade.json").read_text(encoding="utf-8"))
assert manifest["to_version"] == "0.25.0"
assert manifest["to_version"] == "0.25.1"
assert manifest["removed_obsolete_runtime_state"] is True
assert not legacy.exists()
assert not (backup / "legacy-data").exists()
@@ -162,7 +175,7 @@ def deployment_gate_smoke(tmp: Path) -> None:
assert "TimeoutStopSec=120" in remote_unit
for marker in ["PrivateDevices=true", "ProtectKernelLogs=true", "RestrictNamespaces=true", "RestrictRealtime=true", "SystemCallArchitectures=native"]:
assert marker in remote_unit, marker
deploy_source = (ROOT / "scripts/deploy.py").read_text(encoding="utf-8")
deploy_source = read_deploy_source()
assert "def stop_for_managed_update" in deploy_source and ".managed-update" in deploy_source
assert "systemctl', 'stop', SERVICE" in deploy_source
assert "def ensure_service_data_access" in deploy_source
@@ -234,9 +247,9 @@ def npm_migration_smoke(tmp: Path) -> None:
def ui_contract_smoke() -> None:
html = (ROOT / "src/admin/web/index.html").read_text(encoding="utf-8")
app = (ROOT / "src/admin/web/app.js").read_text(encoding="utf-8")
app = read_app_js(ROOT)
settings = (ROOT / "src/admin/web/settings.js").read_text(encoding="utf-8")
vhosts = (ROOT / "src/admin/web/vhosts.js").read_text(encoding="utf-8")
vhosts = read_vhosts_js(ROOT)
streams = (ROOT / "src/admin/web/streams.js").read_text(encoding="utf-8")
redirects = (ROOT / "src/admin/web/redirects.js").read_text(encoding="utf-8")
host404 = (ROOT / "src/admin/web/host404.js").read_text(encoding="utf-8")
@@ -251,7 +264,15 @@ def ui_contract_smoke() -> None:
'id="traffic-scope"',
'id="traffic-chart-countries"',
'id="traffic-chart-client-ips"',
'id="traffic-client-ips"',
'id="traffic-tabs"',
'data-traffic-tab="overview"',
'data-traffic-tab="clients"',
'data-traffic-tab="requests"',
'data-traffic-tab="delivery"',
'id="traffic-chart-user-agents"',
'id="traffic-chart-paths"',
'id="traffic-chart-routes"',
'id="traffic-chart-volume"',
'id="log-capture-user-agent"',
'id="log-capture-country"',
'id="cache-auto-trim-enabled"',
@@ -316,6 +337,8 @@ def ui_contract_smoke() -> None:
for marker in required_html:
assert marker in html, marker
assert '/api/traffic' in traffic and 'rustProxyTrafficPanel' in traffic
for marker in ['function selectTab(', 'function renderDonut(', 'function renderBars(', 'renderBars("client-ips"', 'renderBars("user-agents"', 'renderBars("paths"', 'renderBars("routes"', 'renderBars("volume"']:
assert marker in traffic, marker
assert 'traffic: "/traffic"' in app and 'window.rustProxyTrafficPanel?.refresh()' in app
for marker in ['id="settings-tab-manual-config"', 'id="settings-page-manual-config"', 'id="config-editor"', 'id="save-json"', 'id="export-config"']:
assert marker in html, marker
@@ -439,13 +462,13 @@ def ui_contract_smoke() -> None:
def source_contract_smoke() -> None:
config = (ROOT / "src/config.rs").read_text(encoding="utf-8")
tls = (ROOT / "src/tls.rs").read_text(encoding="utf-8")
main = (ROOT / "src/main.rs").read_text(encoding="utf-8")
server = (ROOT / "src/server.rs").read_text(encoding="utf-8")
logging = (ROOT / "src/logging.rs").read_text(encoding="utf-8")
secret_store = (ROOT / "src/secret_store.rs").read_text(encoding="utf-8")
backup = (ROOT / "src/backup.rs").read_text(encoding="utf-8")
config = rust_source(ROOT, "src/config.rs")
tls = rust_source(ROOT, "src/tls.rs")
main = rust_source(ROOT, "src/main.rs")
server = rust_source(ROOT, "src/server.rs")
logging = rust_source(ROOT, "src/logging.rs")
secret_store = rust_source(ROOT, "src/secret_store.rs")
backup = rust_source(ROOT, "src/backup.rs")
acme_worker = (ROOT / "scripts/acme_provider.py").read_text(encoding="utf-8")
for feature_marker in ["AES_256_GCM", "master.key", "O_NOFOLLOW", "migrate_acme_credentials"]:
assert feature_marker in secret_store, feature_marker
@@ -455,36 +478,36 @@ def source_contract_smoke() -> None:
assert feature_marker in logging, feature_marker
assert "credential_ini_b64" in acme_worker and "credential_file.unlink" in acme_worker
cache = read_cache_source()
settings = (ROOT / "src/settings.rs").read_text(encoding="utf-8")
websocket = (ROOT / "src/proxy/websocket.rs").read_text(encoding="utf-8")
proxy = (ROOT / "src/proxy/mod.rs").read_text(encoding="utf-8")
settings = rust_source(ROOT, "src/settings.rs")
websocket = rust_source(ROOT, "src/proxy/websocket.rs")
proxy = rust_source(ROOT, "src/proxy/mod.rs")
assert '.insert("x-rustproxy-backend-time", value)' in proxy and '.insert("x-rust-proxy-backend-time", value)' not in proxy
pages = (ROOT / "src/pages.rs").read_text(encoding="utf-8")
admin = (ROOT / "src/admin/mod.rs").read_text(encoding="utf-8")
stream = (ROOT / "src/stream.rs").read_text(encoding="utf-8")
runtime = (ROOT / "src/runtime.rs").read_text(encoding="utf-8")
acl = (ROOT / "src/acl.rs").read_text(encoding="utf-8")
pages = rust_source(ROOT, "src/pages.rs")
admin = rust_source(ROOT, "src/admin/mod.rs")
stream = rust_source(ROOT, "src/stream.rs")
runtime = rust_source(ROOT, "src/runtime.rs")
acl = rust_source(ROOT, "src/acl.rs")
policies = (ROOT / "tests/policies.rs").read_text(encoding="utf-8")
http3 = (ROOT / "src/http3.rs").read_text(encoding="utf-8")
redirect = (ROOT / "src/redirect.rs").read_text(encoding="utf-8")
not_found = (ROOT / "src/not_found.rs").read_text(encoding="utf-8")
prometheus = (ROOT / "src/prometheus.rs").read_text(encoding="utf-8")
auth = (ROOT / "src/auth/mod.rs").read_text(encoding="utf-8")
auth_store = (ROOT / "src/auth/store.rs").read_text(encoding="utf-8")
upstream = (ROOT / "src/upstream.rs").read_text(encoding="utf-8")
vhosts_js = (ROOT / "src/admin/web/vhosts.js").read_text(encoding="utf-8")
docker_source = (ROOT / "src/docker.rs").read_text(encoding="utf-8")
http3 = rust_source(ROOT, "src/http3.rs")
redirect = rust_source(ROOT, "src/redirect.rs")
not_found = rust_source(ROOT, "src/not_found.rs")
prometheus = rust_source(ROOT, "src/prometheus.rs")
auth = rust_source(ROOT, "src/auth/mod.rs")
auth_store = rust_source(ROOT, "src/auth/store.rs")
upstream = rust_source(ROOT, "src/upstream.rs")
vhosts_js = read_vhosts_js(ROOT)
docker_source = rust_source(ROOT, "src/docker.rs")
docker_js = (ROOT / "src/admin/web/docker.js").read_text(encoding="utf-8")
docker_agent = (ROOT / "docker-agent/main.go").read_text(encoding="utf-8")
docker_agent = read_docker_agent_source()
html = (ROOT / "src/admin/web/index.html").read_text(encoding="utf-8")
cache_panel_ui = (ROOT / "src/admin/web/cache-panel.js").read_text(encoding="utf-8")
transform_panel = (ROOT / "src/admin/web/transform-panel.js").read_text(encoding="utf-8")
app_js = (ROOT / "src/admin/web/app.js").read_text(encoding="utf-8")
app_js = read_app_js(ROOT)
# 0.24.1: shared library statistics, authenticated runtime metadata and Logging in Settings.
manifest = tomllib.loads((ROOT / "Cargo.toml").read_text(encoding="utf-8"))
system_stats = (ROOT / "src/system_stats/mod.rs").read_text(encoding="utf-8")
linux_stats = (ROOT / "src/system_stats/linux.rs").read_text(encoding="utf-8")
system_stats = rust_source(ROOT, "src/system_stats/mod.rs")
linux_stats = rust_source(ROOT, "src/system_stats/linux.rs")
settings_ui = (ROOT / "src/admin/web/settings.js").read_text(encoding="utf-8")
openapi_0241 = (ROOT / "src/admin/openapi.json").read_text(encoding="utf-8")
assert not (ROOT / "src/system_stats/bsd.rs").exists()
@@ -521,7 +544,7 @@ def source_contract_smoke() -> None:
assert marker in admin, marker
anonymous_block = admin.split("fn anonymous_allowed", 1)[1].split("const CERTIFICATE_API_KEY_HEADER", 1)[0]
assert '"/api/status"' not in anonymous_block
assert '"RuntimeInformation"' in openapi_0241 and '"version": "0.25.0"' in openapi_0241
assert '"RuntimeInformation"' in openapi_0241 and '"version": "0.25.1"' in openapi_0241
assert 'id="panel-logging"' not in html
for marker in ['id="settings-tab-logging"', 'id="settings-page-logging"', 'id="logging-form"', 'id="error-log-form"']:
assert marker in html, marker
@@ -567,7 +590,7 @@ def source_contract_smoke() -> None:
assert marker.lower() in audit_0232.lower(), marker
assert 'external certificate ownership model' in audit_0223.lower() and 'release gate' in audit_0223.lower()
update_check = (ROOT / "src/update_check.rs").read_text(encoding="utf-8")
update_check = rust_source(ROOT, "src/update_check.rs")
update_defaults = tomllib.loads((ROOT / "UPDATE_CHECK.toml").read_text(encoding="utf-8"))
assert update_defaults["repository_url"] == "https://git.linuxiarz.pl/gru/RustProxy"
assert update_defaults["version_url"].endswith("/raw/branch/master/Cargo.toml")
@@ -659,11 +682,11 @@ def source_contract_smoke() -> None:
html = (ROOT / 'src/admin/web/index.html').read_text(encoding='utf-8')
cache_panel_js = (ROOT / 'src/admin/web/cache-panel.js').read_text(encoding='utf-8')
management_js = (ROOT / 'src/admin/web/management.js').read_text(encoding='utf-8')
app_js = (ROOT / 'src/admin/web/app.js').read_text(encoding='utf-8')
vhosts_js = (ROOT / 'src/admin/web/vhosts.js').read_text(encoding='utf-8')
app_js = read_app_js(ROOT)
vhosts_js = read_vhosts_js(ROOT)
cache_rs = read_cache_source()
admin_rs = (ROOT / 'src/admin/mod.rs').read_text(encoding='utf-8')
logging_rs = (ROOT / 'src/logging.rs').read_text(encoding='utf-8')
admin_rs = rust_source(ROOT, 'src/admin/mod.rs')
logging_rs = rust_source(ROOT, 'src/logging.rs')
assert 'bindSettingsForm("pages-form", next => { next.pages = global.read(); }, "pages")' in management_js
assert '"pages" => {' in admin_rs and 'default_pages_section_accepts_the_standard_value_payload' in admin_rs
assert 'pub const LIVE_WINDOW: usize = 256;' in logging_rs and 'pub const HISTORY_CAPACITY: usize = 4096;' in logging_rs
@@ -673,10 +696,10 @@ def source_contract_smoke() -> None:
assert 'dialog.addEventListener("close", () => { if (pinning) return; expanded = null; paused = null;' in management_js
assert 'hash_scheme.as_bytes()' in cache_rs and 'cache_key_separates_frontend_scheme' in cache_rs
assert not (ROOT / 'src/cache.rs').exists() and (ROOT / 'src/cache/mod.rs').is_file()
cache_mod = (ROOT / 'src/cache/mod.rs').read_text(encoding='utf-8')
cache_mod = rust_source(ROOT, 'src/cache/mod.rs')
assert 'impl Cache {\n include!(' not in cache_mod
for fragment in ['lifecycle.rs', 'cold.rs', 'dashboard.rs', 'tickets.rs', 'lookup.rs']:
assert (ROOT / 'src/cache' / fragment).read_text(encoding='utf-8').lstrip().startswith('impl Cache {')
assert rust_source(ROOT, f'src/cache/{fragment}').lstrip().startswith('impl Cache {')
assert 'payload["transform_metrics"] = json!(state.transform_metrics.total())' in admin_rs
for marker in ['payload["transform_resources"]', 'payload["memory_accounting"]', 'non_cache_rss_includes_transform', 'cold_limit_bytes', 'cold_limit_entries']:
assert marker in admin_rs or marker in cache, marker
@@ -695,12 +718,12 @@ def source_contract_smoke() -> None:
assert marker in app_js, marker
assert 'envelope?.config.cache?.enabled === true && cacheToggle.value !== "off"' in vhosts_js
assert 'groups = d.domains || [], shownGroups = visibleDomains(data, groups)' in cache_panel_js and 'replaceChildren(...shownGroups.map(row =>' in cache_panel_js
balance_rs = (ROOT / 'src/balance.rs').read_text(encoding='utf-8')
balance_rs = rust_source(ROOT, 'src/balance.rs')
assert 'traffic_rate_drops_to_zero_after_an_idle_sample_window' in balance_rs and 'if delta == 0 {' in balance_rs and 'sample.rate = 0.0;' in balance_rs
all_admin_js = "\n".join(path.read_text(encoding='utf-8') for path in (ROOT / 'src/admin/web').glob('*.js'))
assert not re.search(r'(?<![A-Za-z])(?:window\.)?(?:alert|confirm)\s*\(', all_admin_js)
assert 'function appConfirm(message, options = {})' in app_js and 'function apiErrorMessage(path, response, value)' in app_js
deploy_py = (ROOT / 'scripts/deploy.py').read_text(encoding='utf-8')
deploy_py = read_deploy_source()
assert "SHELL_PATH_PROFILE = Path('/etc/profile.d/rustproxy-path.sh')" in deploy_py and 'install_shell_path(prefix)' in deploy_py and 'check_shell_path(prefix)' in deploy_py
assert 'grid-column: span 3' in ui_css and '.check>span {' in ui_css and 'transform: translateY(-1px)' in ui_css
assert '.cache-toolbar-status .check' in ui_css and '.docker-toolbar .check' in ui_css and '.docker-alias-field>span:first-child {' in ui_css and 'margin-bottom: 0' in ui_css and '.docker-alias-editor {' in ui_css and 'min-height: 38px' in ui_css and 'height: 30px' in ui_css
@@ -710,7 +733,7 @@ def source_contract_smoke() -> None:
screenshots = {path.name for path in (ROOT / 'docs/screenshots').iterdir() if path.is_file()}
assert 'README.md' in screenshots
assert screenshots <= {'README.md', 'login.png', 'location-headers.png'}
assert 'generated_vhost_inherits_global_server_header_policy' in (ROOT / 'src/docker.rs').read_text(encoding='utf-8')
assert 'generated_vhost_inherits_global_server_header_policy' in rust_source(ROOT, 'src/docker.rs')
assert '.docker-route-grid {' in ui_css and 'align-items: start' in ui_css
assert '.docker-acl-grid {' in ui_css and 'height: 38px' in ui_css
assert 'preserve.checked = assignment?.preserve_host !== false' in docker_js
@@ -724,7 +747,7 @@ def source_contract_smoke() -> None:
'pub targets_admin_listener: bool',
]:
assert marker in config, marker
vhost_source = (ROOT / "src/vhost.rs").read_text(encoding="utf-8")
vhost_source = rust_source(ROOT, "src/vhost.rs")
assert 'is_legacy_main_fallback' in vhost_source
assert "benign_tunnel_close" in websocket and "ConnectionReset" in websocket
assert 'StatusCode::UNAUTHORIZED => Some(include_str!("admin/web/error_401.html"))' in pages
@@ -744,7 +767,7 @@ def source_contract_smoke() -> None:
assert marker in proxy, marker
for marker in ["force_ssl_redirect", "force_ssl_location", "StatusCode::PERMANENT_REDIRECT", "https_redirect_port", "apply_upstream"]:
assert marker in proxy, marker
forwarding = (ROOT / "src/forwarding.rs").read_text(encoding="utf-8")
forwarding = rust_source(ROOT, "src/forwarding.rs")
for marker in ["pub fn apply_upstream", 'headers.remove("x-forwarded-for")', 'headers.remove("x-real-ip")', "upstream_can_withhold_client_ip_without_losing_origin_metadata"]:
assert marker in forwarding, marker
for marker in ["SINK_QUEUE", "FileSink", "LogstashSink", "LokiSink", "Limited::new"]:
@@ -784,7 +807,7 @@ def source_contract_smoke() -> None:
"matching_cold_key",
]:
assert marker in cache, marker
transform = (ROOT / "src/transform/mod.rs").read_text(encoding="utf-8")
transform = rust_source(ROOT, "src/transform/mod.rs")
for marker in [
"pub struct Settings", "pub struct Overrides",
"resolved.enabled = global.enabled && self.enabled",
@@ -805,20 +828,20 @@ def source_contract_smoke() -> None:
assert marker in transform, marker
assert 'collect bounded static response for transformation: {error}' in transform
assert '.context("collect bounded static response for transformation")' not in transform
lookup = (ROOT / 'src/cache/lookup.rs').read_text(encoding='utf-8')
model = (ROOT / 'src/cache/model.rs').read_text(encoding='utf-8')
css_minifier = (ROOT / 'src/transform/css.rs').read_text(encoding='utf-8')
lookup = rust_source(ROOT, 'src/cache/lookup.rs')
model = rust_source(ROOT, 'src/cache/model.rs')
css_minifier = rust_source(ROOT, 'src/transform/css.rs')
assert 'let Some(_read_lease) = self.try_reserve_transient' in lookup
assert 'struct TransientLease {\n _inner: Arc<TransientLeaseInner>,' in model
assert 'struct TransientLease(Arc<TransientLeaseInner>);' not in model
assert 'last_char = out.chars().last();' not in css_minifier
assert css_minifier.count("last_char = Some(' ');") == 1
assert "pub mod transform;" in (ROOT / "src/lib.rs").read_text(encoding="utf-8")
assert "pub mod transform;" in rust_source(ROOT, "src/lib.rs")
for name in ["js.rs", "css.rs"]:
minifier = (ROOT / "src/transform" / name).read_text(encoding="utf-8")
assert "Vec<char>" not in minifier and ".chars().collect" not in minifier
assert "into_bytes()" in minifier
balance = (ROOT / "src/balance.rs").read_text(encoding="utf-8")
balance = rust_source(ROOT, "src/balance.rs")
for marker in [
"pub struct PassiveHealth", "enabled: false", "pub passive_health: PassiveHealth",
"passive_health_is_opt_in_and_uses_separate_thresholds",
@@ -839,12 +862,12 @@ def source_contract_smoke() -> None:
]:
assert marker in cache, marker
assert 'impl Drop for Candidate' not in cache
diagnostics = (ROOT / "src/diagnostics.rs").read_text(encoding="utf-8") + (ROOT / "src/system_stats/mod.rs").read_text(encoding="utf-8") + (ROOT / "src/system_stats/linux.rs").read_text(encoding="utf-8")
diagnostics = rust_source(ROOT, "src/diagnostics.rs") + rust_source(ROOT, "src/system_stats/mod.rs") + rust_source(ROOT, "src/system_stats/linux.rs")
for marker in ['process_rss_anon_bytes', 'process_rss_file_bytes', 'process_rss_shmem_bytes', 'process_data_bytes', 'process_swap_bytes', 'RssAnon', 'RssFile', 'RssShmem', 'VmData', 'VmSwap']:
assert marker in diagnostics, marker
for marker in ['admin_session_passthrough', 'admin_control_plane', 'restore_admin_cookie', 'BEFORE returning a successful RFC 9220 response', 'websocket_phase("tunnel_ready")']:
assert marker in proxy, marker
headers_source = (ROOT / "src/proxy/headers.rs").read_text(encoding="utf-8")
headers_source = rust_source(ROOT, "src/proxy/headers.rs")
assert 'pub fn restore_admin_cookie' in headers_source and 'admin_cookie_can_be_restored_only_from_captured_fragment' in headers_source
assert "state.cache.purge(None)" not in proxy
assert "request.method() == Method::HEAD && cache_ticket.is_some()" in proxy
@@ -860,14 +883,14 @@ def source_contract_smoke() -> None:
assert marker in not_found, marker
assert "RedirectResponse" in proxy and "snapshot.redirect_on" in proxy
assert "#[serde(default)]\n pub tls_verify: bool" in config
upstream = (ROOT / "src/upstream.rs").read_text(encoding="utf-8")
upstream = rust_source(ROOT, "src/upstream.rs")
assert "BackendCertificateVerificationDisabled { supported_schemes }" in upstream
disabled_verifier = upstream.split("impl ServerCertVerifier for BackendCertificateVerificationDisabled", 1)[1].split("pub fn client(", 1)[0]
assert disabled_verifier.count("Ok(HandshakeSignatureValid::assertion())") == 2
assert "WebPkiServerVerifier" not in disabled_verifier
assert "stream_metrics" in runtime and "stream_metrics" in main
for marker in ["PrefixPriority", "RegexInsensitive", "LocationResponse", "pub rewrite: PathRewrite"]:
assert marker in (ROOT / "src/vhost.rs").read_text(encoding="utf-8"), marker
assert marker in rust_source(ROOT, "src/vhost.rs"), marker
for marker in ["rewritten_path", "nginx_style_location_precedence_exact_priority_regex_prefix", "location_rewrites_are_normalized_and_regex_captures_expand", "custom_location_response_is_validated"]:
assert marker in config, marker
for marker in ["local_location_response", "route.rewritten_path(&path)", "encoded_origin_path", "client_ip.to_string()"]:
@@ -885,17 +908,17 @@ def source_contract_smoke() -> None:
assert '<div class="edition">INDEPENDENT BY DESIGN <span>' not in html
assert '<div class="edition">INDEPENDENT BY DESIGN</div>' in html
assert "inspected_metadata" in admin and "certificate_metadata" in admin
balance = (ROOT / "src/balance.rs").read_text(encoding="utf-8")
balance = rust_source(ROOT, "src/balance.rs")
assert "max_concurrent_requests" in balance and "queue_timeout_ms" in balance and "select_retry_wait" in balance
for marker in ["dns_errors_total", "tls_errors_total", "response_header_timeouts_total", "last_failure_detail", "last_recovered_ms"]:
assert marker in balance, marker
tls_source = (ROOT / "src/tls.rs").read_text(encoding="utf-8")
tls_source = rust_source(ROOT, "src/tls.rs")
for marker in ["CertificateMetadata", "not_before_unix", "not_after_unix", "days_remaining", "expiry_status"]:
assert marker in tls_source, marker
assert '"/assets/redirects.js"' in admin and 'include_str!("web/redirects.js")' in admin and '.route("/redirects", get(public::index))' in admin
assert '/Ubuntu%2014.04%20LTS/' in acl and 'repeated_slashes_are_normalized_for_routing' in acl and 'percent_encoded_spaces_are_valid_request_paths' in policies and 'repeated_slashes_route_normally_but_reach_upstream_unchanged' in policies and 'local_location_response_uses_response_rules_without_opening_backend' in policies and 'local_location_response_still_respects_route_acl' in policies
db_source = (ROOT / "src/db.rs").read_text(encoding="utf-8")
acme_source = (ROOT / "src/acme.rs").read_text(encoding="utf-8")
db_source = rust_source(ROOT, "src/db.rs")
acme_source = rust_source(ROOT, "src/acme.rs")
assert "SCHEMA_VERSION: i64 = 7" in db_source and "certificate_renewals" in db_source and "0004_admin_preferences.sql" in db_source and "0005_certificate_api.sql" in db_source and "0006_certificate_api_credentials.sql" in db_source and "0007_secret_store.sql" in db_source
assert "load_renewal_observations" not in acme_source and "renewals.json" not in acme_source
for marker in ["pub kind: &'static str", "pub log_directory: String", "pub(crate) fn certbot_log_directory", 'kind: "issue"', '"renewal-dry-run"']:
@@ -903,28 +926,28 @@ def source_contract_smoke() -> None:
assert '"log_directories":log_directories' in admin and 'crate::acme::certbot_log_directory' in admin
assert "store.load_certificate_renewals().await?" in main and "state.acme.restore_renewals(renewal_observations)" in main
assert 'accept-remote-admin-risk' in main and 'hide = true' in main
settings_rust = (ROOT / 'src/settings.rs').read_text(encoding='utf-8')
settings_rust = rust_source(ROOT, 'src/settings.rs')
assert 'pub struct StartupLocks' in settings_rust and 'fixed by systemd/startup' in settings_rust
logging_rust = (ROOT / 'src/logging.rs').read_text(encoding='utf-8')
logging_rust = rust_source(ROOT, 'src/logging.rs')
assert 'pub per_vhost: bool' in logging_rust and 'default_vhost_log_directory' in logging_rust
pages_rust = (ROOT / 'src/pages.rs').read_text(encoding='utf-8')
pages_rust = rust_source(ROOT, 'src/pages.rs')
assert 'validate_local_response' in pages_rust and '(200..=599).contains(&status)' in pages_rust
assert 'Kind::Builtin' in pages_rust and 'builtin_application_page' in pages_rust
docker_rust = (ROOT / 'src/docker.rs').read_text(encoding='utf-8')
docker_rust = rust_source(ROOT, 'src/docker.rs')
for marker in ['pub poll_interval_seconds: u64', 'pub acl_profile_id: Option<String>', 'pub proxy_timeouts: crate::vhost::ProxyTimeouts', 'refresh(&self, settings: &Settings, force: bool)']:
assert marker in docker_rust, marker
docker_ui = (ROOT / 'src/admin/web/docker.js').read_text(encoding='utf-8')
for marker in ['docker-live-refresh', 'saveAssignment(agentId, row, publish)', 'docker-acl-profile', 'docker-proxy-read', 'dockerOpenApp', 'proxiedUrl(agent, hostname)']:
assert marker in docker_ui, marker
compression_source = (ROOT / 'src/compression.rs').read_text(encoding='utf-8')
metrics_source = (ROOT / 'src/metrics.rs').read_text(encoding='utf-8')
compression_source = rust_source(ROOT, 'src/compression.rs')
metrics_source = rust_source(ROOT, 'src/metrics.rs')
style_source = (ROOT / 'src/admin/web/style.css').read_text(encoding='utf-8')
app_source = (ROOT / 'src/admin/web/app.js').read_text(encoding='utf-8')
vhosts_ui = (ROOT / 'src/admin/web/vhosts.js').read_text(encoding='utf-8')
app_source = read_app_js(ROOT)
vhosts_ui = read_vhosts_js(ROOT)
for marker in ['pub enum AppliedEncoding', 'AppliedEncoding::Gzip', 'AppliedEncoding::Brotli', 'AppliedEncoding::Zstd', 'AppliedEncoding::Deflate']:
assert marker in compression_source, marker
assert 'quality_weights_can_select_each_enabled_codec' in compression_source
diagnostics_source = (ROOT / 'src/diagnostics.rs').read_text(encoding='utf-8') + (ROOT / 'src/system_stats/mod.rs').read_text(encoding='utf-8') + (ROOT / 'src/system_stats/linux.rs').read_text(encoding='utf-8')
diagnostics_source = rust_source(ROOT, 'src/diagnostics.rs') + rust_source(ROOT, 'src/system_stats/mod.rs') + rust_source(ROOT, 'src/system_stats/linux.rs')
for marker in ['pub struct NetworkInterfaceView', 'pub network_interfaces: Vec<NetworkInterfaceView>', 'sysinfo::Networks::new_with_refreshed_list', 'system.network_stats', 'collect_network_systemstat', 'normalize_network_interfaces', 'merge_network_interfaces']:
assert marker in diagnostics_source, marker
assert 'interfaces.truncate(64)' not in diagnostics_source, 'network inventory must not silently truncate interfaces'
@@ -942,14 +965,14 @@ def source_contract_smoke() -> None:
assert 'sparkline-h1' in (ROOT / 'src/admin/web/index.html').read_text(encoding='utf-8')
assert 'body[data-density="compact"]' in style_source and '--ui-control: 22px' in style_source
assert 'status.backend_pool_domains' in vhosts_ui and 'payload["backend_pool_domains"]' in admin
docker_source = (ROOT / 'src/docker.rs').read_text(encoding='utf-8')
docker_source = rust_source(ROOT, 'src/docker.rs')
management_ui = (ROOT / 'src/admin/web/management.js').read_text(encoding='utf-8')
for marker in ['function aliasEditor(', 'host-chip-editor docker-alias-editor', '.docker-force-ssl', '.docker-websockets', '.docker-cache-enabled', '.docker-passive-enabled', '.docker-passive-5xx', '.docker-agent-acl-profile']:
assert marker in docker_ui, marker
for marker in ['"force_ssl": assignment.force_ssl.unwrap_or(false)', '"websockets": assignment.websockets', '"cache": {"enabled": assignment.cache_enabled}', '"passive_health": assignment.passive_health.clone()', 'assignment.acl_profile_id.clone().or_else(|| agent.acl_profile_id.clone())']:
assert marker in docker_source, marker
assert 'agent.acl_profile_id === profile.id' in management_ui
vhost_source = (ROOT / 'src/vhost.rs').read_text(encoding='utf-8')
vhost_source = rust_source(ROOT, 'src/vhost.rs')
for marker in ['DEFAULT_PROXY_CONNECT_TIMEOUT_SECONDS: u64 = 10', 'DEFAULT_PROXY_RESPONSE_HEADER_TIMEOUT_SECONDS: u64 = 120', 'DEFAULT_PROXY_READ_TIMEOUT_SECONDS: u64 = 60', 'DEFAULT_PROXY_SEND_TIMEOUT_SECONDS: u64 = 60', 'proxy_timeouts_have_explicit_safe_defaults_but_keep_null_compatibility']:
assert marker in vhost_source, marker
assert 'defaultProxyTimeouts = () => ({ connect_timeout_seconds: 10, response_header_timeout_seconds: 120, read_timeout_seconds: 60, send_timeout_seconds: 60 })' in vhosts_ui
@@ -958,22 +981,22 @@ def source_contract_smoke() -> None:
assert 'docker-agent-status-badge' in style_source and 'justify-content: center' in style_source
assert '.docker-agent-grid>label' in style_source and '#docker-agent-add' in style_source and 'margin-top: 14px' in style_source
assert 'Bearer token:' in (ROOT / 'docker-agent/install.sh').read_text(encoding='utf-8')
tls_rust = (ROOT / 'src/tls.rs').read_text(encoding='utf-8')
tls_rust = rust_source(ROOT, 'src/tls.rs')
assert 'pub fn validate_certificate_removals' in tls_rust and 'pub fn effective_cipher_suites' in tls_rust
assert "renewals.json" not in main
upgrade_source = (ROOT / "scripts/upgrade.py").read_text(encoding="utf-8")
assert "removed_obsolete_runtime_state" in upgrade_source and "legacy-data" not in upgrade_source and "certs_dir.is_symlink()" in upgrade_source
assert '.managed-update' in main and "flush_hot_to_cold_for_update" in main
openapi = (ROOT / "src/admin/openapi.json").read_text(encoding="utf-8")
for marker in ['"version": "0.25.0"', '"memory_preset"', '"cold_preset"', '"memory_budget_bytes"', '"max_inflight"', '"preset"', '"session_days"', '"session_lifetime_seconds"', '"accept_local_listener_risk"', '"backend_header"', '"backend_timing_header"', '"cold_on_age"', '"match_type"', '"regex_insensitive"', '"NotFoundHost"', '"not_found_hosts"', '"PrometheusSettings"', '"allow_trusted_network_without_token"', '"include_docker_vhosts"', '"include_streams"', '"include_logging"']:
for marker in ['"version": "0.25.1"', '"memory_preset"', '"cold_preset"', '"memory_budget_bytes"', '"max_inflight"', '"preset"', '"session_days"', '"session_lifetime_seconds"', '"accept_local_listener_risk"', '"backend_header"', '"backend_timing_header"', '"cold_on_age"', '"match_type"', '"regex_insensitive"', '"NotFoundHost"', '"not_found_hosts"', '"PrometheusSettings"', '"allow_trusted_network_without_token"', '"include_docker_vhosts"', '"include_streams"', '"include_logging"']:
assert marker in openapi, marker
assert '"software_update"' in openapi and '"update_available"' in openapi
for marker in ['"/api/certificates/deploy"', '"CertificateDeploy"', '"CertificateDeployResult"', '"/api/certificates/reload"']:
assert marker in openapi, marker
for marker in ['"/api/certificate-api"', '"/api/certificate-api/{key_id}"', '"/api/certificates/{certificate_id}/reload"', '"CertificateReloadOneResult"', '"CertificateApiBearer"', '"CertificateApiKey"', '"/api/live-logs"']:
assert marker in openapi, marker
websocket_source = (ROOT / 'src/proxy/websocket.rs').read_text(encoding='utf-8')
proxy_source = (ROOT / 'src/proxy/mod.rs').read_text(encoding='utf-8')
websocket_source = rust_source(ROOT, 'src/proxy/websocket.rs')
proxy_source = rust_source(ROOT, 'src/proxy/mod.rs')
for marker in ['websocket_legacy_support', 'vhWebsocketLegacySupport', 'vhWebsocketLegacySupportHelp']:
assert marker in vhosts_ui or marker in openapi, marker
for marker in ['fn legacy_websocket_origin(', 'request.headers_mut().insert("origin", origin)', 'websocket_legacy_support && !websocket_upstream_h2']:
@@ -982,7 +1005,7 @@ def source_contract_smoke() -> None:
assert marker in websocket_source, marker
for marker in ['certificate_api_credentials', 'upsert_certificate_api_credential', 'revoke_certificate_api_credential', 'verify_certificate_api_credential']:
assert marker in db_source, marker
logging_source = (ROOT / 'src/logging.rs').read_text(encoding='utf-8')
logging_source = rust_source(ROOT, 'src/logging.rs')
for marker in ['LIVE_STREAM_CAPACITY', 'broadcast::Sender<LogEntry>', 'pub fn subscribe']:
assert marker in logging_source, marker
for marker in ['livelogs: bool', 'LiveLogFilters', 'live_logs_command', 'LocalAdminCertificateVerifier', 'detect_local_admin_scheme', 'admin.uri("/api/live-logs")']:
@@ -1005,14 +1028,14 @@ def source_contract_smoke() -> None:
assert '127.0.0.1:9090' in transport_doc and 'accept_local_listener_risk: true' in transport_doc
drop_impl = http3.split('impl Drop for ReceiveBody', 1)[1].split('#[cfg(test)]', 1)[0]
assert 'self.stream.stop_sending(' not in drop_impl
websocket_source = (ROOT / 'src/proxy/websocket.rs').read_text(encoding='utf-8')
websocket_source = rust_source(ROOT, 'src/proxy/websocket.rs')
for marker in ['h3::quinn::RecvStream', 'websocket_rfc9220_on(local)', 'run_h3', 'H3_REQUEST_CANCELLED']:
assert marker in http3 or marker in websocket_source, marker
assert 'upstream: Upgraded' in websocket_source
assert 'timeout(Duration::from_secs(10), upstream)' not in websocket_source
for marker in ['websocket_phase("tunnel_started")', 'websocket_phase("tunnel_closed")', 'h3_reset("H3_REQUEST_CANCELLED")']:
assert marker in websocket_source, marker
logging_source = (ROOT / 'src/logging.rs').read_text(encoding='utf-8')
logging_source = rust_source(ROOT, 'src/logging.rs')
for marker in ['http_version', 'transport', 'websocket_bootstrap', 'websocket_phase', 'websocket_error', 'h3_reset_code', 'upstream_status']:
assert marker in logging_source, marker
assert 'h3_quinn' not in http3 and 'h3_quinn' not in (ROOT / 'tests/http3.rs').read_text(encoding='utf-8')
@@ -1082,13 +1105,32 @@ def reproducible_archive_smoke(tmp: Path) -> None:
with zipfile.ZipFile(first) as archive:
names = archive.namelist()
assert "RustProxy/SHA256SUMS" in names
assert "RustProxy/scripts/rust_source.py" in names
assert "RustProxy/scripts/rust_module_contracts.py" in names
assert "RustProxy/scripts/source_module_contracts.py" in names
for name in names:
if name.startswith("RustProxy/") and name.endswith(".rs"):
assert len(archive.read(name).decode("utf-8").splitlines()) <= 500, name
if name.startswith("RustProxy/src/admin/web/") and name.endswith(".js"):
assert len(archive.read(name).decode("utf-8").splitlines()) <= 500, name
if name.startswith("RustProxy/docker-agent/") and name.endswith(".go"):
assert len(archive.read(name).decode("utf-8").splitlines()) <= 500, name
assert "RustProxy/src/admin/web/error_401.html" in names
assert "RustProxy/src/admin/web/error_404.html" in names
assert "RustProxy/docs/CUSTOM_LOCATIONS.md" in names
assert "RustProxy/docs/WEB_SERVER.md" in names
assert "RustProxy/docs/PHP_FPM.md" in names
assert "RustProxy/src/web_server.rs" in names
for module in ["mod.rs", "config.rs", "filesystem.rs", "security.rs", "response.rs", "request.rs", "routing.rs", "fastcgi.rs", "serve.rs", "htaccess.rs", "tests.rs"]:
assert f"RustProxy/src/web_server/{module}" in names
assert "RustProxy/src/web_server.rs" not in names
assert "RustProxy/src/admin/web/web-server.js" in names
for module in ["app/core.js", "app/runtime.js", "app/settings.js", "vhosts/core.js", "vhosts/editor.js", "vhosts/render.js"]:
assert f"RustProxy/src/admin/web/{module}" in names
assert "RustProxy/src/admin/web/app.js" not in names and "RustProxy/src/admin/web/vhosts.js" not in names
for module in ["model.go", "docker.go", "network.go", "certificate.go", "server.go", "main.go", "main_test.go"]:
assert f"RustProxy/docker-agent/{module}" in names
for module in ["deploy.py", "deploy_core.py", "deploy_runtime.py", "migrate_npm.py", "migrate_npm_core.py", "migrate_npm_api.py"]:
assert f"RustProxy/scripts/{module}" in names
assert "RustProxy/nagios/check_rustproxy.py" in names
assert "RustProxy/nagios/check_rustproxy_api" in names
assert "RustProxy/src/admin/web/host404.js" in names
@@ -1108,6 +1150,8 @@ def reproducible_archive_smoke(tmp: Path) -> None:
def main() -> None:
subprocess.run(["python3", str(ROOT / "scripts/rust_module_contracts.py")], check=True)
subprocess.run(["python3", str(ROOT / "scripts/source_module_contracts.py")], check=True)
with tempfile.TemporaryDirectory(prefix="rustproxy-release-smoke-") as directory:
tmp = Path(directory)
sql_and_backup_smoke(tmp / "sql")
@@ -1119,7 +1163,7 @@ def main() -> None:
source_contract_smoke()
container_contract_smoke()
debian_grafana_contract_smoke()
print("PASS: 0.25.0 SQLite/upgrade, local-response 404 hosts, Prometheus/Grafana diagnostics, Debian packaging, cache/dashboard/logging/Nagios, retained admin/H3/stream/TLS contracts and reproducible source archives.")
print("PASS: 0.25.1 SQLite/upgrade, local-response 404 hosts, Prometheus/Grafana diagnostics, Debian packaging, cache/dashboard/logging/Nagios, retained admin/H3/stream/TLS contracts and reproducible source archives.")
if __name__ == "__main__":
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""Static structure contracts for the split Rust source tree.
These checks are intentionally compiler-independent. They prevent the module split
from regressing into large monoliths, keep unit tests out of implementation files,
and catch the class of dangling-attribute/broken-include mistakes that can otherwise
surface only at cargo check time.
"""
from __future__ import annotations
from pathlib import Path
import re
from rust_source import rust_source
ROOT = Path(__file__).resolve().parents[1]
SRC = ROOT / "src"
MAX_LINES = 500
def is_test_source(path: Path) -> bool:
relative = path.relative_to(SRC)
return path.name == "tests.rs" or "tests" in relative.parts
include_re = re.compile(r'include!\(\s*"([^"]+\.rs)"\s*\)')
test_attr_re = re.compile(r'(?m)^\s*#\[(?:tokio::)?test(?:\([^\]]*\))?\]')
dangling_doc_before_close_re = re.compile(r'(?ms)(?:^\s*///[^\n]*\n){1,8}\s*}')
dangling_attr_before_close_re = re.compile(r'(?ms)^\s*#\[[^\n]+\]\s*\n\s*}')
def delimiters_balanced(text: str) -> bool:
stack: list[str] = []
pairs = {"}": "{", ")": "(", "]": "["}
i = 0
state = "code"
block_depth = 0
raw_hashes = 0
while i < len(text):
char = text[i]
nxt = text[i + 1] if i + 1 < len(text) else ""
if state == "code":
if char == "/" and nxt == "/":
state = "line"; i += 2; continue
if char == "/" and nxt == "*":
state = "block"; block_depth = 1; i += 2; continue
if char == '"':
state = "string"; i += 1; continue
if char == "r":
raw = re.match(r'r(#{0,16})"', text[i:])
if raw:
raw_hashes = len(raw.group(1)); state = "raw"; i += len(raw.group(0)); continue
if char == "'":
# Lifetimes are identifiers; only enter char mode when a nearby closing quote exists.
cursor = i + 1
closing = False
while cursor < min(len(text), i + 8):
if text[cursor] == "\\": cursor += 2; continue
if text[cursor] == "'": closing = True; break
if text[cursor] in "\r\n": break
cursor += 1
if closing:
state = "char"; i += 1; continue
if char in "({[": stack.append(char)
elif char in "})]":
if not stack or stack[-1] != pairs[char]: return False
stack.pop()
i += 1
elif state == "line":
if char == "\n": state = "code"
i += 1
elif state == "block":
if char == "/" and nxt == "*": block_depth += 1; i += 2
elif char == "*" and nxt == "/":
block_depth -= 1; i += 2
if block_depth == 0: state = "code"
else: i += 1
elif state == "string":
if char == "\\": i += 2
elif char == '"': state = "code"; i += 1
else: i += 1
elif state == "char":
if char == "\\": i += 2
elif char == "'": state = "code"; i += 1
else: i += 1
elif state == "raw":
terminator = '"' + "#" * raw_hashes
if text.startswith(terminator, i): i += len(terminator); state = "code"
else: i += 1
return not stack and state in {"code", "line"}
# The 500-line rule applies to every Rust source in the repository, including integration tests.
for path in sorted(ROOT.rglob("*.rs")):
if any(part in {"target", ".git"} for part in path.parts):
continue
lines = path.read_text(encoding="utf-8").splitlines()
assert len(lines) <= MAX_LINES, f"{path.relative_to(ROOT)} has {len(lines)} lines (max {MAX_LINES})"
for path in sorted(SRC.rglob("*.rs")):
text = path.read_text(encoding="utf-8")
lines = text.splitlines()
assert delimiters_balanced(text), f"unbalanced delimiters in split Rust source: {path.relative_to(ROOT)}"
nonblank = [line.strip() for line in lines if line.strip()]
if nonblank:
assert not nonblank[-1].startswith(("#[", "///", "//!")), f"dangling Rust attribute/doc comment at EOF: {path.relative_to(ROOT)}"
assert not dangling_doc_before_close_re.search(text), f"doc comment before closing brace does not document an item: {path.relative_to(ROOT)}"
assert not dangling_attr_before_close_re.search(text), f"attribute before closing brace does not decorate an item: {path.relative_to(ROOT)}"
for target in include_re.findall(text):
included = path.parent / target
assert included.is_file(), f"broken include! in {path.relative_to(ROOT)}: {target}"
included_text = included.read_text(encoding="utf-8")
# include! expands into the caller's existing module/item scope. Inner module docs
# (`//!`) in a split fragment are therefore invalid once another item precedes them.
assert not re.search(r'(?m)^\s*//!', included_text), (
f"inner module doc comment in include! fragment: {included.relative_to(ROOT)}"
)
if not is_test_source(path):
assert not test_attr_re.search(text), f"unit test remains in implementation source: {path.relative_to(ROOT)}"
# Test-only implementation must live in a separate file and be pulled in explicitly.
for match in re.finditer(r'(?m)^\s*#\[cfg\(test\)\]\s*$', text):
tail = text[match.end():].lstrip()
assert tail.startswith("include!(") or tail.startswith("mod tests;"), (
f"test cfg in implementation must target a separate test file: {path.relative_to(ROOT)}"
)
# Explicit regressions from the first Web Server split.
response = (SRC / "web_server/response.rs").read_text(encoding="utf-8")
assert not response.rstrip().endswith("#[derive(Debug, Clone)]"), "dangling derive returned in Web Server response module"
fastcgi = rust_source(ROOT, "src/web_server/fastcgi.rs")
request = (SRC / "web_server/request.rs").read_text(encoding="utf-8")
assert "use anyhow::{Context" in fastcgi or "Context," in fastcgi or ", Context" in fastcgi
assert "htaccess," in fastcgi
assert "request::server_protocol" in fastcgi
assert "pub(super) fn server_protocol" in request
headers = (SRC / "web_server/response/headers.rs").read_text(encoding="utf-8")
assert "htaccess," in headers, "Web Server response headers must import sibling htaccess after module split"
assert "ACCEPT_ENCODING," not in headers, "stale ACCEPT_ENCODING import reintroduced in response headers"
# Regressions exposed by a real `cargo check --locked --all-targets` after the 0.25.1 split.
assert "pub(super) trait FastCgiIo" in fastcgi, "FastCgiIo visibility must match sibling-visible FastCGI helpers"
for relative in ("src/proxy/websocket/tests.rs", "tests/http3.rs"):
text = (ROOT / relative).read_text(encoding="utf-8")
assert 'parse::<h3::ext::Protocol>().unwrap()' not in text, (
f"{relative} uses Result::unwrap() for h3::ext::Protocol even though InvalidProtocol has no Debug"
)
server_tests = (SRC / "server/serve/tests.rs").read_text(encoding="utf-8")
assert server_tests.count("serve_inner(") == server_tests.count("Http2Connections::default()"), (
"direct serve_inner test calls must pass the Http2Connections argument"
)
cache_policy_tests = (SRC / "cache/tests/tests/part_03.rs").read_text(encoding="utf-8")
assert "let plan = plan(" not in cache_policy_tests, "cache policy test shadows the plan() function"
cold_tests = (SRC / "cache/tests/cold_tier_tests/part_02.rs").read_text(encoding="utf-8")
for field in ("scheme:", "authority:", "target:", "tags:", "created_at:"):
assert field in cold_tests, f"ColdEntry test fixture is missing current field {field}"
admin_tests = (SRC / "admin/tests/part_02.rs").read_text(encoding="utf-8")
assert "let configuration = router\n .clone()\n .oneshot(" in admin_tests, (
"admin persistence test must retain Router before Tower::oneshot consumes a clone"
)
web_tests_1 = (SRC / "web_server/tests/part_01.rs").read_text(encoding="utf-8")
assert "ETAG, EXPIRES," not in web_tests_1, "unused EXPIRES import returned in Web Server tests"
print(f"PASS: Rust modules are <= {MAX_LINES} lines, test code is isolated, include! targets resolve, and split-module/all-target regressions are guarded.")
+43
View File
@@ -0,0 +1,43 @@
"""Helpers for source-level contracts over split Rust modules.
The project intentionally keeps Rust source files small and composes several modules
with include!("..."). Static contracts must inspect the logical source, not only the
thin facade file.
"""
from __future__ import annotations
from pathlib import Path
import re
_INCLUDE = re.compile(r'include!\(\s*"([^"]+\.rs)"\s*\)\s*;?')
def expanded_source(path: Path, _stack: tuple[Path, ...] = ()) -> str:
path = path.resolve()
if path in _stack:
chain = " -> ".join(str(item) for item in (*_stack, path))
raise RuntimeError(f"cyclic Rust include!: {chain}")
text = path.read_text(encoding="utf-8")
def replace(match: re.Match[str]) -> str:
child = (path.parent / match.group(1)).resolve()
if not child.is_file():
raise FileNotFoundError(f"{path}: include target does not exist: {match.group(1)}")
return expanded_source(child, (*_stack, path))
return _INCLUDE.sub(replace, text)
def rust_source(root: Path, relative: str) -> str:
"""Read one logical Rust source file, recursively expanding include! fragments."""
return expanded_source(root / relative)
def rust_tree(root: Path, relative_dir: str) -> str:
"""Read every Rust file beneath a module directory (including mod-declared children)."""
directory = root / relative_dir
return "\n".join(
expanded_source(path)
for path in sorted(directory.rglob("*.rs"))
if path.is_file()
)
+5 -3
View File
@@ -2,18 +2,20 @@
"""Source-level security invariants that must hold even before Rust runtime tests."""
from __future__ import annotations
from pathlib import Path
from rust_source import rust_source, rust_tree
ROOT = Path(__file__).resolve().parents[1]
def read(path: str) -> str:
return (ROOT / path).read_text(encoding="utf-8")
target = ROOT / path
return rust_source(ROOT, path) if target.suffix == ".rs" else target.read_text(encoding="utf-8")
def need(text: str, *needles: str) -> None:
for needle in needles:
if needle not in text:
raise AssertionError(f"missing security invariant: {needle}")
web = read("src/web_server.rs")
web = rust_tree(ROOT, "src/web_server")
htaccess = read("src/web_server/htaccess.rs")
acl = read("src/acl.rs")
admin = read("src/admin/mod.rs")
@@ -94,7 +96,7 @@ need(auth,
# Web serving/admin code must not execute shell commands as part of request/config handling.
for path, text in {
"src/web_server.rs": web,
"src/web_server/*.rs": web,
"src/web_server/htaccess.rs": htaccess,
"src/admin/mod.rs": admin,
}.items():
+65
View File
@@ -0,0 +1,65 @@
#!/usr/bin/env python3
"""Compiler-independent source-size and test-isolation contracts for non-Rust modules."""
from __future__ import annotations
from pathlib import Path
import os
import re
import shutil
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[1]
MAX_LINES = 500
def lines(path: Path) -> int:
return len(path.read_text(encoding="utf-8").splitlines())
# Browser implementation modules are bounded. Test harnesses under scripts/ are checked separately.
for path in sorted((ROOT / "src/admin/web").rglob("*.js")):
assert lines(path) <= MAX_LINES, f"{path.relative_to(ROOT)} has {lines(path)} lines (max {MAX_LINES})"
# Docker Agent implementation and Go tests are separate files and all remain bounded.
for path in sorted((ROOT / "docker-agent").glob("*.go")):
assert lines(path) <= MAX_LINES, f"{path.relative_to(ROOT)} has {lines(path)} lines (max {MAX_LINES})"
if not path.name.endswith("_test.go"):
text = path.read_text(encoding="utf-8")
assert not re.search(r"(?m)^func\s+Test[A-Z]", text), f"Go test remains in implementation file: {path.relative_to(ROOT)}"
# Operational Python entry points are split as product modules; large QA harnesses are intentionally exempt.
for pattern in ("deploy*.py", "migrate_npm*.py"):
for path in sorted((ROOT / "scripts").glob(pattern)):
assert lines(path) <= MAX_LINES, f"{path.relative_to(ROOT)} has {lines(path)} lines (max {MAX_LINES})"
for old in (ROOT / "src/admin/web/app.js", ROOT / "src/admin/web/vhosts.js"):
assert not old.exists(), f"oversized browser monolith returned: {old.relative_to(ROOT)}"
for required in (
"src/admin/web/app/core.js", "src/admin/web/app/runtime.js", "src/admin/web/app/settings.js",
"src/admin/web/vhosts/core.js", "src/admin/web/vhosts/editor.js", "src/admin/web/vhosts/render.js",
"docker-agent/model.go", "docker-agent/docker.go", "docker-agent/network.go",
"docker-agent/certificate.go", "docker-agent/server.go",
"scripts/deploy_core.py", "scripts/deploy_runtime.py",
"scripts/migrate_npm_core.py", "scripts/migrate_npm_api.py",
):
assert (ROOT / required).is_file(), f"missing split source module: {required}"
# When Node is available, parse every standalone browser module and both assembled public bundles.
node = shutil.which("node")
if node:
env = dict(os.environ, TERM=os.environ.get("TERM", "xterm"))
standalone = list((ROOT / "src/admin/web").rglob("*.js"))
for path in sorted(standalone):
subprocess.run([node, "--check", str(path)], check=True, env=env, stdout=subprocess.DEVNULL)
from web_assets import read_app_js, read_vhosts_js
with tempfile.TemporaryDirectory(prefix="rustproxy-js-contract-") as directory:
temporary = Path(directory)
for name, content in (("app.js", read_app_js(ROOT)), ("vhosts.js", read_vhosts_js(ROOT))):
target = temporary / name
target.write_text(content, encoding="utf-8")
subprocess.run([node, "--check", str(target)], check=True, env=env, stdout=subprocess.DEVNULL)
# The public vhosts asset wraps three individually valid source modules in one private IIFE.
# Every source part is parsed above; this additional check keeps the assembled browser asset valid.
print(f"PASS: browser, Docker Agent and operational Python implementation modules are <= {MAX_LINES} lines, tests are isolated, and browser source boundaries parse cleanly.")
+1 -1
View File
@@ -85,7 +85,7 @@ def download(destination: Path) -> None:
"""TLS verification uses Python's default trust store; never executes a shell."""
started = time.monotonic()
opener = build_opener(HttpsRedirects())
request = Request(DOWNLOAD_URL, headers={"User-Agent": "RustProxy-build/0.25.0"})
request = Request(DOWNLOAD_URL, headers={"User-Agent": "RustProxy-build/0.25.1"})
with opener.open(request, timeout=30) as response, destination.open("wb") as output:
length = response.headers.get("Content-Length")
if length and int(length) > MAX_ARCHIVE:
+3 -3
View File
@@ -46,7 +46,7 @@ def upgrade(source: Path, target: Path, *, stopped: bool, data_dir: Path | None
for name in MANAGED:
if (target / name).is_symlink():
raise ValueError(f"Managed target {name} is a symlink; refusing to replace it.")
backup = target / ".upgrade-backup" / ("before-0.25.0-" + uuid.uuid4().hex[:12])
backup = target / ".upgrade-backup" / ("before-0.25.1-" + uuid.uuid4().hex[:12])
backup.mkdir(parents=True, mode=0o700)
installed: list[str] = []
moved: list[str] = []
@@ -84,7 +84,7 @@ def upgrade(source: Path, target: Path, *, stopped: bool, data_dir: Path | None
remove_legacy = legacy.exists()
if remove_legacy and (legacy.is_symlink() or not legacy.is_file()):
raise ValueError("Obsolete renewal observation sidecar must be a regular file.")
(backup / "upgrade.json").write_text(json.dumps({"from_directory":str(target), "to_version":"0.25.0", "managed_paths":MANAGED, "removed_obsolete_runtime_state": remove_legacy}, indent=2)+"\n", encoding="utf-8")
(backup / "upgrade.json").write_text(json.dumps({"from_directory":str(target), "to_version":"0.25.1", "managed_paths":MANAGED, "removed_obsolete_runtime_state": remove_legacy}, indent=2)+"\n", encoding="utf-8")
if remove_legacy:
legacy.unlink()
except BaseException:
@@ -115,7 +115,7 @@ def main() -> None:
backup = upgrade(Path(__file__).resolve().parents[1], args.target, stopped=args.stopped, data_dir=data_dir)
except (OSError, ValueError) as error:
parser.exit(2, f"Upgrade refused or rolled back: {error}\n")
print(f"Installed RustProxy 0.25.0. Previous managed source files: {backup}")
print(f"Installed RustProxy 0.25.1. Previous managed source files: {backup}")
print("Preserved data, .env, .git, target and other unmanaged files. No database migration was run.")
print(f"Next: cd {args.target}; ./dev.sh test; ./dev.sh")
+14
View File
@@ -0,0 +1,14 @@
from pathlib import Path
APP_PARTS = ("core.js", "runtime.js", "settings.js")
VHOST_PARTS = ("core.js", "editor.js", "render.js")
def _read(root: Path, folder: str, parts: tuple[str, ...]) -> str:
return "\n".join((root / "src/admin/web" / folder / part).read_text(encoding="utf-8") for part in parts)
def read_app_js(root: Path) -> str:
return _read(root, "app", APP_PARTS)
def read_vhosts_js(root: Path) -> str:
body = _read(root, "vhosts", VHOST_PARTS)
return "(() => {\n" + body + "\n})();\n"
+19 -12
View File
@@ -2,6 +2,8 @@
"""Rust-independent 0.25 Web Server architecture/security contract checks."""
from pathlib import Path
import json
from rust_source import rust_source, rust_tree
from web_assets import read_app_js, read_vhosts_js
ROOT = Path(__file__).resolve().parents[1]
@@ -9,19 +11,23 @@ def need(text: str, *markers: str) -> None:
for marker in markers:
assert marker in text, marker
config = (ROOT / "src/config.rs").read_text()
vhost = (ROOT / "src/vhost.rs").read_text()
web = (ROOT / "src/web_server.rs").read_text()
proxy = (ROOT / "src/proxy/mod.rs").read_text()
admin = (ROOT / "src/admin/mod.rs").read_text()
ui = (ROOT / "src/admin/web/vhosts.js").read_text()
config = rust_source(ROOT, "src/config.rs")
vhost = rust_source(ROOT, "src/vhost.rs")
web_dir = ROOT / "src/web_server"
assert not (ROOT / "src/web_server.rs").exists(), "Web Server monolith must not coexist with src/web_server/mod.rs"
for module in ["mod.rs", "config.rs", "filesystem.rs", "security.rs", "response.rs", "request.rs", "routing.rs", "fastcgi.rs", "serve.rs", "htaccess.rs", "tests.rs"]:
assert (web_dir / module).is_file(), module
web = rust_tree(ROOT, "src/web_server")
proxy = rust_source(ROOT, "src/proxy/mod.rs")
admin = rust_source(ROOT, "src/admin/mod.rs")
ui = read_vhosts_js(ROOT)
web_ui = (ROOT / "src/admin/web/web-server.js").read_text()
app_ui = (ROOT / "src/admin/web/app.js").read_text()
app_ui = read_app_js(ROOT)
html = (ROOT / "src/admin/web/index.html").read_text()
settings_ui = (ROOT / "src/admin/web/settings.js").read_text()
logging = (ROOT / "src/logging.rs").read_text()
metrics = (ROOT / "src/metrics.rs").read_text()
acl = (ROOT / "src/acl.rs").read_text()
logging = rust_source(ROOT, "src/logging.rs")
metrics = rust_source(ROOT, "src/metrics.rs")
acl = rust_source(ROOT, "src/acl.rs")
openapi = json.loads((ROOT / "src/admin/openapi.json").read_text())
need(config,
@@ -90,7 +96,7 @@ need(web,
'HeaderValue::from_static("no-store")',
"builtin_error_response(StatusCode::FORBIDDEN, &method)",
"empty_index_can_go_directly_to_directory_listing",
"RustProxy <b>0.25.0</b>",
"RustProxy <b>0.25.1</b>",
"traversal_and_symlink_escape_are_not_served",
"try_files_rewrites_cache_rules_headers_and_downloads_work",
"precompressed_sidecars_keep_mime_validators_and_range_semantics",
@@ -166,12 +172,13 @@ assert web_service_schema["properties"]["htaccess"]["properties"]["enabled"]["de
assert web_service_schema["properties"]["htaccess"]["properties"]["filename"]["default"] == ".htaccess"
assert web_service_schema["properties"]["hardening"]["properties"]["block_common_attacks"]["default"] is False
assert web_service_schema["properties"]["hardening"]["properties"]["strip_unnecessary_headers"]["default"] is False
htaccess_source = (ROOT / "src/web_server/htaccess.rs").read_text()
htaccess_source = rust_source(ROOT, "src/web_server/htaccess.rs")
need(htaccess_source, "MAX_HTACCESS_FILES", "MAX_HTACCESS_TOTAL_BYTES", "RewriteEngine", "RewriteCond", "RewriteRule", "DirectoryIndex", "AddType", "AddHandler", "ExpiresByType", "Require all granted/denied", "general Files/FilesMatch semantics are not supported", "security/handler directive", "apply_response", "apply_rewrites")
assert "Source" in openapi.get("components", {}).get("schemas", {})
assert "client_ips" in openapi["components"]["schemas"]["TrafficCounters"]["properties"]
assert (ROOT / "docs/WEB_SERVER.md").is_file()
assert (ROOT / "docs/PHP_FPM.md").is_file()
assert (ROOT / "docs/V0.25.1_AUDIT.md").is_file()
assert (ROOT / "docs/V0.25.0_AUDIT.md").is_file()
assert (ROOT / "docs/TODO.md").is_file()
docs_html = (ROOT / "src/admin/web/docs.html").read_text()