v0.27.3
This commit is contained in:
1 parent
e84df445c6
commit
e87a382a7e
109 files changed
+1092
-2925
No files matched your search
@@ -0,0 +1,10 @@
|
||||
# Developer and operator tools
|
||||
|
||||
These files are not required by the running RustProxy proxy process.
|
||||
|
||||
- `checks/` - compiler-independent source/release/documentation contracts and UI checks.
|
||||
- `dev/` - local mock services and diagnostic traffic/compression helpers.
|
||||
- `audit/` - explicit operator audit/probe tools.
|
||||
- `legacy/` - retained compatibility/release-test helpers; not the supported production workflow.
|
||||
|
||||
Production packages do not need this tree unless a packaging/test workflow intentionally ships it.
|
||||
Executable
+63
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only anonymous admin access probes against a running RustProxy.
|
||||
No cookie, password or token is sent; response bodies are never printed.
|
||||
Only HTTP loopback URLs are accepted. Exit 1 on an unexpected disclosure/status.
|
||||
This is a focused regression probe, NOT a penetration test or full security audit.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import ipaddress
|
||||
import json
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.request import Request, build_opener, ProxyHandler, HTTPRedirectHandler
|
||||
from urllib.error import HTTPError, URLError
|
||||
|
||||
PRIVATE = ('/api/status','/api/config','/api/backends','/api/traffic','/api/cache','/api/transform','/api/logs','/api/file-logs','/api/file-logs/query','/api/audit',
|
||||
'/api/events','/api/certificates','/api/acme','/api/auth/session','/metrics','/api/metrics',
|
||||
'/docs/openapi.json','/docs/init.js','/docs/style.css','/docs/swagger-ui-bundle.js',
|
||||
'/docs/swagger-ui.css','/docs/licenses.txt','/assets/app.js','/assets/langs.json','/assets/vhosts.js',
|
||||
'/assets/management.js','/assets/integrations.js','/assets/settings.js','/assets/file-logs.js','/assets/acme.js',
|
||||
'/assets/traffic-panel.js','/assets/cache-panel.js','/assets/transform-panel.js','/api/does-not-exist')
|
||||
|
||||
class NoRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(self,*args,**kwargs):
|
||||
return None
|
||||
|
||||
def base_url(value: str) -> str:
|
||||
url=urlsplit(value)
|
||||
if url.scheme!='http' or url.username or url.password or url.path not in ('','/') or url.query or url.fragment:
|
||||
raise argparse.ArgumentTypeError('Use the HTTP loopback administrator origin, e.g. http://127.0.0.1:9090')
|
||||
try:
|
||||
loopback=url.hostname=='localhost' or ipaddress.ip_address(url.hostname).is_loopback
|
||||
if not loopback or not url.port:raise ValueError()
|
||||
except (ValueError,TypeError):raise argparse.ArgumentTypeError('Explicit loopback address and port required')
|
||||
return value.rstrip('/')
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--url',type=base_url,default='http://127.0.0.1:9090');args=parser.parse_args()
|
||||
opener=build_opener(ProxyHandler({}),NoRedirect());failed=0;count=0
|
||||
def check(path,expected,validate):
|
||||
nonlocal failed,count
|
||||
count+=1
|
||||
try:
|
||||
request=Request(args.url+path,headers={'Accept':'application/json'})
|
||||
try:response=opener.open(request,timeout=5)
|
||||
except HTTPError as error:response=error
|
||||
with response:
|
||||
status=response.code;headers=response.headers;body=response.read(8388609 if path=='/assets/login-langs.json' else 131073)
|
||||
ok=status==expected and headers.get('Cache-Control')=='no-store' and validate(body,headers)
|
||||
except (OSError,ValueError,KeyError,TypeError,URLError):status='unavailable';ok=False
|
||||
if not ok:failed+=1
|
||||
print(f'{"PASS" if ok else "FAIL"} {path}: HTTP {status}; expected {expected}')
|
||||
def denied(body,headers):return json.loads(body)=={'error':'auth.required'}
|
||||
for path in PRIVATE:check(path,401,denied)
|
||||
check('/docs',303,lambda b,h:h.get('Location')=='/?next=docs')
|
||||
check('/docs/',303,lambda b,h:h.get('Location')=='/?next=docs')
|
||||
check('/healthz',200,lambda b,h:json.loads(b)=={'status':'ok'})
|
||||
check('/api/auth/status',200,lambda b,h:isinstance((v:=json.loads(b)).get('setup_required'),bool) and set(v)=={'setup_required'})
|
||||
check('/',200,lambda b,h:b'/assets/login.js' in b and all(x not in b for x in [b'/assets/app.js',b'config-editor',b'loginLead']))
|
||||
check('/assets/login-langs.json',200,lambda b,h:all(set(p['translations'])<=set(json.loads((Path(__file__).resolve().parents[2]/'src/admin/web/login-keys.json').read_text())) for p in json.loads(b)['languages']))
|
||||
print(f'{count-failed}/{count} anonymous probes passed. No authenticated/mutation/TLS checks performed.')
|
||||
raise SystemExit(bool(failed))
|
||||
if __name__=='__main__':main()
|
||||
Executable
+1441
File diff suppressed because it is too large.
Load diff
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rust-independent documentation/Reader consistency checks for the 0.27.3 release."""
|
||||
from pathlib import Path
|
||||
import re
|
||||
from rust_source import rust_source
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
DOCS = ROOT / "docs"
|
||||
|
||||
required = [
|
||||
ROOT / "README.md",
|
||||
ROOT / "ROADMAP.md",
|
||||
ROOT / "CHANGELOG.md",
|
||||
DOCS / "README.md",
|
||||
DOCS / "WEB_SERVER.md",
|
||||
DOCS / "TODO.md",
|
||||
DOCS / "V0.27.3_IMPLEMENTATION.md",
|
||||
DOCS / "V0.27.2_IMPLEMENTATION.md",
|
||||
DOCS / "V0.27.1_IMPLEMENTATION.md",
|
||||
DOCS / "V0.27.0_IMPLEMENTATION.md",
|
||||
DOCS / "V0.26.7_IMPLEMENTATION.md",
|
||||
DOCS / "V0.25.5_AUDIT.md",
|
||||
DOCS / "V0.25.4_AUDIT.md",
|
||||
DOCS / "V0.25.3_AUDIT.md",
|
||||
DOCS / "V0.25.2_AUDIT.md",
|
||||
DOCS / "V0.25.1_AUDIT.md",
|
||||
DOCS / "V0.25.0_AUDIT.md",
|
||||
DOCS / "TEST_REPORT.md",
|
||||
DOCS / "CHANGELOG_TECHNICAL_ARCHIVE.md",
|
||||
DOCS / "ROADMAP_HISTORY.md",
|
||||
]
|
||||
for path in required:
|
||||
assert path.is_file(), f"missing required documentation: {path.relative_to(ROOT)}"
|
||||
|
||||
index = (DOCS / "README.md").read_text(encoding="utf-8")
|
||||
for marker in ["Functional documentation", "Development and testing", "Technical documentation", "Historical or version-labelled context", "TODO.md", "V0.25.5_AUDIT.md", "V0.25.4_AUDIT.md", "V0.25.3_AUDIT.md", "V0.25.2_AUDIT.md", "V0.25.1_AUDIT.md", "V0.25.0_AUDIT.md", "CHANGELOG_TECHNICAL_ARCHIVE.md", "ROADMAP_HISTORY.md", "/docs"]:
|
||||
assert marker in index, marker
|
||||
for document in sorted(DOCS.glob("*.md")):
|
||||
if document.name == "README.md" or (document.name.startswith("V") and document.name.endswith("_AUDIT.md")):
|
||||
continue
|
||||
assert document.name in index, f"documentation index does not classify {document.name}"
|
||||
|
||||
reader_html = (ROOT / "src/admin/web/docs.html").read_text(encoding="utf-8")
|
||||
reader_js = (ROOT / "src/admin/web/docs.js").read_text(encoding="utf-8")
|
||||
build = (ROOT / "build.rs").read_text(encoding="utf-8")
|
||||
admin = rust_source(ROOT, "src/admin/mod.rs")
|
||||
for marker in ["docs-tab-reader", "docs-reader-list", "docs-reader-content", "docs-tab-api"]:
|
||||
assert marker in reader_html, marker
|
||||
for marker in ["renderMarkdown", "selectDocument", 'get("/docs/content.json")', "docsByPath", "docsCategoryHistorical"]:
|
||||
assert marker in reader_js, marker
|
||||
documentation_rs = rust_source(ROOT, "src/admin/documentation.rs") + (ROOT / "src/admin/documentation/documents.rs").read_text(encoding="utf-8")
|
||||
assert "READER_DOCUMENTS" in documentation_rs and "reader_bundle" in documentation_rs
|
||||
assert 'include_str!(concat!(env!("CARGO_MANIFEST_DIR")' in documentation_rs
|
||||
assert 'docs-content.json' not in documentation_rs and 'documentation_bundle' not in build
|
||||
for document in sorted(DOCS.glob("*.md")):
|
||||
marker = f'"{document.relative_to(ROOT).as_posix()}"'
|
||||
assert marker in documentation_rs, f"Reader does not embed {document.relative_to(ROOT)}"
|
||||
assert '"/docs/content.json"' in admin
|
||||
|
||||
readme = (ROOT / "README.md").read_text(encoding="utf-8")
|
||||
roadmap = (ROOT / "ROADMAP.md").read_text(encoding="utf-8")
|
||||
changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8")
|
||||
assert readme.startswith("# RustProxy 0.27.3")
|
||||
assert "docs/WEB_SERVER.md" in readme and "docs/README.md" in readme
|
||||
assert readme.count("## What changed in ") == 1, "README duplicates historical release notes; use CHANGELOG/archive"
|
||||
assert "## Release history" in readme and "docs/ROADMAP_HISTORY.md" in readme
|
||||
assert roadmap.startswith("# RustProxy roadmap") and "## 0.27.3" in roadmap and "## 0.27.2" in roadmap and "## 0.27.1" in roadmap and "## 0.27.0" in roadmap and "## 0.26.7" in roadmap and "## 0.26.6" in roadmap and "## 0.26.5" in roadmap and "## 0.26.4" in roadmap and "## 0.26.0" in roadmap and "## 0.25.5" in roadmap and "## 0.25.4" in roadmap and "## 0.25.3" in roadmap and "## 0.25.2" in roadmap and "## 0.25.1" in roadmap and "## 0.25.0" in roadmap
|
||||
assert "## 0.24." not in roadmap and "## Web Server follow-up" in roadmap and "docs/ROADMAP_HISTORY.md" in roadmap
|
||||
assert changelog.startswith("# Changelog\n")
|
||||
for release in ["0.27.3", "0.27.2", "0.27.1", "0.27.0", "0.26.7", "0.26.6", "0.26.5", "0.26.4", "0.26.1", "0.26.0", "0.25.6", "0.25.5", "0.25.4", "0.25.3"]:
|
||||
assert f"## {release}" in changelog, release
|
||||
assert changelog.index("## 0.27.3") < changelog.index("## 0.27.2") < changelog.index("## 0.27.1") < changelog.index("## 0.27.0") < changelog.index("## 0.26.7") < changelog.index("## 0.26.6") < changelog.index("## 0.26.5") < changelog.index("## 0.26.4") < changelog.index("## 0.26.1") < changelog.index("## 0.26.0") < changelog.index("## 0.25.6") < changelog.index("## 0.25.5")
|
||||
assert "CHANGELOG_TECHNICAL_ARCHIVE.md" in changelog
|
||||
for developer_only in ["Clean the remaining compiler warnings", "Update package/source/UI/OpenAPI/release identity", "source regressions and update release identity"]:
|
||||
assert developer_only not in changelog, f"developer-only CHANGELOG detail: {developer_only}"
|
||||
|
||||
link_re = re.compile(r"\[[^\]]+\]\(([^)]+)\)")
|
||||
paths = [ROOT / "README.md", ROOT / "ROADMAP.md", ROOT / "CHANGELOG.md", *sorted(DOCS.glob("*.md"))]
|
||||
for source in paths:
|
||||
text = source.read_text(encoding="utf-8")
|
||||
assert "Run compiler-enabled `cargo fmt --all`" not in text, f"obsolete release-host checklist remains in {source.relative_to(ROOT)}"
|
||||
for raw in link_re.findall(text):
|
||||
target = raw.strip().split()[0].strip("<>")
|
||||
if not target or target.startswith(("http://", "https://", "mailto:", "#", "/")):
|
||||
continue
|
||||
target = target.split("#", 1)[0].split("?", 1)[0]
|
||||
if not target:
|
||||
continue
|
||||
resolved = (source.parent / target).resolve()
|
||||
assert resolved.exists(), f"broken documentation link: {source.relative_to(ROOT)} -> {target}"
|
||||
|
||||
install = (DOCS / "INSTALL.md").read_text(encoding="utf-8")
|
||||
for marker in ["https://rustproxy.linuxiarz.pl/", "https://rustproxy-opn.linuxiarz.pl/", "Optional: Debian 13+ one-line source install/update"]:
|
||||
assert marker in install, marker
|
||||
|
||||
print("PASS: documentation index, Reader/API split, release docs and local Markdown links are consistent.")
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate language-pack content and emit the same-shaped bundle used by UI QA.
|
||||
Rust build.rs performs the actual compile-time discovery and embedding.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
def bundle(root=ROOT):
|
||||
languages=[];locales=set()
|
||||
for path in sorted((root/'langs').glob('*.json')):
|
||||
assert re.fullmatch(r'[A-Za-z0-9_-]{1,64}',path.stem),path
|
||||
assert path.stat().st_size<=262144,path
|
||||
pack=json.loads(path.read_text(encoding='utf-8'));assert set(pack)=={'meta','translations'},f'unexpected top-level language-pack keys: {set(pack)-{"meta","translations"}} in {path}';meta=pack['meta'];translations=pack['translations']
|
||||
assert all(isinstance(meta.get(key),str) and 0<len(meta[key].encode())<=128 for key in ['locale','name','native_name','direction']),path
|
||||
assert meta['direction'] in ('ltr','rtl'),path
|
||||
assert re.fullmatch(r'[A-Za-z0-9]{1,8}(?:-[A-Za-z0-9]{1,8})*',meta['locale']),path
|
||||
assert meta['locale'].lower() not in locales, 'duplicate locale'
|
||||
locales.add(meta['locale'].lower())
|
||||
assert isinstance(translations,dict) and len(translations)<=4096,path
|
||||
assert all(isinstance(v,str) and 0<len(k.encode())<=128 and len(v.encode())<=16384 for k,v in translations.items()),path
|
||||
languages.append({'id':path.stem,**pack})
|
||||
assert 1<=len(languages)<=128
|
||||
fallback=next(pack for pack in languages if pack['id']=='en_US')['translations']
|
||||
html=(root/'src/admin/web/index.html').read_text()
|
||||
keys=set(re.findall(r'data-i18n(?:-placeholder|-aria)?="([^"]+)"',html))
|
||||
js='\n'.join(p.read_text() for p in (root/'src/admin/web').glob('**/*.js'))
|
||||
keys.update(re.findall(r'\bt\("([^"\n]+)"\)',js))
|
||||
keys.update(re.findall(r'"(vh[A-Z][A-Za-z]+)"',js))
|
||||
assert keys<=fallback.keys(),f'Missing fallback keys: {keys-fallback.keys()}'
|
||||
for language in languages:
|
||||
translations=language['translations']
|
||||
assert keys<=translations.keys(),f"Missing UI keys in {language['id']}: {keys-translations.keys()}"
|
||||
result={'default':'en_US','languages':languages}
|
||||
assert len(json.dumps(result,ensure_ascii=False).encode())<=8388608
|
||||
return result
|
||||
if __name__=='__main__':
|
||||
parser=argparse.ArgumentParser();parser.add_argument('--output',type=Path);args=parser.parse_args()
|
||||
data=bundle()
|
||||
if args.output:args.output.write_text(json.dumps(data,ensure_ascii=False)+'\n')
|
||||
print(f"PASS: {len(data['languages'])} language packs, metadata, fallback UI keys and bounds.")
|
||||
Executable
+218
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Static regression checks for bounded-memory/background-worker contracts.
|
||||
|
||||
This is intentionally Rust-toolchain independent. It does not replace runtime RSS/
|
||||
heap profiling; it prevents the source-level bounds audited for this release from
|
||||
silently disappearing before compiler-enabled acceptance runs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from rust_source import rust_source
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def read(path: str) -> str:
|
||||
target = ROOT / path
|
||||
return rust_source(ROOT, path) if target.suffix == ".rs" else target.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def require(path: str, *markers: str) -> None:
|
||||
text = read(path)
|
||||
for marker in markers:
|
||||
assert marker in text, f"{path}: missing memory contract: {marker}"
|
||||
|
||||
|
||||
def forbid(path: str, *markers: str) -> None:
|
||||
text = read(path)
|
||||
for marker in markers:
|
||||
assert marker not in text, f"{path}: forbidden memory pattern returned: {marker}"
|
||||
|
||||
|
||||
# Transform path: hard aggregate/decode/concurrency budgets and no JSON DOM tree.
|
||||
require(
|
||||
"src/transform/mod.rs",
|
||||
"DEFAULT_MEMORY_BUDGET_BYTES: usize = 128 * 1024 * 1024",
|
||||
"pub memory_budget_bytes: usize",
|
||||
"pub max_inflight: usize",
|
||||
"try_reserve_transform_memory(required_memory, settings.memory_budget_bytes)",
|
||||
"try_reserve_transform_slot(settings.max_inflight)",
|
||||
"TRANSFORM_MEMORY_IN_USE_BYTES: AtomicUsize",
|
||||
"TRANSFORM_INFLIGHT: AtomicUsize",
|
||||
"pub struct ResourceUsage",
|
||||
"pub fn resource_usage(settings: &Settings)",
|
||||
"memory_reserved_bytes: TRANSFORM_MEMORY_IN_USE_BYTES.load",
|
||||
"Limited::new(body, settings.max_body_bytes)",
|
||||
".try_reserve_exact(read)",
|
||||
'"memory_pressure"',
|
||||
'"concurrency_pressure"',
|
||||
"aggregation_is_bounded_even_when_content_length_is_false",
|
||||
"transform_memory_reservation_obeys_runtime_budget",
|
||||
"transform_inflight_reservation_obeys_runtime_limit",
|
||||
)
|
||||
require("src/transform/json.rs", "IgnoredAny::deserialize", "Vec::with_capacity(body.len())")
|
||||
forbid("src/transform/json.rs", "use serde_json::Value", "from_slice::<Value>")
|
||||
require("src/transform/css.rs", "let mut last_char: Option<char> = None")
|
||||
require("src/transform/html.rs", "enum RawElement", "fn raw_tag(tag: &str)")
|
||||
|
||||
# Cache background work: all previously high-retention paths have hard bounds.
|
||||
require(
|
||||
"src/cache/model.rs",
|
||||
"COLD_DELETE_FALLBACKS: usize = 2",
|
||||
"COLD_CLEANUP_SLOTS: usize = 1",
|
||||
"BACKGROUND_REVALIDATION_LIMIT: usize = 8",
|
||||
"pub cold_limit_bytes: u64",
|
||||
"pub cold_limit_entries: usize",
|
||||
"pub hot_auto_trim_enabled: bool",
|
||||
"pub hot_auto_trim_after_seconds: u64",
|
||||
"pub hot_auto_trim_target_percent: u8",
|
||||
)
|
||||
require(
|
||||
"src/admin/mod.rs",
|
||||
'payload["transform_resources"]',
|
||||
'payload["memory_accounting"]',
|
||||
'"non_cache_rss_includes_transform": true',
|
||||
)
|
||||
require(
|
||||
"src/cache/cold.rs",
|
||||
"BinaryHeap::with_capacity(requested_limit.saturating_add(1))",
|
||||
"self.cold_delete_fallbacks.clone().try_acquire_owned()",
|
||||
"self.cleanup_slots.clone().try_acquire_owned()",
|
||||
"std::mem::take(&mut delete)",
|
||||
)
|
||||
require(
|
||||
"src/cache/tickets.rs",
|
||||
"active.len() >= BACKGROUND_REVALIDATION_LIMIT",
|
||||
)
|
||||
require(
|
||||
"src/cache/collapse.rs",
|
||||
".saturating_mul(64)",
|
||||
".clamp(64, 256)",
|
||||
"settings.proactive_refresh_max_inflight as u64",
|
||||
)
|
||||
require(
|
||||
"src/cache/lifecycle.rs",
|
||||
"mpsc::channel(COLD_QUEUE)",
|
||||
"cold_delete_fallbacks: Arc::new(Semaphore::new(COLD_DELETE_FALLBACKS))",
|
||||
"cleanup_slots: Arc::new(Semaphore::new(COLD_CLEANUP_SLOTS))",
|
||||
"cache.trim_hot_idle();",
|
||||
"fn trim_hot_idle(&self)",
|
||||
)
|
||||
|
||||
# COLD bodies stay disk-backed and the long-lived index must not retain full
|
||||
# response header/validator allocations for every object.
|
||||
cache_model = read("src/cache/model/cache.rs")
|
||||
cold_entry = cache_model.split("struct ColdEntry {", 1)[1].split("#[derive(Default)]", 1)[0]
|
||||
assert "headers: HeaderMap" not in cold_entry, "COLD index retains full response headers"
|
||||
assert "origin_etag: Option<HeaderValue>" not in cold_entry, "COLD index retains ETag HeaderValue"
|
||||
assert "origin_last_modified: Option<HeaderValue>" not in cold_entry, "COLD index retains Last-Modified HeaderValue"
|
||||
assert "body_sha256: Key" in cold_entry, "COLD digest must use the compact 32-byte representation"
|
||||
assert "has_validator: bool" in cold_entry, "COLD index must retain validator presence without full values"
|
||||
require(
|
||||
"src/cache/model/cold_metadata.rs",
|
||||
"tokio::fs::read(&entry.meta_path).await",
|
||||
"pairs_to_headers(&file.headers)",
|
||||
"body_sha256 != entry.body_sha256",
|
||||
)
|
||||
|
||||
# Rate-limit client state stores a vhost allocation once and reclaims expired
|
||||
# fixed-window buckets without waiting for the global cardinality ceiling.
|
||||
require(
|
||||
"src/rate_limit.rs",
|
||||
"buckets: HashMap<String, HashMap<IpAddr, Bucket>>",
|
||||
"fn sweep_expired(&mut self, now: Instant, force: bool)",
|
||||
"now.duration_since(bucket.started) < Duration::from_secs(bucket.window_seconds)",
|
||||
"state.sweep_expired(now, false)",
|
||||
"state.sweep_expired(now, true)",
|
||||
"clients.shrink_to_fit()",
|
||||
"self.buckets.shrink_to_fit()",
|
||||
"pub fn maintenance(&self)",
|
||||
"MAX_BUCKETS: usize = 65_536",
|
||||
)
|
||||
require(
|
||||
"src/runtime/constructors.rs",
|
||||
"app.rate_limiter.maintenance()",
|
||||
"spawn_blocking(crate::diagnostics::trim_allocator)",
|
||||
)
|
||||
|
||||
# Compression remains response-streaming; no complete-body collection is added.
|
||||
require(
|
||||
"src/compression.rs",
|
||||
"No full-body aggregation is introduced.",
|
||||
"CompressionLevel::Fastest",
|
||||
"Compression::new(service)",
|
||||
)
|
||||
|
||||
# Other long-lived/background facilities: queue/task/worker cardinality is bounded
|
||||
# or owned by a cancellation-aware supervisor/TaskTracker.
|
||||
require(
|
||||
"src/logging.rs",
|
||||
"HISTORY_CAPACITY: usize = 4096",
|
||||
"LIVE_STREAM_CAPACITY: usize = 4096",
|
||||
"mpsc::channel(SINK_QUEUE)",
|
||||
"self.sink.try_send(entry)",
|
||||
"pub capture_user_agent: bool",
|
||||
"pub capture_country: bool",
|
||||
)
|
||||
require(
|
||||
"src/traffic.rs",
|
||||
"QUEUE_CAPACITY: usize = 2048",
|
||||
"TOP_CAPACITY: usize = 32",
|
||||
"MAX_VHOSTS: usize = 64",
|
||||
"self.tx.try_send(sample)",
|
||||
"client_ips: TopTable",
|
||||
)
|
||||
require(
|
||||
"src/diagnostics.rs",
|
||||
"tokio::task::spawn_blocking(move || system_stats::sample(previous)).await",
|
||||
"stop.cancelled()",
|
||||
"pub fn trim_allocator() -> bool",
|
||||
)
|
||||
require(
|
||||
"src/runtime.rs",
|
||||
"Duration::from_secs(60)",
|
||||
"spawn_blocking(crate::diagnostics::trim_allocator)",
|
||||
)
|
||||
require(
|
||||
"src/upstream.rs",
|
||||
"Semaphore::new(16)",
|
||||
".try_acquire_owned()",
|
||||
"tokio::time::timeout(DNS_TIMEOUT, lookup)",
|
||||
)
|
||||
require(
|
||||
"src/upstream/client.rs",
|
||||
"pool_recycle_requests",
|
||||
"pool_recycle_seconds",
|
||||
"http1_max_buf_size(keepalive.http1_max_buffer_bytes)",
|
||||
"ArcSwap<RawProxyClient>",
|
||||
"self.current.store(Arc::new(client))",
|
||||
'"recycled upstream client pool"',
|
||||
)
|
||||
require(
|
||||
"src/balance.rs",
|
||||
"if checks.len() >= 16 { break; }",
|
||||
"checks.abort_all()",
|
||||
)
|
||||
require(
|
||||
"src/stream.rs",
|
||||
"Semaphore::new(host.max_connections)",
|
||||
"mpsc::channel(64)",
|
||||
"JoinSet::new()",
|
||||
)
|
||||
require(
|
||||
"src/server.rs",
|
||||
"Semaphore::new(max_connections)",
|
||||
"let mut tasks = JoinSet::new()",
|
||||
)
|
||||
require(
|
||||
"src/runtime.rs",
|
||||
"tokio_util::task::TaskTracker::new()",
|
||||
"update_slots: Arc::new(Semaphore::new(8))",
|
||||
"inflight: Arc::new(Semaphore::new(max_inflight))",
|
||||
)
|
||||
require("src/acme.rs", "slot: Arc::new(Semaphore::new(1))", ".try_acquire_owned()")
|
||||
require("src/docker.rs", "shutdown.cancelled()", "Duration::from_secs(5)")
|
||||
require("src/update_check.rs", "shutdown.cancelled()", "tokio::time::sleep(interval)")
|
||||
|
||||
print("PASS: bounded-memory transform/cache contracts and background-worker limits are present.")
|
||||
Executable
+98
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Detect mixed source trees and preserve the known obsolete 0.0.1 auth module."""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
LEGACY_AUTH = "4df9d473c9214365d91c83072f97aafab6ede4edcf4e1369b5c3679ded5cff85"
|
||||
|
||||
|
||||
def check(root: Path, repair_known: bool = False) -> list[str]:
|
||||
messages: list[str] = []
|
||||
|
||||
# 0.25.1 split-source migration: these monoliths are obsolete when the new
|
||||
# bounded module tree is present. Preserve any local edits instead of silently
|
||||
# deleting them when a source archive is extracted over an older checkout.
|
||||
obsolete_split_files = (
|
||||
"src/admin/web/app.js",
|
||||
"src/admin/web/vhosts.js",
|
||||
"src/web_server.rs",
|
||||
)
|
||||
split_markers = {
|
||||
"src/admin/web/app.js": "src/admin/web/app/core.js",
|
||||
"src/admin/web/vhosts.js": "src/admin/web/vhosts/core.js",
|
||||
"src/web_server.rs": "src/web_server/mod.rs",
|
||||
}
|
||||
for relative in obsolete_split_files:
|
||||
obsolete = root / relative
|
||||
marker = root / split_markers[relative]
|
||||
if not obsolete.exists() or not marker.is_file():
|
||||
continue
|
||||
if not repair_known:
|
||||
raise ValueError(
|
||||
f"Obsolete 0.25.0 source monolith {relative} remains beside the 0.25.1 split tree. "
|
||||
"Run ./dev.sh repair (or any normal ./dev.sh command) to preserve it under .upgrade-backup/."
|
||||
)
|
||||
backup_root = root / ".upgrade-backup" / "0.25.1-split"
|
||||
target = backup_root / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if target.exists():
|
||||
# Never overwrite a prior backup; if the bytes match, the current stale
|
||||
# copy can simply be removed. Otherwise force the operator to reconcile it.
|
||||
if target.read_bytes() != obsolete.read_bytes():
|
||||
raise ValueError(
|
||||
f"Backup target {target.relative_to(root)} already contains different data; "
|
||||
f"move {relative} outside the source tree manually before rebuilding."
|
||||
)
|
||||
obsolete.unlink()
|
||||
else:
|
||||
obsolete.rename(target)
|
||||
messages.append(
|
||||
f"Preserved obsolete {relative} as {target.relative_to(root)}"
|
||||
)
|
||||
|
||||
legacy = root / "src/auth.rs"
|
||||
if legacy.exists() and (root / "src/auth/mod.rs").is_file():
|
||||
if legacy.is_symlink() or hashlib.sha256(legacy.read_bytes()).hexdigest() != LEGACY_AUTH:
|
||||
raise ValueError("Both src/auth.rs and src/auth/mod.rs exist. The single-file module is not the known 0.0.1 file. Preserve your edits and move it outside src before rebuilding. See docs/MIGRATION.md.")
|
||||
if not repair_known:
|
||||
raise ValueError("Known obsolete src/auth.rs from 0.0.1 found. Run ./dev.sh repair or start ./dev.sh to preserve it in .upgrade-backup/legacy-auth/.")
|
||||
directory = root / ".upgrade-backup/legacy-auth"
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
target = directory / "auth.rs.0.0.1"
|
||||
if target.exists():
|
||||
if hashlib.sha256(target.read_bytes()).hexdigest() != LEGACY_AUTH:
|
||||
raise ValueError("Legacy backup path contains a different file; refusing to overwrite it.")
|
||||
legacy.unlink() # Byte-identical content was already preserved.
|
||||
else:
|
||||
legacy.rename(target)
|
||||
messages.append("Preserved obsolete src/auth.rs as .upgrade-backup/legacy-auth/auth.rs.0.0.1")
|
||||
for module in (root / "src").rglob("mod.rs"):
|
||||
flat = module.parent.with_suffix(".rs")
|
||||
if flat.is_file():
|
||||
raise ValueError(f"Conflicting Rust modules: {flat.relative_to(root)} and {module.relative_to(root)}")
|
||||
for path in ["src/auth/mod.rs", "src/auth/store.rs", "src/backup/snapshot.rs", "src/upstream.rs", "src/tls.rs", "src/proxy/websocket.rs", "src/cache/mod.rs", "src/cache/model.rs", "src/cache/lifecycle.rs", "src/cache/cold.rs", "src/cache/dashboard.rs", "src/cache/tickets.rs", "src/cache/lookup.rs", "src/cache/policy.rs", "src/cache/fill.rs", "src/cache/tests.rs", "src/settings.rs", "src/compression.rs", "src/diagnostics.rs", "src/traffic.rs", "src/system_stats/mod.rs", "src/system_stats/linux.rs", "src/vhost.rs", "src/web_server/mod.rs", "src/web_server/config.rs", "src/web_server/filesystem.rs", "src/web_server/security.rs", "src/web_server/response.rs", "src/web_server/request.rs", "src/web_server/routing.rs", "src/web_server/fastcgi.rs", "src/web_server/htaccess.rs", "src/web_server/serve.rs", "src/web_server/tests.rs", "src/balance.rs", "src/security.rs", "src/admin/web/vhosts/core.js", "src/admin/web/vhosts/editor.js", "src/admin/web/vhosts/render.js", "src/admin/web/web-server.js", "src/admin/web/streams.js", "src/admin/web/host404.js", "src/admin/web/app/core.js", "src/admin/web/app/preferences.js", "src/admin/web/app/config-history.js", "src/admin/web/app/runtime.js", "src/admin/web/app/settings.js", "src/admin/web/management.js", "src/admin/web/drafts.js", "src/pages.rs", "tools/dev/mock_server.py", "tools/dev/mock_auth.py", "src/forwarding.rs", "src/tinyauth.rs", "src/http3.rs", "src/stream.rs", "src/admin/web/integrations.js", "src/admin/web/traffic-panel.js", "src/prometheus.rs", "src/admin/web/settings.js", "src/admin/web/docker.js", "src/docker.rs", "docker-agent/main.go", "docker-agent/model.go", "docker-agent/docker.go", "docker-agent/network.go", "docker-agent/certificate.go", "docker-agent/server.go", "docker-agent/main_test.go", "docker-agent/rustproxy-docker-agent.service", "docker-agent/install.sh", "docs/DOCKER_DISCOVERY.md", "docs/WEB_SERVER.md", "docs/PHP_FPM.md", "docs/README.md", "docs/TODO.md", "docs/V0.25.5_AUDIT.md", "docs/V0.25.4_AUDIT.md", "docs/V0.25.3_AUDIT.md", "docs/V0.25.2_AUDIT.md", "docs/CHANGELOG_TECHNICAL_ARCHIVE.md", "docs/ROADMAP_HISTORY.md", "tools/checks/docs_check.py", "tools/checks/web_server_contracts.py", "tools/checks/rust_source.py", "tools/checks/rust_module_contracts.py", "tools/checks/source_module_contracts.py", "scripts/runtime/acme_provider.py", "src/basic_auth.rs", "scripts/deploy/deploy.py", "scripts/deploy/deploy_core.py", "scripts/deploy/deploy_runtime.py", "scripts/migration/migrate_npm.py", "scripts/migration/migrate_npm_core.py", "scripts/migration/migrate_npm_api.py", "docs/NPM_MIGRATION.md", "docs/STREAMS.md", "install.sh", "install-debian13.sh", "update.sh", "build.rs", "scripts/build/swagger_assets.py", "src/admin/documentation.rs", "src/admin/openapi.json", "langs/en_US.json", "langs/pl_pl.json", "Dockerfile", ".dockerignore", "docs/CONTAINER.md", "docs/PROMETHEUS.md", "docs/NOT_FOUND_HOSTS.md", "grafana/rustproxy-overview.json"]:
|
||||
if not (root / path).is_file():
|
||||
raise ValueError(f"Incomplete RustProxy 0.27.3 source tree: missing {path}")
|
||||
text = (root / "src/lib.rs").read_text(encoding="utf-8")
|
||||
if '#[path = "auth/mod.rs"]' not in text:
|
||||
raise ValueError("src/lib.rs is not the 0.27.3 entry point; extract the complete package, not individual files.")
|
||||
return messages
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2])
|
||||
parser.add_argument("--repair-known", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
for line in check(args.root.resolve(), args.repair_known):
|
||||
print(line, file=sys.stderr)
|
||||
except (OSError, ValueError) as error:
|
||||
parser.exit(2, f"Source preflight failed: {error}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+1304
File diff suppressed because it is too large.
Load diff
Executable
+165
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Static structure contracts for the split Rust source tree.
|
||||
|
||||
These checks are intentionally compiler-independent. They prevent the module split
|
||||
from regressing into large monoliths, keep unit tests out of implementation files,
|
||||
and catch the class of dangling-attribute/broken-include mistakes that can otherwise
|
||||
surface only at cargo check time.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from rust_source import rust_source
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SRC = ROOT / "src"
|
||||
MAX_LINES = 500
|
||||
|
||||
|
||||
def is_test_source(path: Path) -> bool:
|
||||
relative = path.relative_to(SRC)
|
||||
return path.name == "tests.rs" or path.stem.endswith("_tests") or "tests" in relative.parts
|
||||
|
||||
|
||||
include_re = re.compile(r'include!\(\s*"([^"]+\.rs)"\s*\)')
|
||||
test_attr_re = re.compile(r'(?m)^\s*#\[(?:tokio::)?test(?:\([^\]]*\))?\]')
|
||||
dangling_doc_before_close_re = re.compile(r'(?ms)(?:^\s*///[^\n]*\n){1,8}\s*}')
|
||||
dangling_attr_before_close_re = re.compile(r'(?ms)^\s*#\[[^\n]+\]\s*\n\s*}')
|
||||
|
||||
def delimiters_balanced(text: str) -> bool:
|
||||
stack: list[str] = []
|
||||
pairs = {"}": "{", ")": "(", "]": "["}
|
||||
i = 0
|
||||
state = "code"
|
||||
block_depth = 0
|
||||
raw_hashes = 0
|
||||
while i < len(text):
|
||||
char = text[i]
|
||||
nxt = text[i + 1] if i + 1 < len(text) else ""
|
||||
if state == "code":
|
||||
if char == "/" and nxt == "/":
|
||||
state = "line"; i += 2; continue
|
||||
if char == "/" and nxt == "*":
|
||||
state = "block"; block_depth = 1; i += 2; continue
|
||||
if char == '"':
|
||||
state = "string"; i += 1; continue
|
||||
if char == "r":
|
||||
raw = re.match(r'r(#{0,16})"', text[i:])
|
||||
if raw:
|
||||
raw_hashes = len(raw.group(1)); state = "raw"; i += len(raw.group(0)); continue
|
||||
if char == "'":
|
||||
# Lifetimes are identifiers; only enter char mode when a nearby closing quote exists.
|
||||
cursor = i + 1
|
||||
closing = False
|
||||
while cursor < min(len(text), i + 8):
|
||||
if text[cursor] == "\\": cursor += 2; continue
|
||||
if text[cursor] == "'": closing = True; break
|
||||
if text[cursor] in "\r\n": break
|
||||
cursor += 1
|
||||
if closing:
|
||||
state = "char"; i += 1; continue
|
||||
if char in "({[": stack.append(char)
|
||||
elif char in "})]":
|
||||
if not stack or stack[-1] != pairs[char]: return False
|
||||
stack.pop()
|
||||
i += 1
|
||||
elif state == "line":
|
||||
if char == "\n": state = "code"
|
||||
i += 1
|
||||
elif state == "block":
|
||||
if char == "/" and nxt == "*": block_depth += 1; i += 2
|
||||
elif char == "*" and nxt == "/":
|
||||
block_depth -= 1; i += 2
|
||||
if block_depth == 0: state = "code"
|
||||
else: i += 1
|
||||
elif state == "string":
|
||||
if char == "\\": i += 2
|
||||
elif char == '"': state = "code"; i += 1
|
||||
else: i += 1
|
||||
elif state == "char":
|
||||
if char == "\\": i += 2
|
||||
elif char == "'": state = "code"; i += 1
|
||||
else: i += 1
|
||||
elif state == "raw":
|
||||
terminator = '"' + "#" * raw_hashes
|
||||
if text.startswith(terminator, i): i += len(terminator); state = "code"
|
||||
else: i += 1
|
||||
return not stack and state in {"code", "line"}
|
||||
|
||||
|
||||
# The 500-line rule applies to every Rust source in the repository, including integration tests.
|
||||
for path in sorted(ROOT.rglob("*.rs")):
|
||||
if any(part in {"target", ".git"} for part in path.parts):
|
||||
continue
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
assert len(lines) <= MAX_LINES, f"{path.relative_to(ROOT)} has {len(lines)} lines (max {MAX_LINES})"
|
||||
|
||||
for path in sorted(SRC.rglob("*.rs")):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
assert delimiters_balanced(text), f"unbalanced delimiters in split Rust source: {path.relative_to(ROOT)}"
|
||||
|
||||
nonblank = [line.strip() for line in lines if line.strip()]
|
||||
if nonblank:
|
||||
assert not nonblank[-1].startswith(("#[", "///", "//!")), f"dangling Rust attribute/doc comment at EOF: {path.relative_to(ROOT)}"
|
||||
assert not dangling_doc_before_close_re.search(text), f"doc comment before closing brace does not document an item: {path.relative_to(ROOT)}"
|
||||
assert not dangling_attr_before_close_re.search(text), f"attribute before closing brace does not decorate an item: {path.relative_to(ROOT)}"
|
||||
|
||||
for target in include_re.findall(text):
|
||||
included = path.parent / target
|
||||
assert included.is_file(), f"broken include! in {path.relative_to(ROOT)}: {target}"
|
||||
included_text = included.read_text(encoding="utf-8")
|
||||
# include! expands into the caller's existing module/item scope. Inner module docs
|
||||
# (`//!`) in a split fragment are therefore invalid once another item precedes them.
|
||||
assert not re.search(r'(?m)^\s*//!', included_text), (
|
||||
f"inner module doc comment in include! fragment: {included.relative_to(ROOT)}"
|
||||
)
|
||||
|
||||
if not is_test_source(path):
|
||||
assert not test_attr_re.search(text), f"unit test remains in implementation source: {path.relative_to(ROOT)}"
|
||||
# Test-only implementation must live in a separate file and be pulled in explicitly.
|
||||
for match in re.finditer(r'(?m)^\s*#\[cfg\(test\)\]\s*$', text):
|
||||
tail = text[match.end():].lstrip()
|
||||
assert tail.startswith("include!(") or tail.startswith("mod tests;"), (
|
||||
f"test cfg in implementation must target a separate test file: {path.relative_to(ROOT)}"
|
||||
)
|
||||
|
||||
# Explicit regressions from the first Web Server split.
|
||||
response = (SRC / "web_server/response.rs").read_text(encoding="utf-8")
|
||||
assert not response.rstrip().endswith("#[derive(Debug, Clone)]"), "dangling derive returned in Web Server response module"
|
||||
fastcgi = rust_source(ROOT, "src/web_server/fastcgi.rs")
|
||||
request = (SRC / "web_server/request.rs").read_text(encoding="utf-8")
|
||||
assert "use anyhow::{Context" in fastcgi or "Context," in fastcgi or ", Context" in fastcgi
|
||||
assert "htaccess," in fastcgi
|
||||
assert "request::server_protocol" in fastcgi
|
||||
assert "pub(super) fn server_protocol" in request
|
||||
headers = (SRC / "web_server/response/headers.rs").read_text(encoding="utf-8")
|
||||
assert "htaccess," in headers, "Web Server response headers must import sibling htaccess after module split"
|
||||
assert "ACCEPT_ENCODING," not in headers, "stale ACCEPT_ENCODING import reintroduced in response headers"
|
||||
|
||||
# Regressions exposed by a real `cargo check --locked --all-targets` after the 0.25.1 split.
|
||||
assert "pub(super) trait FastCgiIo" in fastcgi, "FastCgiIo visibility must match sibling-visible FastCGI helpers"
|
||||
for relative in ("src/proxy/websocket/tests.rs", "tests/http3.rs"):
|
||||
text = (ROOT / relative).read_text(encoding="utf-8")
|
||||
assert 'parse::<h3::ext::Protocol>().unwrap()' not in text, (
|
||||
f"{relative} uses Result::unwrap() for h3::ext::Protocol even though InvalidProtocol has no Debug"
|
||||
)
|
||||
server_tests = (SRC / "server/serve/tests.rs").read_text(encoding="utf-8")
|
||||
assert server_tests.count("serve_inner(") == server_tests.count("Http2Connections::default()"), (
|
||||
"direct serve_inner test calls must pass the Http2Connections argument"
|
||||
)
|
||||
cache_policy_tests = (SRC / "cache/tests/tests/part_03.rs").read_text(encoding="utf-8")
|
||||
assert "let plan = plan(" not in cache_policy_tests, "cache policy test shadows the plan() function"
|
||||
cold_tests = (SRC / "cache/tests/cold_tier_tests/part_02.rs").read_text(encoding="utf-8")
|
||||
for field in ("scheme:", "authority:", "target:", "tags:", "created_at:"):
|
||||
assert field in cold_tests, f"ColdEntry test fixture is missing current field {field}"
|
||||
admin_tests = (SRC / "admin/tests/part_02.rs").read_text(encoding="utf-8")
|
||||
assert "let configuration = router\n .clone()\n .oneshot(" in admin_tests, (
|
||||
"admin persistence test must retain Router before Tower::oneshot consumes a clone"
|
||||
)
|
||||
web_tests_1 = (SRC / "web_server/tests/part_01.rs").read_text(encoding="utf-8")
|
||||
assert "ETAG, EXPIRES," not in web_tests_1, "unused EXPIRES import returned in Web Server tests"
|
||||
|
||||
print(f"PASS: Rust modules are <= {MAX_LINES} lines, test code is isolated, include! targets resolve, and split-module/all-target regressions are guarded.")
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
"""Helpers for source-level contracts over split Rust modules.
|
||||
|
||||
The project intentionally keeps Rust source files small and composes several modules
|
||||
with include!("..."). Static contracts must inspect the logical source, not only the
|
||||
thin facade file.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
_INCLUDE = re.compile(r'include!\(\s*"([^"]+\.rs)"\s*\)\s*;?')
|
||||
|
||||
|
||||
def expanded_source(path: Path, _stack: tuple[Path, ...] = ()) -> str:
|
||||
path = path.resolve()
|
||||
if path in _stack:
|
||||
chain = " -> ".join(str(item) for item in (*_stack, path))
|
||||
raise RuntimeError(f"cyclic Rust include!: {chain}")
|
||||
text = path.read_text(encoding="utf-8")
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
child = (path.parent / match.group(1)).resolve()
|
||||
if not child.is_file():
|
||||
raise FileNotFoundError(f"{path}: include target does not exist: {match.group(1)}")
|
||||
return expanded_source(child, (*_stack, path))
|
||||
|
||||
return _INCLUDE.sub(replace, text)
|
||||
|
||||
|
||||
def rust_source(root: Path, relative: str) -> str:
|
||||
"""Read one logical Rust source file, recursively expanding include! fragments."""
|
||||
return expanded_source(root / relative)
|
||||
|
||||
|
||||
def rust_tree(root: Path, relative_dir: str) -> str:
|
||||
"""Read every Rust file beneath a module directory (including mod-declared children)."""
|
||||
directory = root / relative_dir
|
||||
return "\n".join(
|
||||
expanded_source(path)
|
||||
for path in sorted(directory.rglob("*.rs"))
|
||||
if path.is_file()
|
||||
)
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Source-level security invariants that must hold even before Rust runtime tests."""
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from rust_source import rust_source, rust_tree
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
def read(path: str) -> str:
|
||||
target = ROOT / path
|
||||
return rust_source(ROOT, path) if target.suffix == ".rs" else target.read_text(encoding="utf-8")
|
||||
|
||||
def need(text: str, *needles: str) -> None:
|
||||
for needle in needles:
|
||||
if needle not in text:
|
||||
raise AssertionError(f"missing security invariant: {needle}")
|
||||
|
||||
web = rust_tree(ROOT, "src/web_server")
|
||||
htaccess = read("src/web_server/htaccess.rs")
|
||||
acl = read("src/acl.rs")
|
||||
admin = read("src/admin/mod.rs")
|
||||
auth = read("src/auth/mod.rs")
|
||||
web_ui = read("src/admin/web/web-server.js")
|
||||
|
||||
# Preset regression: selecting a preset must actually materialize normal config fields.
|
||||
need(web_ui,
|
||||
"function presetChanged()",
|
||||
'if ($("web-service-preset").value !== "custom") applyPreset();',
|
||||
'$("web-service-preset").addEventListener("change", presetChanged);')
|
||||
|
||||
# Request target/path canonicalization: reject double decoding, encoded separators and dot traversal.
|
||||
need(acl,
|
||||
"ambiguous encoded path character",
|
||||
"ambiguous path segments",
|
||||
"!matches!(decoded, b'/' | b'\\\\' | b'%' | b'?' | b'#' | b';')",
|
||||
"path.len() <= 8192")
|
||||
|
||||
# FastCGI must not inherit client proxy settings (httpoxy) and must not control HTTP framing.
|
||||
need(web,
|
||||
"HTTP_PROXY is a well-known CGI/FastCGI injection vector",
|
||||
'if name.as_str() == "proxy"',
|
||||
'"content-length"',
|
||||
'"transfer-encoding"',
|
||||
'"connection"',
|
||||
'"upgrade"',
|
||||
'"proxy-authenticate"',
|
||||
'"proxy-authorization"',
|
||||
"(200..=599).contains(&code)",
|
||||
"unsupported informational/invalid status",
|
||||
"reserved_fastcgi_param",
|
||||
'"PHP_ADMIN_VALUE"',
|
||||
"pass_authorization",
|
||||
"max_request_body_bytes",
|
||||
"StatusCode::PAYLOAD_TOO_LARGE",
|
||||
"max_response_header_bytes",
|
||||
"stderr_log_bytes")
|
||||
|
||||
# File-system confinement and .htaccess loading must not follow attacker-controlled symlinks.
|
||||
need(web, "libc::openat", "libc::O_NOFOLLOW", "secure filesystem confinement")
|
||||
need(htaccess,
|
||||
"read_htaccess_nofollow",
|
||||
"libc::O_NOFOLLOW",
|
||||
"fs::symlink_metadata",
|
||||
".htaccess scan escaped the Web Service document root",
|
||||
"must not be a symlink",
|
||||
"MAX_HTACCESS_FILES",
|
||||
"MAX_HTACCESS_TOTAL_BYTES")
|
||||
need(web, "htaccess_configuration_file_cannot_be_a_symlink")
|
||||
|
||||
# PHP source must fail closed instead of becoming a static download when FPM is unavailable.
|
||||
need(web,
|
||||
"php_source_extensions_are_not_served_without_php_fpm",
|
||||
"php_fpm_failure_returns_gateway_error_instead_of_php_source",
|
||||
"fastcgi_never_forwards_proxy_header_as_http_proxy",
|
||||
"fastcgi_response_cannot_control_downstream_framing_or_hop_headers",
|
||||
"fastcgi_response_rejects_informational_status_codes",
|
||||
"php_custom_fastcgi_params_cannot_override_core_or_php_admin_values",
|
||||
"php_request_body_limit_returns_413_before_fastcgi_connect")
|
||||
|
||||
# Admin plane: exact anonymous allowlist + cookie/CSRF/origin/browser marker + CSP/frame denial.
|
||||
need(admin,
|
||||
"fn anonymous_allowed",
|
||||
'single_header(headers, "origin")',
|
||||
'single_header(headers, "x-rustproxy-request") != Some("1")',
|
||||
'single_header(headers, "x-csrf-token")',
|
||||
'"auth.csrf"',
|
||||
"frame-ancestors 'none'",
|
||||
"object-src 'none'",
|
||||
"anonymous_operational_paths_fail_closed_before_extractors",
|
||||
"duplicate_csrf_or_marker_is_rejected_and_session_survives")
|
||||
need(auth,
|
||||
"HttpOnly; SameSite=Strict",
|
||||
'if secure { "; Secure" } else { "" }',
|
||||
"Argon2id",
|
||||
"(12..=128).contains(&value.len())")
|
||||
|
||||
# Web serving/admin code must not execute shell commands as part of request/config handling.
|
||||
for path, text in {
|
||||
"src/web_server/*.rs": web,
|
||||
"src/web_server/htaccess.rs": htaccess,
|
||||
"src/admin/mod.rs": admin,
|
||||
}.items():
|
||||
if "Command::new" in text or "std::process::Command" in text:
|
||||
raise AssertionError(f"unexpected process execution primitive in {path}")
|
||||
|
||||
print("PASS: production security source invariants (paths, FastCGI, htaccess, admin auth/CSRF, no shell execution) are present.")
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline minimum-version gates for network-facing RustSec advisories reviewed 2026-10-04.
|
||||
|
||||
This is deliberately not a replacement for `cargo audit`: it prevents known-fixed
|
||||
packages from being accidentally downgraded when the lockfile changes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
LOCK = (ROOT / "Cargo.lock").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def versions() -> dict[str, list[str]]:
|
||||
found: dict[str, list[str]] = {}
|
||||
for block in LOCK.split("[[package]]")[1:]:
|
||||
match = re.search(r'\n?name = "([^"]+)"\nversion = "([^"]+)"', block)
|
||||
if match:
|
||||
found.setdefault(match.group(1), []).append(match.group(2))
|
||||
return found
|
||||
|
||||
|
||||
def triplet(value: str) -> tuple[int, int, int]:
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)", value)
|
||||
if not match:
|
||||
raise AssertionError(f"unsupported version syntax: {value}")
|
||||
return tuple(map(int, match.groups()))
|
||||
|
||||
|
||||
packages = versions()
|
||||
minimums = {
|
||||
# RUSTSEC-2026-0285
|
||||
"rustls": (0, 23, 45),
|
||||
# RUSTSEC-2026-0185 and RUSTSEC-2026-0037
|
||||
"quinn-proto": (0, 11, 15),
|
||||
# RUSTSEC-2026-0104 (also covers 0049/0098/0099)
|
||||
"rustls-webpki": (0, 103, 13),
|
||||
# RUSTSEC-2025-0009
|
||||
"ring": (0, 17, 12),
|
||||
# RUSTSEC-2024-0363
|
||||
"sqlx": (0, 8, 1),
|
||||
# RUSTSEC-2026-0258
|
||||
"h2": (0, 4, 16),
|
||||
# RUSTSEC-2026-0007
|
||||
"bytes": (1, 11, 1),
|
||||
# RUSTSEC-2026-0009
|
||||
"time": (0, 3, 47),
|
||||
}
|
||||
for package, minimum in minimums.items():
|
||||
locked = packages.get(package)
|
||||
if not locked:
|
||||
raise AssertionError(f"expected security-critical dependency {package} in Cargo.lock")
|
||||
for value in locked:
|
||||
if triplet(value) < minimum:
|
||||
raise AssertionError(f"{package} {value} is below reviewed patched minimum {'.'.join(map(str, minimum))}")
|
||||
|
||||
# 2026 malicious-crate campaign snapshot reviewed against RustSec on 2026-10-04.
|
||||
# Keep the exact arrayref version gate because earlier arrayref releases are legitimate.
|
||||
for value in packages.get("arrayref", []):
|
||||
if value == "0.3.10":
|
||||
raise AssertionError("Cargo.lock contains compromised arrayref 0.3.10")
|
||||
|
||||
malicious_packages = {
|
||||
"greentic-setup", "greentic-setup-dev", "append-only-vec", "proc-macro1",
|
||||
"internment", "proc-macro-en", "onering", "logflux", "exploration",
|
||||
"sui-execution-cut", "mysten-metrics", "microsoftsystem64",
|
||||
"pretty-changelog-logger", "safe-agent-rs", "logtrace", "jfrog_quotes",
|
||||
"tree-sitter-pkl", "custom-req-on-workers", "sophosfirewall-python",
|
||||
"statsrelay-protobuf", "time_calibrator", "tracings", "tracing_checks",
|
||||
"rpc-check", "tracing-check", "clob-sdk", "polymarkets-rs-clob-client",
|
||||
"polymarkets-client-sdk", "polymarket-client-sdks", "finch-rst",
|
||||
}
|
||||
present = sorted(malicious_packages.intersection(packages))
|
||||
if present:
|
||||
raise AssertionError(f"Cargo.lock contains RustSec-listed malicious package(s): {', '.join(present)}")
|
||||
|
||||
print("PASS: Cargo.lock stays above reviewed RustSec minimums and excludes the reviewed 2026 malicious-crate set (snapshot 2026-10-04).")
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compiler-independent source-size and test-isolation contracts for non-Rust modules."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MAX_LINES = 500
|
||||
|
||||
|
||||
def lines(path: Path) -> int:
|
||||
return len(path.read_text(encoding="utf-8").splitlines())
|
||||
|
||||
# Browser implementation modules are bounded. QA harnesses under tools/checks are checked separately.
|
||||
for path in sorted((ROOT / "src/admin/web").rglob("*.js")):
|
||||
assert lines(path) <= MAX_LINES, f"{path.relative_to(ROOT)} has {lines(path)} lines (max {MAX_LINES})"
|
||||
|
||||
# Docker Agent implementation and Go tests are separate files and all remain bounded.
|
||||
for path in sorted((ROOT / "docker-agent").glob("*.go")):
|
||||
assert lines(path) <= MAX_LINES, f"{path.relative_to(ROOT)} has {lines(path)} lines (max {MAX_LINES})"
|
||||
if not path.name.endswith("_test.go"):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
assert not re.search(r"(?m)^func\s+Test[A-Z]", text), f"Go test remains in implementation file: {path.relative_to(ROOT)}"
|
||||
|
||||
# Application-coupled Python entry points are split from external QA/dev tooling.
|
||||
for directory in ("scripts/runtime", "scripts/build", "scripts/deploy", "scripts/migration"):
|
||||
for path in sorted((ROOT / directory).glob("*.py")):
|
||||
assert lines(path) <= MAX_LINES, f"{path.relative_to(ROOT)} has {lines(path)} lines (max {MAX_LINES})"
|
||||
|
||||
for old in (ROOT / "src/admin/web/app.js", ROOT / "src/admin/web/vhosts.js"):
|
||||
assert not old.exists(), f"oversized browser monolith returned: {old.relative_to(ROOT)}"
|
||||
for required in (
|
||||
"src/admin/web/app/core.js", "src/admin/web/app/preferences.js", "src/admin/web/app/config-history.js", "src/admin/web/app/runtime.js", "src/admin/web/app/settings.js",
|
||||
"src/admin/web/vhosts/core.js", "src/admin/web/vhosts/editor.js", "src/admin/web/vhosts/render.js",
|
||||
"docker-agent/model.go", "docker-agent/docker.go", "docker-agent/network.go",
|
||||
"docker-agent/certificate.go", "docker-agent/server.go",
|
||||
"scripts/deploy/deploy_core.py", "scripts/deploy/deploy_runtime.py",
|
||||
"scripts/migration/migrate_npm_core.py", "scripts/migration/migrate_npm_api.py",
|
||||
):
|
||||
assert (ROOT / required).is_file(), f"missing split source module: {required}"
|
||||
|
||||
|
||||
# When Node is available, parse every standalone browser module and both assembled public bundles.
|
||||
node = shutil.which("node")
|
||||
if node:
|
||||
env = dict(os.environ, TERM=os.environ.get("TERM", "xterm"))
|
||||
standalone = list((ROOT / "src/admin/web").rglob("*.js"))
|
||||
for path in sorted(standalone):
|
||||
subprocess.run([node, "--check", str(path)], check=True, env=env, stdout=subprocess.DEVNULL)
|
||||
from web_assets import read_app_js, read_vhosts_js
|
||||
with tempfile.TemporaryDirectory(prefix="rustproxy-js-contract-") as directory:
|
||||
temporary = Path(directory)
|
||||
for name, content in (("app.js", read_app_js(ROOT)), ("vhosts.js", read_vhosts_js(ROOT))):
|
||||
target = temporary / name
|
||||
target.write_text(content, encoding="utf-8")
|
||||
subprocess.run([node, "--check", str(target)], check=True, env=env, stdout=subprocess.DEVNULL)
|
||||
# The public vhosts asset wraps three individually valid source modules in one private IIFE.
|
||||
# Every source part is parsed above; this additional check keeps the assembled browser asset valid.
|
||||
|
||||
print(f"PASS: browser, Docker Agent and application-coupled Python modules are <= {MAX_LINES} lines, tooling is separated, tests are isolated, and browser source boundaries parse cleanly.")
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
from pathlib import Path
|
||||
|
||||
APP_PARTS = ("core.js", "preferences.js", "config-history.js", "runtime.js", "settings.js")
|
||||
VHOST_PARTS = ("core.js", "editor.js", "render.js")
|
||||
|
||||
def _read(root: Path, folder: str, parts: tuple[str, ...]) -> str:
|
||||
return "\n".join((root / "src/admin/web" / folder / part).read_text(encoding="utf-8") for part in parts)
|
||||
|
||||
def read_app_js(root: Path) -> str:
|
||||
return _read(root, "app", APP_PARTS)
|
||||
|
||||
def read_vhosts_js(root: Path) -> str:
|
||||
body = _read(root, "vhosts", VHOST_PARTS)
|
||||
return "(() => {\n" + body + "\n})();\n"
|
||||
Executable
+197
@@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rust-independent 0.25 Web Server architecture/security contract checks."""
|
||||
from pathlib import Path
|
||||
import json
|
||||
from rust_source import rust_source, rust_tree
|
||||
from web_assets import read_app_js, read_vhosts_js
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
def need(text: str, *markers: str) -> None:
|
||||
for marker in markers:
|
||||
assert marker in text, marker
|
||||
|
||||
config = rust_source(ROOT, "src/config.rs")
|
||||
vhost = rust_source(ROOT, "src/vhost.rs")
|
||||
web_dir = ROOT / "src/web_server"
|
||||
assert not (ROOT / "src/web_server.rs").exists(), "Web Server monolith must not coexist with src/web_server/mod.rs"
|
||||
for module in ["mod.rs", "config.rs", "filesystem.rs", "security.rs", "response.rs", "request.rs", "routing.rs", "fastcgi.rs", "serve.rs", "htaccess.rs", "tests.rs"]:
|
||||
assert (web_dir / module).is_file(), module
|
||||
web = rust_tree(ROOT, "src/web_server")
|
||||
proxy = rust_source(ROOT, "src/proxy/mod.rs")
|
||||
admin = rust_source(ROOT, "src/admin/mod.rs")
|
||||
ui = read_vhosts_js(ROOT)
|
||||
web_ui = (ROOT / "src/admin/web/web-server.js").read_text()
|
||||
app_ui = read_app_js(ROOT)
|
||||
html = (ROOT / "src/admin/web/index.html").read_text()
|
||||
settings_ui = (ROOT / "src/admin/web/settings.js").read_text()
|
||||
logging = rust_source(ROOT, "src/logging.rs")
|
||||
metrics = rust_source(ROOT, "src/metrics.rs")
|
||||
acl = rust_source(ROOT, "src/acl.rs")
|
||||
openapi = json.loads((ROOT / "src/admin/openapi.json").read_text())
|
||||
|
||||
need(config,
|
||||
"pub web_services: Vec<crate::web_server::Service>",
|
||||
"pub enum CompiledSource",
|
||||
"CompiledSource::WebServer",
|
||||
"unknown Web Service",
|
||||
"web_server_main_source_needs_no_backend_pool",
|
||||
"web_server_and_reverse_proxy_can_share_one_vhost",
|
||||
"one_web_service_can_be_reused_by_multiple_vhosts",
|
||||
"missing_source_fields_remain_reverse_proxy_for_old_configs")
|
||||
need(vhost,
|
||||
"pub enum Source",
|
||||
"WebServer { web_service_id: String }",
|
||||
"pub source: Source",
|
||||
"vhost requires at least one backend member for Reverse Proxy sources",
|
||||
"Web Server location cannot pin a reverse-proxy backend")
|
||||
need(web,
|
||||
"pub struct Service",
|
||||
"pub document_root: PathBuf",
|
||||
"pub index: Vec<String>",
|
||||
"pub directory_listing: bool",
|
||||
"pub mime_types: BTreeMap<String, String>",
|
||||
"pub default_mime_type: String",
|
||||
"pub cache_control: Option<String>",
|
||||
"pub cache_rules: Vec<CacheRule>",
|
||||
"pub response_headers: BTreeMap<String, String>",
|
||||
"pub try_files: Vec<String>",
|
||||
"pub rewrites: Vec<RewriteRule>",
|
||||
"pub precompressed: Precompressed",
|
||||
"pub download_extensions: Vec<String>",
|
||||
"pub symlink_policy: SymlinkPolicy",
|
||||
"pub php: Php",
|
||||
"pub htaccess: HtaccessConfig",
|
||||
"pub hardening: Hardening",
|
||||
"hardening_blocks_common_probe_paths_and_trace",
|
||||
"hardening_strips_fingerprint_headers_after_source_rules",
|
||||
"HTTP_PROXY is a well-known CGI/FastCGI injection vector",
|
||||
'if name.as_str() == "proxy"',
|
||||
'"content-length"',
|
||||
'"proxy-authenticate"',
|
||||
'"proxy-authorization"',
|
||||
"mod htaccess",
|
||||
"htaccess::apply_rewrites",
|
||||
"htaccess::protected_component",
|
||||
"htaccess_wordpress_style_front_controller_and_protected_files_work",
|
||||
"htaccess_directory_index_mime_headers_expires_and_nested_scope_work",
|
||||
"htaccess_strict_mode_rejects_unsupported_directives",
|
||||
"htaccess_security_sensitive_unsupported_directives_fail_closed",
|
||||
"php_source_extensions_are_not_served_without_php_fpm",
|
||||
"htaccess_php_addhandler_requires_php_fpm",
|
||||
"FastCgiEndpoint",
|
||||
"FCGI_BEGIN_REQUEST",
|
||||
"try_acquire_owned",
|
||||
"CONTENT_DISPOSITION",
|
||||
"CONTENT_ENCODING",
|
||||
"libc::openat",
|
||||
"libc::O_NOFOLLOW",
|
||||
"ReaderStream::new(file.take(body_length))",
|
||||
"IF_NONE_MATCH", "IF_MODIFIED_SINCE", "IF_RANGE", "RANGE",
|
||||
"StatusCode::NOT_MODIFIED", "StatusCode::PARTIAL_CONTENT", "StatusCode::RANGE_NOT_SATISFIABLE",
|
||||
"listing_escapes_html_and_encodes_urls",
|
||||
"root_directory_listing_is_repeatable_across_refreshes",
|
||||
"index_precedes_listing_and_missing_index_falls_back_to_listing_per_directory",
|
||||
"mime_overrides_default_mime_and_cache_control_are_applied_to_files",
|
||||
'HeaderValue::from_static("no-store")',
|
||||
"builtin_error_response(StatusCode::FORBIDDEN, &method)",
|
||||
"empty_index_can_go_directly_to_directory_listing",
|
||||
"RustProxy <b>0.27.3</b>",
|
||||
"traversal_and_symlink_escape_are_not_served",
|
||||
"try_files_rewrites_cache_rules_headers_and_downloads_work",
|
||||
"precompressed_sidecars_keep_mime_validators_and_range_semantics",
|
||||
"php_fpm_executes_index_and_try_files_front_controller_without_source_disclosure",
|
||||
"php_fpm_executes_configured_directory_index",
|
||||
"php_fpm_failure_returns_gateway_error_instead_of_php_source",
|
||||
"inside_root_symlink_policy_allows_only_targets_beneath_root",
|
||||
"secure filesystem confinement currently requires a Unix platform")
|
||||
need(proxy,
|
||||
"if let crate::config::CompiledSource::WebServer(service) = &route.source",
|
||||
"let static_method = request.method().clone()",
|
||||
"crate::web_server::serve(",
|
||||
"static_headers",
|
||||
"response.extensions_mut().insert(StaticSourceResponse)",
|
||||
"let custom_error =",
|
||||
"crate::pages::replace_error",
|
||||
"apply_response_rules(",
|
||||
"finish_headers(",
|
||||
"crate::compression::encode(")
|
||||
assert "crate::web_server::serve(service, &request" not in proxy
|
||||
basic_identity_pos = proxy.index("basic_identity =")
|
||||
identity_pos = proxy.index("let identity =")
|
||||
web_source_pos = proxy.index("CompiledSource::WebServer(service)", basic_identity_pos)
|
||||
assert basic_identity_pos < web_source_pos
|
||||
assert identity_pos < web_source_pos
|
||||
assert web_source_pos < proxy.index("let (backend, backend_lease, affinity_cookie)")
|
||||
custom_error_pos = proxy.index("let custom_error =")
|
||||
assert proxy.find("apply_response_rules(", custom_error_pos) > custom_error_pos
|
||||
need(acl, "ambiguous encoded path character", '"/%2e%2e/secret"', '"/a%2fb"', '"/a%5cb"', '"/bad%"')
|
||||
need(admin, '"/web-server"', '"/assets/web-server.js"', '"web-services"', '"vhost_metrics"')
|
||||
need(html, 'data-panel="web-server"', 'id="panel-web-server"', 'id="web-service-dialog"',
|
||||
'id="web-service-tabs"', 'data-web-service-page="general"', 'data-web-service-page="routing"',
|
||||
'data-web-service-page="mime"', 'data-web-service-page="cache"', 'data-web-service-page="delivery"',
|
||||
'data-web-service-page="php"', 'data-web-service-page="security"', 'data-web-service-page="htaccess"',
|
||||
'id="web-service-htaccess-enabled"', 'id="web-service-htaccess-help"', 'id="htaccess-help-dialog"', 'id="web-service-try-files"',
|
||||
'id="web-service-cache-rules"', 'id="web-service-precompressed"', 'id="web-service-compression"', 'id="web-service-compression-zstd"', 'id="web-service-compression-br"', 'id="web-service-compression-gzip"', 'id="web-service-compression-deflate"', 'id="web-service-php-enabled"',
|
||||
'id="web-service-php-pass-authorization"', 'id="web-service-php-body-limit"',
|
||||
'id="web-service-php-header-limit"', 'id="web-service-php-stderr-limit"',
|
||||
'id="web-service-php-custom-params"', 'id="web-service-php-values"', 'id="php-help-dialog"',
|
||||
'id="web-service-symlink-policy"', 'id="web-service-preset"', 'id="web-service-preset-apply"',
|
||||
'id="web-service-hardening-attacks"', 'id="web-service-hardening-headers"', 'data-logging-settings-tab="outputs"',
|
||||
'data-logging-settings-tab="format"', 'data-logging-settings-tab="errors"')
|
||||
need(settings_ui, "selectLoggingTab", 'data-logging-settings-page', 'selectLoggingTab("outputs")')
|
||||
need(web_ui, 'saveSection("web-services"', "directory_listing", "web_service_id", "mime_types", "default_mime_type",
|
||||
"cache_control", "cache_rules", "response_headers", "try_files", "rewrites", "precompressed", "compression",
|
||||
"download_extensions", "symlink_policy", "php", "htaccess", "hardening", "presetConfig", "phpFrontController", "staticCacheRules",
|
||||
"pass_authorization", "max_request_body_bytes", "max_response_header_bytes", "stderr_log_bytes", "custom_params", "php_values",
|
||||
"syncHtaccess", "htaccess-help-dialog", "php-help-dialog", "parseFastCgiParams", "parsePhpValues", "validatePhpCustomData", "activateTab", "initTabs", "presetChanged",
|
||||
'button = el("button", t(key), "quiet")', 'button.dataset.tab = name', 'button.setAttribute("role", "tab")', "vhost-detail",
|
||||
'el("span", t(key), "small-label")', "if (envelope) render();")
|
||||
need(logging, '"route"', '"cache_effective_enabled"', '"transform_effective_enabled"', 'value["cache_enabled"]', 'value["transform_enabled"]')
|
||||
need(metrics, "pub struct VhostMetricsView", "ewma_duration_us", "pub fn view(&self) -> VhostMetricsView")
|
||||
assert 'item.append(text("span"' not in web_ui
|
||||
need(app_ui, 'function showStartupFailure(error)', '$("login").hidden = true', 'const info = await api("/api/auth/session")')
|
||||
assert '$("login").hidden = false; document.documentElement.classList.remove("app-loading")' not in app_ui
|
||||
need(ui, 'type: "web_server"', "web_service_id", "sourceReverseProxy", "vhBackendRequired", "backend-diagnostics-card", "status.vhost_metrics", "vhostTrafficRate")
|
||||
assert "WebService" in openapi.get("components", {}).get("schemas", {})
|
||||
web_service_schema = openapi["components"]["schemas"]["WebService"]
|
||||
assert web_service_schema["properties"]["index"].get("minItems", 0) == 0
|
||||
assert web_service_schema["properties"]["index"]["maxItems"] == 16
|
||||
assert web_service_schema["properties"]["mime_types"]["maxProperties"] == 64
|
||||
assert web_service_schema["properties"]["default_mime_type"]["default"] == "application/octet-stream"
|
||||
assert web_service_schema["properties"]["cache_control"]["nullable"] is True
|
||||
assert web_service_schema["properties"]["compression"]["$ref"].endswith("/CompressionOverrides")
|
||||
compression_override_schema = openapi["components"]["schemas"]["CompressionOverrides"]
|
||||
for prop in ["enabled", "zstd", "br", "gzip", "deflate"]:
|
||||
assert prop in compression_override_schema["properties"], prop
|
||||
for prop in ["cache_rules", "response_headers", "try_files", "rewrites", "precompressed", "compression", "download_extensions", "symlink_policy", "php", "htaccess", "hardening"]:
|
||||
assert prop in web_service_schema["properties"], prop
|
||||
assert web_service_schema["properties"]["symlink_policy"]["default"] == "deny"
|
||||
assert web_service_schema["properties"]["php"]["properties"]["enabled"]["default"] is False
|
||||
php_schema = web_service_schema["properties"]["php"]["properties"]
|
||||
assert php_schema["pass_authorization"]["default"] is True
|
||||
assert php_schema["max_request_body_bytes"]["default"] == 0
|
||||
assert php_schema["max_response_header_bytes"]["default"] == 65536
|
||||
assert php_schema["stderr_log_bytes"]["default"] == 4096
|
||||
assert php_schema["custom_params"]["maxProperties"] == 64
|
||||
assert php_schema["php_values"]["maxProperties"] == 64
|
||||
assert web_service_schema["properties"]["htaccess"]["properties"]["enabled"]["default"] is False
|
||||
assert web_service_schema["properties"]["htaccess"]["properties"]["filename"]["default"] == ".htaccess"
|
||||
assert web_service_schema["properties"]["hardening"]["properties"]["block_common_attacks"]["default"] is False
|
||||
assert web_service_schema["properties"]["hardening"]["properties"]["strip_unnecessary_headers"]["default"] is False
|
||||
htaccess_source = rust_source(ROOT, "src/web_server/htaccess.rs")
|
||||
need(htaccess_source, "MAX_HTACCESS_FILES", "MAX_HTACCESS_TOTAL_BYTES", "RewriteEngine", "RewriteCond", "RewriteRule", "DirectoryIndex", "AddType", "AddHandler", "ExpiresByType", "Require all granted/denied", "general Files/FilesMatch semantics are not supported", "security/handler directive", "apply_response", "apply_rewrites")
|
||||
assert "Source" in openapi.get("components", {}).get("schemas", {})
|
||||
assert "client_ips" in openapi["components"]["schemas"]["TrafficCounters"]["properties"]
|
||||
assert (ROOT / "docs/WEB_SERVER.md").is_file()
|
||||
assert (ROOT / "docs/PHP_FPM.md").is_file()
|
||||
assert (ROOT / "docs/V0.25.5_AUDIT.md").is_file()
|
||||
assert (ROOT / "docs/V0.25.4_AUDIT.md").is_file()
|
||||
assert (ROOT / "docs/V0.25.2_AUDIT.md").is_file()
|
||||
assert (ROOT / "docs/V0.25.0_AUDIT.md").is_file()
|
||||
assert (ROOT / "docs/TODO.md").is_file()
|
||||
docs_html = (ROOT / "src/admin/web/docs.html").read_text()
|
||||
docs_js = (ROOT / "src/admin/web/docs.js").read_text()
|
||||
need(docs_html, "docs-tab-reader", "docs-reader-content", "docs-tab-api")
|
||||
need(docs_js, 'get("/docs/content.json")', "renderMarkdown", "selectDocument")
|
||||
print("PASS: 0.25 Web Server separation, pipeline, filesystem, HTTP, UI/API and regression contracts are present.")
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from "node:fs";
|
||||
import vm from "node:vm";
|
||||
|
||||
class ClassList {
|
||||
constructor() { this.values = new Set(); }
|
||||
toggle(name, active) { active ? this.values.add(name) : this.values.delete(name); }
|
||||
}
|
||||
class Element {
|
||||
constructor(id = "") {
|
||||
this.id = id;
|
||||
this.value = "";
|
||||
this.checked = false;
|
||||
this.disabled = false;
|
||||
this.hidden = false;
|
||||
this.open = false;
|
||||
this.dataset = {};
|
||||
this.children = [];
|
||||
this.listeners = new Map();
|
||||
this.classList = new ClassList();
|
||||
this.attributes = new Map();
|
||||
this.tabIndex = 0;
|
||||
this.textContent = "";
|
||||
this.type = "";
|
||||
this.readOnly = false;
|
||||
this.elements = [];
|
||||
}
|
||||
addEventListener(name, callback) { this.listeners.set(name, callback); }
|
||||
dispatch(name) { return this.listeners.get(name)?.({ target: this, preventDefault() {} }); }
|
||||
setAttribute(name, value) { this.attributes.set(name, String(value)); }
|
||||
append(...items) { this.children.push(...items); }
|
||||
replaceChildren(...items) { this.children = [...items]; }
|
||||
showModal() { this.open = true; }
|
||||
close() { this.open = false; }
|
||||
reset() {}
|
||||
}
|
||||
|
||||
const ids = [
|
||||
"htaccess-help-close","htaccess-help-dialog","php-help-close","php-help-dialog","web-service-cache-control","web-service-cache-rules",
|
||||
"web-service-close","web-service-default-mime","web-service-dialog","web-service-download-extensions",
|
||||
"web-service-editing","web-service-form","web-service-hardening-attacks","web-service-hardening-headers",
|
||||
"web-service-htaccess-enabled","web-service-htaccess-filename","web-service-htaccess-help","web-service-htaccess-recursive",
|
||||
"web-service-htaccess-strict","web-service-id","web-service-index","web-service-list","web-service-listing",
|
||||
"web-service-mime-types","web-service-name","web-service-new","web-service-php-concurrency",
|
||||
"web-service-php-connect-timeout","web-service-php-enabled","web-service-php-endpoint","web-service-php-extensions",
|
||||
"web-service-php-request-timeout","web-service-php-pass-authorization","web-service-php-body-limit","web-service-php-header-limit",
|
||||
"web-service-php-stderr-limit","web-service-php-custom-params","web-service-php-values","web-service-php-help",
|
||||
"web-service-compression","web-service-compression-options","web-service-compression-zstd","web-service-compression-br","web-service-compression-gzip","web-service-compression-deflate","web-service-precompressed","web-service-precompressed-br","web-service-precompressed-gzip",
|
||||
"web-service-preset","web-service-preset-apply","web-service-preset-summary","web-service-reset",
|
||||
"web-service-response-headers","web-service-rewrites","web-service-root","web-service-search",
|
||||
"web-service-symlink-policy","web-service-tabs","web-service-try-files"
|
||||
];
|
||||
const elements = new Map(ids.map(id => [id, new Element(id)]));
|
||||
const pages = ["general","routing","htaccess","mime","cache","delivery","php","security"].map(name => {
|
||||
const page = new Element(); page.dataset.webServicePage = name; return page;
|
||||
});
|
||||
|
||||
elements.get("web-service-php-endpoint").value = "unix:/run/php/php-fpm.sock";
|
||||
elements.get("web-service-php-connect-timeout").value = "1000";
|
||||
elements.get("web-service-php-request-timeout").value = "30000";
|
||||
elements.get("web-service-php-concurrency").value = "64";
|
||||
elements.get("web-service-php-pass-authorization").checked = true;
|
||||
elements.get("web-service-php-body-limit").value = "0";
|
||||
elements.get("web-service-php-header-limit").value = "65536";
|
||||
elements.get("web-service-php-stderr-limit").value = "4096";
|
||||
elements.get("web-service-compression").value = "";
|
||||
for (const name of ["zstd", "br", "gzip", "deflate"]) elements.get(`web-service-compression-${name}`).value = "";
|
||||
|
||||
const document = {
|
||||
body: new Element("body"),
|
||||
querySelectorAll(selector) {
|
||||
if (selector === "#web-service-tabs button") return elements.get("web-service-tabs").children;
|
||||
if (selector === "[data-web-service-page]") return pages;
|
||||
return [];
|
||||
}
|
||||
};
|
||||
let savedSection = null;
|
||||
const context = {
|
||||
console,
|
||||
structuredClone,
|
||||
document,
|
||||
window: {},
|
||||
envelope: null,
|
||||
busy: false,
|
||||
$: id => {
|
||||
const value = elements.get(id);
|
||||
if (!value) throw new Error(`missing fake element ${id}`);
|
||||
return value;
|
||||
},
|
||||
t: key => key,
|
||||
el: (_tag, text = "", className = "") => {
|
||||
const node = new Element();
|
||||
node.textContent = text ?? "";
|
||||
if (className) node.classList.values.add(className);
|
||||
return node;
|
||||
},
|
||||
toast: () => {},
|
||||
saveSection: async (name, value) => { savedSection = { name, value: structuredClone(value) }; return true; },
|
||||
appConfirm: async () => true,
|
||||
};
|
||||
vm.createContext(context);
|
||||
vm.runInContext(fs.readFileSync("src/admin/web/drafts.js", "utf8"), context, { filename: "drafts.js" });
|
||||
vm.runInContext(fs.readFileSync("src/admin/web/web-server.js", "utf8"), context, { filename: "web-server.js" });
|
||||
|
||||
// Explicitly disabling scoped dynamic compression must hide and disable codec
|
||||
// inheritance controls without erasing their saved selections.
|
||||
const compressionMode = elements.get("web-service-compression");
|
||||
const compressionOptions = elements.get("web-service-compression-options");
|
||||
const compressionZstd = elements.get("web-service-compression-zstd");
|
||||
compressionZstd.value = "off";
|
||||
compressionMode.value = "off";
|
||||
compressionMode.dispatch("change");
|
||||
if (!compressionOptions.hidden) throw new Error("Disabled Web Service compression did not hide codec overrides");
|
||||
for (const name of ["zstd", "br", "gzip", "deflate"]) {
|
||||
if (!elements.get(`web-service-compression-${name}`).disabled) throw new Error(`Disabled Web Service compression left ${name} override active`);
|
||||
}
|
||||
compressionMode.value = "on";
|
||||
compressionMode.dispatch("change");
|
||||
if (compressionOptions.hidden) throw new Error("Re-enabled Web Service compression did not restore codec overrides");
|
||||
if (compressionZstd.value !== "off") throw new Error("Compression visibility toggle erased the saved codec selection");
|
||||
|
||||
const preset = elements.get("web-service-preset");
|
||||
preset.value = "wordpress";
|
||||
preset.dispatch("change");
|
||||
|
||||
const expectedIndex = "index.php\nindex.html\nindex.htm";
|
||||
const expectedTry = "$uri\n$uri/\n/index.php?$query_string";
|
||||
if (elements.get("web-service-index").value !== expectedIndex) throw new Error("WordPress preset did not populate index files");
|
||||
if (elements.get("web-service-try-files").value !== expectedTry) throw new Error("WordPress preset did not populate try_files");
|
||||
if (!elements.get("web-service-php-enabled").checked) throw new Error("WordPress preset did not enable PHP-FPM");
|
||||
if (!elements.get("web-service-precompressed").checked) throw new Error("WordPress preset did not enable precompressed sidecars");
|
||||
if (!elements.get("web-service-cache-rules").value.includes("ext:css = public, max-age=604800")) throw new Error("WordPress preset did not populate cache rules");
|
||||
|
||||
preset.value = "downloads";
|
||||
preset.dispatch("change");
|
||||
if (elements.get("web-service-index").value !== "") throw new Error("Downloads preset should clear index files");
|
||||
if (!elements.get("web-service-listing").checked) throw new Error("Downloads preset should enable directory listing");
|
||||
if (elements.get("web-service-php-enabled").checked) throw new Error("Downloads preset should disable PHP-FPM");
|
||||
if (elements.get("web-service-cache-control").value !== "public, max-age=3600") throw new Error("Downloads preset cache policy missing");
|
||||
|
||||
// The save path must serialize the values materialized by preset selection, not
|
||||
// the previous/custom values that were present when the dialog opened.
|
||||
context.envelope = { config: { web_services: [] } };
|
||||
elements.get("web-service-id").value = "web1";
|
||||
elements.get("web-service-name").value = "web1";
|
||||
elements.get("web-service-root").value = "/srv/web1";
|
||||
elements.get("web-service-php-pass-authorization").checked = false;
|
||||
elements.get("web-service-php-body-limit").value = "1048576";
|
||||
elements.get("web-service-php-header-limit").value = "131072";
|
||||
elements.get("web-service-php-stderr-limit").value = "8192";
|
||||
elements.get("web-service-php-custom-params").value = "APP_ENV = production";
|
||||
elements.get("web-service-php-values").value = "memory_limit = 256M";
|
||||
preset.value = "wordpress";
|
||||
preset.dispatch("change");
|
||||
elements.get("web-service-compression").value = "on";
|
||||
elements.get("web-service-compression-gzip").value = "off";
|
||||
await elements.get("web-service-form").dispatch("submit");
|
||||
if (savedSection?.name !== "web-services" || savedSection.value.length !== 1) throw new Error("WordPress preset save did not persist a Web Service");
|
||||
const saved = savedSection.value[0];
|
||||
if (saved.index.join("\n") !== expectedIndex) throw new Error("Saved WordPress preset lost index files");
|
||||
if (saved.try_files.join("\n") !== expectedTry) throw new Error("Saved WordPress preset lost try_files");
|
||||
if (!saved.php.enabled) throw new Error("Saved WordPress preset lost PHP-FPM enablement");
|
||||
if (!saved.precompressed.enabled) throw new Error("Saved WordPress preset lost precompressed enablement");
|
||||
if (saved.compression?.enabled !== true || saved.compression?.gzip !== false) throw new Error("Saved WordPress preset lost compression/codec override");
|
||||
if (saved.php.pass_authorization !== false) throw new Error("Saved WordPress preset lost Authorization forwarding choice");
|
||||
if (saved.php.max_request_body_bytes !== 1048576 || saved.php.max_response_header_bytes !== 131072 || saved.php.stderr_log_bytes !== 8192) throw new Error("Saved WordPress preset lost advanced FastCGI limits");
|
||||
if (saved.php.custom_params.APP_ENV !== "production") throw new Error("Saved WordPress preset lost custom FastCGI parameters");
|
||||
if (saved.php.php_values.memory_limit !== "256M") throw new Error("Saved WordPress preset lost PHP_VALUE settings");
|
||||
if (!saved.cache_rules.some(rule => rule.matcher === "ext:css" && rule.cache_control.includes("604800"))) throw new Error("Saved WordPress preset lost cache rules");
|
||||
|
||||
console.log("PASS: Web Service preset selection materializes and persists CMS/download settings.");
|
||||
Executable
+147
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
"""LOOPBACK-ONLY fake Tinyauth protocol peer. NEVER use this as real authentication.
|
||||
A cookie named mock_auth with value allow approves a request. Any client can mint it.
|
||||
This exists to test RustProxy auth_request, identity forwarding and fail-closed paths.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
from collections import deque
|
||||
from http.cookies import SimpleCookie
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import ipaddress
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
class State:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.status = None
|
||||
self.delay_ms = 0
|
||||
self.location = None
|
||||
self.calls = 0
|
||||
self.recent = deque(maxlen=16)
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
def __init__(self, address, state=None):
|
||||
if not ipaddress.ip_address(address[0]).is_loopback:
|
||||
raise ValueError("The fake auth server may bind only a numeric loopback address")
|
||||
self.state = state or State()
|
||||
super().__init__(address, Handler)
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
server_version = "RustProxy-Mock-Auth/0.4.0"
|
||||
sys_version = ""
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.connection.settimeout(5)
|
||||
def reply(self, status, value=None, headers=()):
|
||||
data = b"" if value is None else json.dumps(value, ensure_ascii=True).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("Content-Type", "application/json")
|
||||
for key, value in headers:
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
try:
|
||||
self.wfile.write(data)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
pass
|
||||
def do_GET(self):
|
||||
path = urlsplit(self.path).path
|
||||
if self.headers.get("Transfer-Encoding") or self.headers.get("Content-Length", "0") != "0":
|
||||
self.close_connection = True
|
||||
return self.reply(400, {"error":"GET verification requests must have no body"})
|
||||
state = self.server.state
|
||||
if path == "/api/auth/nginx":
|
||||
cookie = SimpleCookie()
|
||||
try:
|
||||
cookie.load(self.headers.get("Cookie", ""))
|
||||
except Exception:
|
||||
pass
|
||||
allowed = "mock_auth" in cookie and cookie["mock_auth"].value == "allow"
|
||||
with state.lock:
|
||||
state.calls += 1
|
||||
# Do not store Cookie, Authorization, API keys, or query strings.
|
||||
state.recent.append({"method":self.headers.get("X-Original-Method"),
|
||||
"client":self.headers.get("X-Real-IP"), "host":self.headers.get("X-Forwarded-Host"),
|
||||
"proto":self.headers.get("X-Forwarded-Proto"),
|
||||
"forwarded":self.headers.get("Forwarded"),"xff":self.headers.get("X-Forwarded-For")})
|
||||
status = state.status if state.status is not None else (204 if allowed else 401)
|
||||
delay_ms, location = state.delay_ms, state.location
|
||||
time.sleep(delay_ms / 1000)
|
||||
headers = []
|
||||
if 200 <= status < 300:
|
||||
headers = [("Remote-User", "mock-user"), ("Remote-Email", "mock@example.test"), ("Remote-Groups", "testers")]
|
||||
if location:
|
||||
headers.append(("X-Tinyauth-Location", location))
|
||||
return self.reply(status, headers=headers)
|
||||
if path == "/state":
|
||||
with state.lock:
|
||||
return self.reply(200, {"calls":state.calls,"status":state.status,"delay_ms":state.delay_ms,"recent":list(state.recent)})
|
||||
if path == "/login":
|
||||
return self.reply(200, {"warning":"Fake login fixture, not authentication", "test_cookie":"mock_auth=allow", "instructions":"Send this cookie to the protected local test vhost."})
|
||||
self.reply(404, {"error":"not found"})
|
||||
def do_POST(self):
|
||||
if urlsplit(self.path).path != "/control":
|
||||
self.close_connection = True
|
||||
return self.reply(405, {"error":"only POST /control is supported"})
|
||||
try:
|
||||
if self.headers.get("Transfer-Encoding") or len(self.headers.get_all("Content-Length", [])) != 1:
|
||||
raise ValueError("one Content-Length is required")
|
||||
size = int(self.headers["Content-Length"])
|
||||
if not 0 <= size <= 4096:
|
||||
raise ValueError("body limit is 4096 bytes")
|
||||
data = self.rfile.read(size)
|
||||
if len(data) != size:
|
||||
raise ValueError("incomplete request")
|
||||
payload = json.loads(data)
|
||||
if not isinstance(payload, dict) or set(payload) - {"status", "delay_ms", "location", "reset"}:
|
||||
raise ValueError("invalid fields")
|
||||
status = payload.get("status")
|
||||
if status is not None and (type(status) is not int or status not in (200, 204, 401, 403, 500, 503)):
|
||||
raise ValueError("unsupported verification status")
|
||||
delay = payload.get("delay_ms", 0)
|
||||
if type(delay) is not int or not 0 <= delay <= 15000:
|
||||
raise ValueError("delay must be 0..15000 ms")
|
||||
location = payload.get("location")
|
||||
if location is not None and (not isinstance(location, str) or len(location) > 2048
|
||||
or any(ord(c) < 32 or ord(c) > 126 for c in location)):
|
||||
raise ValueError("invalid redirect header")
|
||||
with self.server.state.lock:
|
||||
state = self.server.state
|
||||
if payload.get("reset"):
|
||||
state.calls = 0
|
||||
state.recent.clear()
|
||||
state.status, state.delay_ms, state.location = status, delay, location
|
||||
except (ValueError, KeyError, TimeoutError, OSError) as error:
|
||||
self.close_connection = True
|
||||
return self.reply(400, {"error":str(error)})
|
||||
self.reply(200, {"updated":True})
|
||||
def do_HEAD(self):
|
||||
self.do_GET()
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--port", type=int, default=9300)
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.port <= 65535:
|
||||
parser.error("port must be 1..65535")
|
||||
server = Server(("127.0.0.1", args.port))
|
||||
print(f"FAKE AUTH - LOCAL TESTING ONLY: http://127.0.0.1:{args.port}/api/auth/nginx", flush=True)
|
||||
print("Approve with Cookie: mock_auth=allow. /control configures failures/delays. Ctrl+C stops.", flush=True)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+191
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local HTTP/1.1 origin for manual cache, health, compression and pool tests.
|
||||
Only Python's standard library is used. This is not a production application.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import ipaddress
|
||||
import json
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
MAX_BODY = 2 * 1024 * 1024
|
||||
MAX_WORKERS = 32
|
||||
|
||||
class State:
|
||||
def __init__(self, name="origin-a", ttl=30):
|
||||
self.name, self.ttl = name, ttl
|
||||
self.healthy, self.delay_ms, self.sequence = True, 0, 0
|
||||
self.lock = threading.Lock()
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
def __init__(self, address, state):
|
||||
self.state = state
|
||||
self.slots = threading.BoundedSemaphore(MAX_WORKERS)
|
||||
super().__init__(address, Handler)
|
||||
def process_request(self, request, client_address):
|
||||
if not self.slots.acquire(blocking=False):
|
||||
self.shutdown_request(request)
|
||||
return
|
||||
try:
|
||||
super().process_request(request, client_address)
|
||||
except BaseException:
|
||||
self.slots.release()
|
||||
raise
|
||||
def process_request_thread(self, request, client_address):
|
||||
try:
|
||||
super().process_request_thread(request, client_address)
|
||||
finally:
|
||||
self.slots.release()
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
server_version = "RustProxy-Mock/0.3.0"
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.connection.settimeout(15)
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
def send(self, status, body=b"", content_type="application/json; charset=utf-8", control="no-store", extra=None):
|
||||
if isinstance(body, dict):
|
||||
body = (json.dumps(body, ensure_ascii=False) + "\n").encode()
|
||||
if isinstance(body, str):
|
||||
body = body.encode()
|
||||
if status in (204, 304):
|
||||
body = b""
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
if status != 204:
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", control)
|
||||
self.send_header("X-Mock-Origin", self.server.state.name)
|
||||
for name, value in (extra or {}).items():
|
||||
self.send_header(name, value)
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
try:
|
||||
self.wfile.write(body)
|
||||
except (BrokenPipeError, ConnectionResetError, TimeoutError):
|
||||
self.close_connection = True
|
||||
def integer(self, query, key, default, maximum):
|
||||
value = int(query.get(key, [str(default)])[0])
|
||||
if not 0 <= value <= maximum:
|
||||
raise ValueError(key)
|
||||
return value
|
||||
def do_HEAD(self):
|
||||
self.do_GET()
|
||||
def do_GET(self):
|
||||
target = urlsplit(self.path)
|
||||
q = parse_qs(target.query)
|
||||
state = self.server.state
|
||||
try:
|
||||
delay = self.integer(q, "ms", 800 if target.path == "/slow" else 0, 10000)
|
||||
size = self.integer(q, "size", 65536, MAX_BODY)
|
||||
ttl = self.integer(q, "ttl", state.ttl, 604800)
|
||||
except ValueError:
|
||||
return self.send(400, {"error":"invalid query parameter"})
|
||||
with state.lock:
|
||||
state.sequence += 1
|
||||
sequence, healthy = state.sequence, state.healthy
|
||||
delay += state.delay_ms
|
||||
if delay:
|
||||
time.sleep(min(delay, 10000) / 1000)
|
||||
payload = {"origin":state.name, "sequence":sequence, "generated_at_ns":time.time_ns(),
|
||||
"nonce":secrets.token_hex(16), "path":target.path,
|
||||
"accept_language":self.headers.get("Accept-Language", ""), "healthy":healthy}
|
||||
if target.path == "/health":
|
||||
return self.send(200 if healthy else 503, payload)
|
||||
if target.path == "/state":
|
||||
return self.send(200, {**payload, "delay_ms":state.delay_ms, "ttl":state.ttl})
|
||||
if target.path == "/":
|
||||
return self.send(200, """<!doctype html><meta charset="utf-8"><title>RustProxy mock origin</title>
|
||||
<h1>RustProxy test origin</h1><p>Each uncached request generates a new sequence and nonce.</p>
|
||||
<ul><li><a href="/cache">/cache</a> - public JSON, 30-second freshness by default</li>
|
||||
<li><a href="/nocache">/nocache</a> - no-store JSON</li><li><a href="/private">/private</a> - private JSON</li>
|
||||
<li><a href="/cookie">/cookie</a> - Set-Cookie, bypass shared cache</li><li><a href="/large">/large?size=65536</a> - compressible bytes</li>
|
||||
<li><a href="/slow">/slow?ms=800</a> - controlled delay</li><li><a href="/health">/health</a> - active probe target</li>
|
||||
<li><a href="/status/503">/status/503</a> - custom error page test</li></ul>
|
||||
<p>POST /control with JSON {"healthy":false,"delay_ms":0} to test failover/recovery.</p>""", "text/html; charset=utf-8")
|
||||
if target.path == "/large":
|
||||
prefix = (json.dumps(payload) + "\n").encode()
|
||||
body = (prefix + b"RustProxy compression and cache fixture.\n" * (size // 30 + 1))[:size]
|
||||
return self.send(200, body, "text/plain; charset=utf-8", f"public, max-age={ttl}")
|
||||
if target.path.startswith("/status/"):
|
||||
try:
|
||||
status = int(target.path.rsplit("/", 1)[1])
|
||||
if status not in [200, 204, 400, 401, 403, 404, 408, 413, 429, 500, 502, 503, 504]:
|
||||
raise ValueError()
|
||||
except ValueError:
|
||||
return self.send(400, {"error":"unsupported status"})
|
||||
return self.send(status, payload, extra={"WWW-Authenticate":'Basic realm="mock"'} if status == 401 else None)
|
||||
if target.path == "/cache":
|
||||
return self.send(200, payload, control=f"public, max-age={ttl}", extra={"Vary":"Accept-Language"})
|
||||
if target.path == "/private":
|
||||
return self.send(200, payload, control="private, max-age=60")
|
||||
if target.path == "/cookie":
|
||||
return self.send(200, payload, control=f"public, max-age={ttl}", extra={"Set-Cookie":"mock-session=demo; HttpOnly; SameSite=Lax; Path=/"})
|
||||
self.send(200 if target.path in ("/nocache", "/slow") else 404, payload)
|
||||
def do_POST(self):
|
||||
# Always consume or close a body before reusing an HTTP/1.1 connection.
|
||||
if self.headers.get("Transfer-Encoding") or len(self.headers.get_all("Content-Length", [])) != 1:
|
||||
self.close_connection = True
|
||||
return self.send(411, {"error":"one Content-Length is required"})
|
||||
try:
|
||||
length = int(self.headers["Content-Length"])
|
||||
if not 0 <= length <= 4096:
|
||||
raise ValueError()
|
||||
raw = self.rfile.read(length)
|
||||
if len(raw) != length:
|
||||
raise ValueError()
|
||||
data = json.loads(raw)
|
||||
if self.path != "/control" or not isinstance(data, dict) or set(data) - {"healthy", "delay_ms"}:
|
||||
raise ValueError()
|
||||
if "healthy" in data and not isinstance(data["healthy"], bool):
|
||||
raise ValueError()
|
||||
if "delay_ms" in data and (type(data["delay_ms"]) is not int or not 0 <= data["delay_ms"] <= 10000):
|
||||
raise ValueError()
|
||||
except (ValueError, TimeoutError):
|
||||
self.close_connection = True
|
||||
return self.send(400, {"error":"expected healthy: boolean and/or delay_ms: 0..10000"})
|
||||
with self.server.state.lock:
|
||||
for key, value in data.items():
|
||||
setattr(self.server.state, key, value)
|
||||
self.send(200, {"updated":True})
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--port", type=int, default=9001)
|
||||
parser.add_argument("--name", default="origin-a")
|
||||
parser.add_argument("--ttl", type=int, default=30)
|
||||
parser.add_argument("--allow-remote", action="store_true", help="Explicitly expose this development-only server on a non-loopback IPv4 address")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
ip = ipaddress.ip_address(args.host)
|
||||
except ValueError:
|
||||
parser.error("--host must be an IPv4 address")
|
||||
if ip.version != 4:
|
||||
parser.error("this development helper accepts IPv4 bindings only")
|
||||
if not ip.is_loopback and not args.allow_remote:
|
||||
parser.error("non-loopback binding requires --allow-remote; /control is unauthenticated")
|
||||
if not 0 <= args.port <= 65535 or not 0 <= args.ttl <= 604800:
|
||||
parser.error("invalid port or TTL")
|
||||
if not args.name or len(args.name) > 64 or any(ord(c) < 32 or ord(c) > 126 for c in args.name):
|
||||
parser.error("--name must contain 1..64 printable ASCII characters")
|
||||
server = Server((args.host, args.port), State(args.name, args.ttl))
|
||||
print(f"Mock origin {args.name}: http://{args.host}:{server.server_port} (development only)", flush=True)
|
||||
print("Endpoints: /cache /nocache /private /cookie /large /slow /health /status/503; POST /control", flush=True)
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.2)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+260
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import time
|
||||
import zlib
|
||||
|
||||
import brotli
|
||||
import requests
|
||||
import zstandard
|
||||
|
||||
|
||||
TESTS = {
|
||||
"identity": "identity",
|
||||
"gzip": "gzip",
|
||||
"deflate": "deflate",
|
||||
"brotli": "br",
|
||||
"zstd": "zstd",
|
||||
"all": "zstd, br, gzip, deflate",
|
||||
}
|
||||
|
||||
|
||||
def decompress(data, content_encoding):
|
||||
encodings = [
|
||||
e.strip().lower()
|
||||
for e in content_encoding.split(",")
|
||||
if e.strip() and e.strip().lower() != "identity"
|
||||
]
|
||||
|
||||
# Content-Encoding usuwa się w odwrotnej kolejności.
|
||||
for encoding in reversed(encodings):
|
||||
if encoding == "gzip":
|
||||
data = gzip.decompress(data)
|
||||
|
||||
elif encoding == "deflate":
|
||||
try:
|
||||
data = zlib.decompress(data)
|
||||
except zlib.error:
|
||||
# Niektóre serwery wysyłają raw DEFLATE.
|
||||
data = zlib.decompress(data, -zlib.MAX_WBITS)
|
||||
|
||||
elif encoding == "br":
|
||||
data = brotli.decompress(data)
|
||||
|
||||
elif encoding == "zstd":
|
||||
dctx = zstandard.ZstdDecompressor()
|
||||
with dctx.stream_reader(io.BytesIO(data)) as reader:
|
||||
data = reader.read()
|
||||
|
||||
else:
|
||||
raise RuntimeError(f"Nieznany Content-Encoding: {encoding}")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def human_size(size):
|
||||
if size < 1024:
|
||||
return f"{size} B"
|
||||
if size < 1024 * 1024:
|
||||
return f"{size / 1024:.1f} KB"
|
||||
return f"{size / 1024 / 1024:.2f} MB"
|
||||
|
||||
|
||||
def request_test(url, accept_encoding, verify_ssl=True, timeout=15, revalidate=False):
|
||||
headers = {
|
||||
"Accept-Encoding": accept_encoding,
|
||||
"User-Agent": "compression-tester/1.0",
|
||||
}
|
||||
if revalidate:
|
||||
headers["Cache-Control"] = "no-cache"
|
||||
|
||||
start = time.perf_counter()
|
||||
|
||||
r = requests.get(
|
||||
url,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
verify=verify_ssl,
|
||||
stream=True,
|
||||
allow_redirects=True,
|
||||
)
|
||||
|
||||
r.raw.decode_content = False
|
||||
raw = r.raw.read()
|
||||
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
content_encoding = r.headers.get("Content-Encoding", "identity")
|
||||
|
||||
try:
|
||||
decoded = decompress(raw, content_encoding)
|
||||
error = None
|
||||
except Exception as exc:
|
||||
decoded = b""
|
||||
error = str(exc)
|
||||
|
||||
return {
|
||||
"status": r.status_code,
|
||||
"requested": accept_encoding,
|
||||
"received": content_encoding,
|
||||
"raw_size": len(raw),
|
||||
"decoded_size": len(decoded),
|
||||
"time": elapsed,
|
||||
"decoded": decoded,
|
||||
"error": error,
|
||||
"headers": r.headers,
|
||||
"final_url": r.url,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Tester HTTP Content-Encoding: zstd/br/gzip/deflate"
|
||||
)
|
||||
|
||||
parser.add_argument("url", help="URL do przetestowania")
|
||||
parser.add_argument(
|
||||
"--insecure",
|
||||
action="store_true",
|
||||
help="Wyłącz weryfikację certyfikatu SSL",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=int,
|
||||
default=15,
|
||||
help="Timeout HTTP w sekundach",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--revalidate",
|
||||
action="store_true",
|
||||
help="Wyslij Cache-Control: no-cache (wymusza rewalidacje i moze resetowac Age do 0)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.insecure:
|
||||
requests.packages.urllib3.disable_warnings()
|
||||
|
||||
print()
|
||||
print(f"URL: {args.url}")
|
||||
print("=" * 100)
|
||||
|
||||
results = {}
|
||||
|
||||
for name, accept_encoding in TESTS.items():
|
||||
try:
|
||||
result = request_test(
|
||||
args.url,
|
||||
accept_encoding,
|
||||
verify_ssl=not args.insecure,
|
||||
timeout=args.timeout,
|
||||
revalidate=args.revalidate,
|
||||
)
|
||||
results[name] = result
|
||||
except Exception as exc:
|
||||
results[name] = {
|
||||
"error": f"REQUEST ERROR: {exc}"
|
||||
}
|
||||
|
||||
identity_body = None
|
||||
|
||||
if "identity" in results:
|
||||
identity_body = results["identity"].get("decoded")
|
||||
|
||||
print(
|
||||
f"{'TEST':<10}"
|
||||
f"{'HTTP':<7}"
|
||||
f"{'REQUEST':<27}"
|
||||
f"{'RESPONSE':<12}"
|
||||
f"{'RAW':<12}"
|
||||
f"{'DECODED':<12}"
|
||||
f"{'RATIO':<10}"
|
||||
f"{'TIME':<10}"
|
||||
f"{'BODY'}"
|
||||
)
|
||||
|
||||
print("-" * 120)
|
||||
|
||||
failed = False
|
||||
|
||||
for name, result in results.items():
|
||||
if result.get("error"):
|
||||
print(f"{name:<10} ERROR: {result['error']}")
|
||||
failed = True
|
||||
continue
|
||||
|
||||
raw_size = result["raw_size"]
|
||||
decoded_size = result["decoded_size"]
|
||||
|
||||
if decoded_size:
|
||||
ratio = raw_size / decoded_size
|
||||
ratio_text = f"{ratio * 100:.1f}%"
|
||||
else:
|
||||
ratio_text = "-"
|
||||
|
||||
if identity_body is not None and result["decoded"]:
|
||||
if result["decoded"] == identity_body:
|
||||
body_status = "OK"
|
||||
else:
|
||||
body_status = "DIFF"
|
||||
else:
|
||||
body_status = "-"
|
||||
|
||||
print(
|
||||
f"{name:<10}"
|
||||
f"{result['status']:<7}"
|
||||
f"{result['requested']:<27}"
|
||||
f"{result['received']:<12}"
|
||||
f"{human_size(raw_size):<12}"
|
||||
f"{human_size(decoded_size):<12}"
|
||||
f"{ratio_text:<10}"
|
||||
f"{result['time']:.3f}s "
|
||||
f"{body_status}"
|
||||
)
|
||||
|
||||
print()
|
||||
print("=" * 100)
|
||||
print("Szczegóły nagłówków:")
|
||||
print()
|
||||
|
||||
for name, result in results.items():
|
||||
if result.get("error"):
|
||||
continue
|
||||
|
||||
headers = result["headers"]
|
||||
|
||||
print(f"[{name}]")
|
||||
print(f" URL finalny: {result['final_url']}")
|
||||
print(f" HTTP status: {result['status']}")
|
||||
print(f" Content-Encoding: {headers.get('Content-Encoding', '-')}")
|
||||
print(f" Content-Type: {headers.get('Content-Type', '-')}")
|
||||
print(f" Content-Length: {headers.get('Content-Length', '-')}")
|
||||
print(f" Vary: {headers.get('Vary', '-')}")
|
||||
print(f" Cache-Control: {headers.get('Cache-Control', '-')}")
|
||||
print(f" Age: {headers.get('Age', '-')}")
|
||||
print(f" X-Cache: {headers.get('X-Cache', '-')}")
|
||||
print(f" X-Cache-Hits: {headers.get('X-Cache-Hits', '-')}")
|
||||
print(f" Cache-Status: {headers.get('Cache-Status', '-')}")
|
||||
print(f" ETag: {headers.get('ETag', '-')}")
|
||||
print(f" Server: {headers.get('Server', '-')}")
|
||||
print()
|
||||
|
||||
print("SHA256 po dekompresji:")
|
||||
print()
|
||||
|
||||
for name, result in results.items():
|
||||
body = result.get("decoded")
|
||||
|
||||
if body:
|
||||
sha = hashlib.sha256(body).hexdigest()
|
||||
print(f"{name:<10} {sha}")
|
||||
|
||||
if failed:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+97
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Legacy SQLite snapshot helper retained for offline release tests.
|
||||
|
||||
Operator backups must use `rust-proxy backup create`, which includes the encrypted
|
||||
Secret Store key material and all durable data. This script is not a supported
|
||||
backup/restore interface after the Secret Store migration.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import stat
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
def backup(data_dir: Path, destination: Path) -> Path:
|
||||
os.umask(0o077)
|
||||
if data_dir.is_symlink() or not data_dir.is_dir():
|
||||
raise ValueError("Data directory must be a real directory, not a symlink.")
|
||||
if os.name == "posix" and stat.S_IMODE(data_dir.stat().st_mode) & 0o077:
|
||||
raise ValueError("Data directory must have mode 700.")
|
||||
source = data_dir / "config.sqlite3"
|
||||
if source.is_symlink() or not source.is_file():
|
||||
raise ValueError("config.sqlite3 must be a regular, existing file.")
|
||||
destination.mkdir(mode=0o700, parents=False, exist_ok=False)
|
||||
target = destination / "config.sqlite3"
|
||||
# The online backup API incorporates committed WAL contents; a plain file copy does not.
|
||||
reader = sqlite3.connect(f"file:{quote(str(source.resolve()), safe='/')}?mode=ro", uri=True, timeout=10)
|
||||
writer = sqlite3.connect(target)
|
||||
try:
|
||||
reader.backup(writer, pages=256, sleep=0.05)
|
||||
check = writer.execute("PRAGMA quick_check").fetchone()[0]
|
||||
if check != "ok":
|
||||
raise RuntimeError(f"SQLite integrity check failed: {check}")
|
||||
schema = writer.execute("PRAGMA user_version").fetchone()[0]
|
||||
revision, document = writer.execute("SELECT revision, document FROM configuration WHERE id = 1").fetchone()
|
||||
config = json.loads(document)
|
||||
finally:
|
||||
writer.close()
|
||||
reader.close()
|
||||
os.chmod(target, 0o600)
|
||||
digest = hashlib.sha256()
|
||||
with target.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
checksum = digest.hexdigest()
|
||||
certificate_hashes = {}
|
||||
names = set()
|
||||
settings = config.get("tls", {})
|
||||
for binding in settings.get("certificates", []):
|
||||
names.update([binding["cert_file"], binding["key_file"]])
|
||||
cert_dir = data_dir / "certs"
|
||||
if cert_dir.exists():
|
||||
if cert_dir.is_symlink() or not cert_dir.is_dir():
|
||||
raise ValueError("Certificate directory must not be a symlink.")
|
||||
for name in ["default.pem", "default.key"]:
|
||||
if (cert_dir / name).exists():
|
||||
names.add(name)
|
||||
if len(names) > 130:
|
||||
raise ValueError("Too many configured certificate files.")
|
||||
if names:
|
||||
(destination / "certs").mkdir(mode=0o700)
|
||||
for name in sorted(names):
|
||||
if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_.-]{1,96}", name) or ".." in name:
|
||||
raise ValueError("Unsafe certificate filename in the saved configuration.")
|
||||
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
||||
with os.fdopen(os.open(cert_dir / name, flags), "rb") as stream:
|
||||
info = os.fstat(stream.fileno())
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_size > 131072:
|
||||
raise ValueError("Certificate input must be a regular file up to 128 KiB.")
|
||||
content = stream.read(131073)
|
||||
if len(content) > 131072:
|
||||
raise ValueError("Certificate input grew beyond 128 KiB.")
|
||||
with (destination / "certs" / name).open("xb") as stream:
|
||||
stream.write(content)
|
||||
os.chmod(destination / "certs" / name, 0o600)
|
||||
certificate_hashes[name] = hashlib.sha256(content).hexdigest()
|
||||
metadata = {"project":"RustProxy", "created_at_unix":int(time.time()), "schema":schema,
|
||||
"configuration_revision":revision, "sha256":checksum, "certificate_sha256":certificate_hashes,
|
||||
"warning":"Contains password hashes, session state and configured TLS private keys. Keep private. Revoke sessions after restoring."}
|
||||
(destination / "backup.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8")
|
||||
return target
|
||||
|
||||
|
||||
def main() -> None:
|
||||
raise SystemExit(
|
||||
"tools/legacy/backup.py is retired for operator backups. "
|
||||
"Use: rust-proxy --data-dir <data-dir> backup create --output <backup.rpb>"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user