From 480c3ec7691426adea3150f893aaf6ee13618bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Gruszczy=C5=84ski?= Date: Sun, 18 Oct 2026 08:26:35 +0200 Subject: [PATCH] v0.34.0_8 --- AGENTS.md | 6 +- CHANGELOG.md | 12 +- README.md | 74 +-- ROADMAP.md | 22 +- SHA256SUMS | 111 +--- docs/API.md | 2 +- docs/CACHE.md | 2 +- docs/CACHE_HOTPATH_0.34.0.md | 9 +- docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md | 147 ------ docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md | 137 ----- docs/PROXY_1KB_OPT3_DEV_0.34.0.md | 88 ---- docs/PROXY_1KB_OPT4_DEV_0.34.0.md | 97 ---- docs/PROXY_1KB_OPT5_DEV_0.34.0.md | 93 ---- docs/PROXY_HOTPATH_0.34.0.md | 84 --- docs/PROXY_RPS_0.34.0.md | 78 --- docs/README.md | 6 +- docs/RELEASE_0.34.0_CACHE_FIX_PL.md | 93 ---- docs/ROADMAP_HISTORY.md | 2 +- docs/SECURITY_AUDIT_0.31.0.md | 148 ------ docs/SETTINGS.md | 2 +- docs/TEST_REPORT_0.34.0.md | 236 --------- docs/TODO.md | 19 - docs/VHOSTS.md | 2 +- docs/WEB_SERVER.md | 4 +- scripts/performance/README.md | 354 ------------- scripts/performance/ab_1kb_dev.py | 108 ---- scripts/performance/benchmark.py | 479 ------------------ scripts/performance/checks_cache.py | 180 ------- scripts/performance/checks_policy.py | 184 ------- scripts/performance/checks_proxy.py | 135 ----- scripts/performance/checks_static.py | 132 ----- scripts/performance/common.py | 448 ---------------- scripts/performance/concurrency_regression.py | 138 ----- scripts/performance/lb_regression.py | 223 -------- scripts/performance/peer.py | 223 -------- scripts/performance/profile_syscalls.py | 284 ----------- scripts/performance/regression.py | 107 ---- scripts/performance/resources.py | 228 --------- scripts/performance/simulate_cache_034.py | 112 ---- scripts/performance/simulate_nginx_gap_034.py | 98 ---- scripts/performance/soak.py | 325 ------------ scripts/performance/telemetry.py | 47 -- scripts/performance/transport_clients.py | 191 ------- scripts/performance/transport_regression.py | 281 ---------- scripts/performance/wrk_report.lua | 14 - scripts/preflight.py | 2 +- scripts/tests/cache_age_policy_034_model.py | 62 --- scripts/tests/cache_age_transform_034_live.py | 212 -------- scripts/tests/cache_hot_034_static_test.py | 34 -- scripts/tests/cache_scope_034_static_test.py | 40 -- ...shboard_chart_headers_0335_browser_test.py | 97 ---- ...oard_chart_typography_0337_browser_test.py | 168 ------ ...dashboard_chart_width_0333_browser_test.py | 110 ---- ...shboard_cpu_autoscale_0335_browser_test.py | 128 ----- ...board_cpu_fixed_scale_0335_browser_test.py | 149 ------ .../dashboard_layout_0331_browser_test.py | 199 -------- .../dashboard_metric_visuals_ui_test.mjs | 134 ----- .../dashboard_modal_0333_browser_test.py | 176 ------- ...board_overview_labels_0337_browser_test.py | 102 ---- scripts/tests/dashboard_presets_sql_test.py | 37 -- .../dashboard_resizing_0333_browser_test.py | 200 -------- .../dashboard_resume_rate_charts_ui_test.mjs | 115 ----- ...shboard_stable_scales_0336_browser_test.py | 129 ----- ...dashboard_title_color_0337_browser_test.py | 92 ---- .../tests/overview_samples_0330_ui_test.mjs | 116 ----- scripts/tests/panel_html.mjs | 22 - scripts/tests/panel_html.py | 18 - .../tests/progressbar_0332_browser_test.py | 107 ---- scripts/tests/proxy_1kb_opt3_static_test.py | 37 -- scripts/tests/proxy_1kb_opt4_static_test.py | 52 -- scripts/tests/proxy_1kb_opt5_static_test.py | 58 --- .../tests/proxy_hotpath_034_static_test.py | 47 -- scripts/tests/rps_hotpath_034_static_test.py | 129 ----- .../tests/sendfile_034_panel_browser_test.py | 82 --- scripts/tests/sendfile_034_static_test.py | 44 -- .../tests/sidebar_nav_0333_browser_test.py | 116 ----- .../system_ram_meter_0332_browser_test.py | 72 --- scripts/tests/test_stream_tcp.py | 137 ----- scripts/tests/test_stream_udp.py | 135 ----- scripts/tests/upgrade_performance_test.py | 80 --- .../vhost_backend_popup_0337_browser_test.py | 268 ---------- .../web_observability_0330_browser_test.py | 140 ----- .../web_observability_0330_source_test.py | 84 --- scripts/tests/web_server_0330_ui_test.py | 153 ------ scripts/tests/web_server_0334_browser_test.py | 155 ------ src/admin/documentation/documents.rs | 11 - src/admin/tests.rs | 2 +- src/logging/error_runtime.rs | 88 ++-- src/logging/format.rs | 24 +- src/logging/runtime.rs | 72 ++- src/logging/tests.rs | 63 +++ src/proxy/tests/authority_tests.rs | 70 ++- 92 files changed, 298 insertions(+), 10035 deletions(-) delete mode 100644 docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md delete mode 100644 docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md delete mode 100644 docs/PROXY_1KB_OPT3_DEV_0.34.0.md delete mode 100644 docs/PROXY_1KB_OPT4_DEV_0.34.0.md delete mode 100644 docs/PROXY_1KB_OPT5_DEV_0.34.0.md delete mode 100644 docs/PROXY_HOTPATH_0.34.0.md delete mode 100644 docs/PROXY_RPS_0.34.0.md delete mode 100644 docs/RELEASE_0.34.0_CACHE_FIX_PL.md delete mode 100644 docs/SECURITY_AUDIT_0.31.0.md delete mode 100644 docs/TEST_REPORT_0.34.0.md delete mode 100644 docs/TODO.md delete mode 100644 scripts/performance/README.md delete mode 100755 scripts/performance/ab_1kb_dev.py delete mode 100755 scripts/performance/benchmark.py delete mode 100755 scripts/performance/checks_cache.py delete mode 100755 scripts/performance/checks_policy.py delete mode 100755 scripts/performance/checks_proxy.py delete mode 100755 scripts/performance/checks_static.py delete mode 100755 scripts/performance/common.py delete mode 100755 scripts/performance/concurrency_regression.py delete mode 100755 scripts/performance/lb_regression.py delete mode 100755 scripts/performance/peer.py delete mode 100755 scripts/performance/profile_syscalls.py delete mode 100755 scripts/performance/regression.py delete mode 100755 scripts/performance/resources.py delete mode 100755 scripts/performance/simulate_cache_034.py delete mode 100755 scripts/performance/simulate_nginx_gap_034.py delete mode 100755 scripts/performance/soak.py delete mode 100755 scripts/performance/telemetry.py delete mode 100755 scripts/performance/transport_clients.py delete mode 100755 scripts/performance/transport_regression.py delete mode 100644 scripts/performance/wrk_report.lua delete mode 100755 scripts/tests/cache_age_policy_034_model.py delete mode 100755 scripts/tests/cache_age_transform_034_live.py delete mode 100755 scripts/tests/cache_hot_034_static_test.py delete mode 100755 scripts/tests/cache_scope_034_static_test.py delete mode 100755 scripts/tests/dashboard_chart_headers_0335_browser_test.py delete mode 100755 scripts/tests/dashboard_chart_typography_0337_browser_test.py delete mode 100755 scripts/tests/dashboard_chart_width_0333_browser_test.py delete mode 100755 scripts/tests/dashboard_cpu_autoscale_0335_browser_test.py delete mode 100755 scripts/tests/dashboard_cpu_fixed_scale_0335_browser_test.py delete mode 100755 scripts/tests/dashboard_layout_0331_browser_test.py delete mode 100644 scripts/tests/dashboard_metric_visuals_ui_test.mjs delete mode 100755 scripts/tests/dashboard_modal_0333_browser_test.py delete mode 100755 scripts/tests/dashboard_overview_labels_0337_browser_test.py delete mode 100755 scripts/tests/dashboard_presets_sql_test.py delete mode 100755 scripts/tests/dashboard_resizing_0333_browser_test.py delete mode 100644 scripts/tests/dashboard_resume_rate_charts_ui_test.mjs delete mode 100755 scripts/tests/dashboard_stable_scales_0336_browser_test.py delete mode 100755 scripts/tests/dashboard_title_color_0337_browser_test.py delete mode 100644 scripts/tests/overview_samples_0330_ui_test.mjs delete mode 100644 scripts/tests/panel_html.mjs delete mode 100755 scripts/tests/panel_html.py delete mode 100755 scripts/tests/progressbar_0332_browser_test.py delete mode 100755 scripts/tests/proxy_1kb_opt3_static_test.py delete mode 100755 scripts/tests/proxy_1kb_opt4_static_test.py delete mode 100755 scripts/tests/proxy_1kb_opt5_static_test.py delete mode 100755 scripts/tests/proxy_hotpath_034_static_test.py delete mode 100755 scripts/tests/rps_hotpath_034_static_test.py delete mode 100755 scripts/tests/sendfile_034_panel_browser_test.py delete mode 100755 scripts/tests/sendfile_034_static_test.py delete mode 100755 scripts/tests/sidebar_nav_0333_browser_test.py delete mode 100755 scripts/tests/system_ram_meter_0332_browser_test.py delete mode 100755 scripts/tests/test_stream_tcp.py delete mode 100755 scripts/tests/test_stream_udp.py delete mode 100755 scripts/tests/upgrade_performance_test.py delete mode 100755 scripts/tests/vhost_backend_popup_0337_browser_test.py delete mode 100755 scripts/tests/web_observability_0330_browser_test.py delete mode 100755 scripts/tests/web_observability_0330_source_test.py delete mode 100755 scripts/tests/web_server_0330_ui_test.py delete mode 100755 scripts/tests/web_server_0334_browser_test.py diff --git a/AGENTS.md b/AGENTS.md index cb64bc3..b86e307 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ Maintain English documentation and Markdown files. Keep translations in discover CHANGELOG.md uses version headings with no release dates. Patch increments may be bundled; substantial optional integrations raise the minor version. 0.7.0 adds managed renewal, server deployment helpers, Basic Auth and visual locations. The reported compression threshold type mismatch is removed without narrowing. Cloudflare Tunnel and Docker discovery are deferred. -Deliver a complete archive with dev.sh, tests and an updated verification report. +Deliver a complete archive with dev.sh and the existing tests. Keep verification results outside the repository; do not add iteration reports, standalone benchmark scripts, or micro-optimization notes to CHANGELOG.md. Do not include databases, credentials, real certificates/keys, target directories, node_modules or fabricated Cargo.lock content. Preserve an actually resolved lockfile when a compiler-enabled environment can produce it. @@ -40,8 +40,8 @@ Keep production Rust source files within 500 lines. Rust unit and integration **test files are exempt** from this limit: do not create numbered `part_XX.rs` fragments to satisfy source-size checks. Consolidate coherent test suites in `tests.rs` and preserve the existing test cases. -Run `./dev.sh check` and `./dev.sh test` when Rust is available. Run the SQL and upgrade -scripts even when it is not. UI tests use a simulated API; the Python protocol-peer +Run `./dev.sh check` and `./dev.sh test` when Rust is available. Run +existing SQL and upgrade checks when available; do not restore removed auxiliary scripts. UI tests use a simulated API; the Python protocol-peer self-test is not a test of RustProxy. Record these distinctions accurately. Do not mark a stage accepted or production-ready on the basis of static inspection. Review real protocol behavior, restored data and resource bounds before expanding diff --git a/CHANGELOG.md b/CHANGELOG.md index b132b94..90a87a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ - Source-only until OPT5 compilation and Rust tests. The compiled OPT4 input passed 151 live HTTP checks (0 failed, 1 optional Zstandard skipped). Brief shared-host OPT4/OPT3/nginx tests are mixed at 16/64 connections; - see `docs/PROXY_1KB_OPT5_DEV_0.34.0.md`. No version increment. + No version increment. - DEV 1 KiB follow-up (OPT4): keep ordinary unmodified public request targets borrowed through URI rewriting, avoid creating a metering body adapter for @@ -34,7 +34,7 @@ for complex values, TLS/H2/H3 authority handling and request limits. - OPT3 compiled binary passed 151 live HTTP checks, 0 failed, 1 skipped (optional Zstandard). OPT4 was subsequently compiled and tested similarly; - see `docs/PROXY_1KB_OPT4_DEV_0.34.0.md` and newer OPT5 analysis. + Results should be checked against matched configurations. - DEV 1 KiB throughput follow-up (OPT3): keep normalized safe routing @@ -50,7 +50,7 @@ fields and untrusted IPv6 forwarding. User-supplied OPT2-FIX1 binary passed 89 targeted live checks. A compiled OPT3 executable was subsequently tested (151 passing HTTP cases; one skipped), but short shared-host A/B did not - establish a stable RPS gain; see `docs/PROXY_1KB_OPT3_DEV_0.34.0.md`. + establish a stable RPS gain. - DEV 1 KiB accounting follow-up: move per-domain HIT/MISS/BYPASS, reason, stored-entry and saved-body statistics outside the HOT cache's shared mutex. @@ -64,7 +64,7 @@ - Extend Rust regressions for parallel BYPASS/HIT counting, reset semantics and saturated analytics. The supplied **previous** DEV executable passes 151 actual HTTP checks; these newest source changes are uncompiled and their RPS - uplift unmeasured. See `docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md`. + uplift unmeasured. - DEV 1 KiB reverse-proxy throughput follow-up: identify HTTP/1.1 upstream @@ -77,7 +77,7 @@ sanitization when the corresponding header is absent. Preserve analytics, history, counters, all security checks and cookie handling when present. - Add source guards, Rust regressions and a measured-configuration report in - `docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md`. This **source** is not compiled; + this source update. This **source** is not compiled; newly introduced code performance and full protocol parity remain unverified. No version increment and no claim of reaching 100k RPS / 80% nginx. @@ -102,7 +102,7 @@ supervisor rotation, and keep H1/H2 protocol telemetry. No altered upstream keepalive defaults or cache query policies. - Add source-only invariants, Rust regression cases and reproducible live before/after/nginx benchmarking instructions. This revision has not yet been - compiled or measured for RPS; see docs/PROXY_RPS_0.34.0.md. + compiled or measured for RPS. - Cache regression fix: interpret local max TTL as a cap on remaining freshness at receipt, not on the upstream Age clock. Preserve RFC 9111 Age and transformed response validators; prohibit private/no-store caching. diff --git a/README.md b/README.md index 57ffb10..12fbbd4 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,5 @@ # RustProxy 0.34.0 -## DEV: OPT5 multi-hot-path tuning (source candidate; uncompiled) - -OPT5 builds on the **compiled OPT4** (151/151 executed live HTTP checks passed; -1 optional Zstandard check skipped). The shared-host 1,040-byte HTTP/1.1 -comparison shows OPT4 at about 39,054 RPS with 16 connections and 36,425 RPS -with 64; mixed results versus OPT3 mean there is no proven broad RPS gain. -OPT5 removes unnecessary ID hashing and unauthenticated original-URL -construction, avoids per-success health-detail allocation, batches bounded -traffic analytics, avoids routine status/User-Agent allocations, and fast-paths -unoffered response compression. All policies, observability, caches and -protocol support remain enabled; no version bump. **OPT5 has not been built -or measured.** See [`docs/PROXY_1KB_OPT5_DEV_0.34.0.md`](docs/PROXY_1KB_OPT5_DEV_0.34.0.md). - -## DEV: OPT4 small-response proxy hot path (subsequently compiled) - -The compiled OPT4 binary passed 151 real HTTP acceptance checks (1 optional -Zstandard case skipped). OPT4 removes avoidable allocations and string work -for ordinary uncached HTTP/1.1 requests, with regressions for empty request -bodies, URI/Host/Connection semantics and Unicode log truncation. Its brief -shared-host RPS comparison is mixed, so no stable improvement is claimed. No settings or functionality disabled; -version remains 0.34.0. Details: -[`docs/PROXY_1KB_OPT4_DEV_0.34.0.md`](docs/PROXY_1KB_OPT4_DEV_0.34.0.md). - - -## DEV: small-response proxy hot path, OPT3 source candidate - -Seven compatible optimizations reduce path copies and work constructing -forwarded headers, log IPs, hop-by-hop headers, request IDs and cache-off -reason totals. HTTP/1.1 proxy with a 1,040-byte response remains the first -benchmark priority; local Web Server is secondary. No version or policy changes. -A subsequently supplied compiled OPT3 executable passed 151 real HTTP -assertions (one optional codec skipped). Two short same-host benchmark runs -did **not** establish a stable RPS gain over OPT2. See the historical -[`OPT3 source report`](docs/PROXY_1KB_OPT3_DEV_0.34.0.md) -and the newer [`OPT4 analysis`](docs/PROXY_1KB_OPT4_DEV_0.34.0.md). - -## DEV: cache counter contention and saturated analytics (source candidate) - -Per-domain cache outcome/reason accounting now uses bounded immutable indexes -and atomic counters rather than the HOT cache's global store mutex on every -request. The analytics producer reserves a queue slot before cloning fields; -full-queue/drop behavior and its counter remain unchanged. Per-vhost -latency histograms now update one exclusive atomic bin per completion and -reconstruct unchanged cumulative Prometheus buckets during scraping. The supplied -previous DEV binary passed 151 live HTTP assertions, but this newest code -**has not been built**. No version bump, no measured RPS claim. See -[`docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md`](docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md). - ## 0.34.0: HOT HIT critical-section follow-up (source candidate) Immutable response headers are shared until a HIT; the mutable per-response @@ -57,12 +9,6 @@ shorter paths. The existing stale-for-revalidation policy makes the previous full HOT sweep unnecessary. Exact LRU, generation barriers, BAN, COLD, transformation/ETag, TTL and cache bypass settings remain unchanged. -This is **uncompiled source**; do not infer network RPS or production acceptance -from source checks. See -[`docs/CACHE_HOTPATH_0.34.0.md`](docs/CACHE_HOTPATH_0.34.0.md). - -## 0.34.0: reverse-proxy hot-path optimization revision (source candidate) - This revision retains the **0.34.0** version and all prior scoped-cache, `sendfile`, security, logging and vhost/administration controls. In the uncached reverse-proxy hot path, request observation now passes the immutable @@ -87,9 +33,7 @@ subscribers only while subscribers exist; history and external sinks remain. These changes have **not** been compiled or timed in this environment, and there is no claimed 0.34.0 RPS improvement. Build and run the regression and -nginx benchmark matrix before deployment. See -[`docs/PROXY_HOTPATH_0.34.0.md`](docs/PROXY_HOTPATH_0.34.0.md) for invariants, -checks and exact measurement commands. +nginx benchmark matrix before deployment. ## 0.34.0: scoped cache, proxy accounting and opt-in Linux sendfile @@ -116,12 +60,6 @@ existing streaming path. The default is `off`; toggling this transport option does not purge HOT/COLD cache. **Native compilation and live protocol tests remain pending; do not deploy this candidate without them.** -An offline cache-retention simulation for a 20-vhost workload shows 92.123% -fewer invalidated objects and 92.113% fewer cache misses than the old global -purge policy **under its stated synthetic assumptions**. These are not RPS -figures; real 0.34.0 performance remains unmeasured. - -See `docs/TEST_REPORT_0.34.0.md` for verification scope and remaining gates. ## 0.33.8: request-path performance and compatibility verification @@ -150,12 +88,9 @@ to exercise the existing documented behavior, including scheme-isolated cache keys, purge generations, private administration responses and bundled error pages. The LB panel, database schema, default TLS/caching policies and bounded pool -recycling retain their existing behavior. Live process regression, verified TLS, -H2/H3/QUIC and WebSocket runners, repeated nginx comparisons, resource sampling, -syscall profiling and a mixed-traffic soak are included in -[`scripts/performance/`](scripts/performance/README.md). See the -[verification contract](docs/TEST_REPORT.md) and the separately generated -[performance results](docs/PERFORMANCE_0.33.8.md) for measured outcomes and limits. +recycling retain their existing behavior. Historical verification methods and +results are documented in [docs/TEST_REPORT.md](docs/TEST_REPORT.md) and +[docs/PERFORMANCE_0.33.8.md](docs/PERFORMANCE_0.33.8.md). ## 0.33.7: named dashboard presets, panel components and chart typography @@ -780,7 +715,6 @@ Historical release information remains in [CHANGELOG.md](CHANGELOG.md) and [docs - [docs/FILE_LOGS.md](docs/FILE_LOGS.md) - stored-log explorer, protocol-error records, bounds and filesystem security. - [docs/PHP_FPM.md](docs/PHP_FPM.md) - implemented FastCGI/PHP-FPM integration, safety model and runtime requirements. - [docs/TEST_REPORT.md](docs/TEST_REPORT.md) - verification results and environment limitations. -- [docs/TODO.md](docs/TODO.md) - unresolved follow-up only. ## Release history diff --git a/ROADMAP.md b/ROADMAP.md index afe356e..4dcf000 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -12,15 +12,12 @@ retain 0.34.0, TLS/H2/H3, WebSocket, cache, ACL, logs and bounded queues. - [ ] Compile and test OPT5; run full real-protocol regression and >=7 30-second CPU-isolated A/B/nginx runs before accepting a throughput gain. - - [x] OPT4 source candidate: borrow original no-rewrite public target, bypass the request body adapter only for bodies reporting end-of-stream, avoid redundant upstream URI refcount operations, and simplify common forwarding/header/log branches without weakening validation. - [x] Add OPT4 regression tests for body accounting and permits, Unicode, URI identity, and safe Connection header filtering. -- [ ] Complete OPT4/OPT5 Rust unit suites and extended TLS/H2/H3/WS/LB - acceptance; the short OPT4 RPS comparison did not prove a stable gain. - [x] DEV 1 KiB: remove shared HOT store mutex from cache outcome/reason @@ -30,8 +27,6 @@ - [x] Move per-vhost histogram aggregation to scrape time; retain the same cumulative Prometheus and Docker-vhost bucket semantics while one exclusive atomic bin is updated per completed exchange. -- [ ] Build/test revised counters source; repeat 1 KiB source A/B and pinned - nginx benchmark (100k RPS / 80% target still open). - [x] Share compiled route handles and reuse middleware routing in forwarding; avoid exact-host wildcard allocation and full-route cloning. @@ -39,10 +34,6 @@ between leases; reduce health locks and fallback/retry header snapshots. - [x] Convert three exchange metadata locks to atomic tags; avoid unused live log clones and add Rust regression fixtures for modified semantics. -- [ ] Compile this hot-path revision, run Rust tests and verify live LB health, - sticky/session behavior and HTTP/2 graceful shutdown retries. -- [ ] Profile CPU flamegraphs and measure real gains vs 0.33.8/nginx; tune - idle pool/recycle only as a separately labelled experiment. - [x] Disconnect keepalive/LB/diagnostic-only vhost saves from cache revisioning. - [x] Invalidate only a vhost scope when its origin/representation policy changes. @@ -50,16 +41,11 @@ interrupted content update or restart. - [x] Reduce response-body per-frame backend mutex operations without losing metrics. - [x] Add Rust regression cases and preserve unchanged LB controls/API. -- [ ] Compile, run the full Rust suites and repeat live HTTP/TLS/H2/H3/WS/LB regressions with a compatible Rust toolchain. -- [ ] Run paired 0.33.8 vs 0.34.0 vs nginx benchmarks; record real RPS/p95/p99, CPU, RSS, cache retention after saves, and rollback on failed acceptance. - [x] Implement opt-in Linux HTTP/1.1 `sendfile(2)` and a per-Web-Service Delivery toggle, with TLS/H2/H3/range/cache/compression fallback and bounded per-connection slots. Implementation is source-only until native tests run. -- [ ] Compile and validate native `sendfile`/fallback functionality and safety - on the target Linux OS; compare live WWW RPS against nginx. -- [ ] Complete long-running soak, external dependency integration and staged rollout. ## 0.33.8 - Performance with retained functionality @@ -76,10 +62,6 @@ - [x] Add repeatable live HTTP, verified TLS/H2/H3/WebSocket, nginx comparison, resource sampling, syscall profiling and mixed-traffic soak runners. Measured outcomes and limits are recorded in `docs/PERFORMANCE_0.33.8.md` and raw evidence. -- [ ] Complete deployment-specific 24-hour soak and remote-network testing with - actual PHP-FPM, Tinyauth and DNS infrastructure before production rollout. -- [ ] Evaluate further proxy pool/queue costs and a secure static-file sendfile - path from measured profiles; preserve TLS/H2/H3 and all observability semantics. ## 0.33.7 - named dashboard presets, focused HTML extraction and chart typography @@ -107,8 +89,6 @@ sentence case, all other panels uppercase); use mono for full legends, captions and chart data across all chart families. - [x] Validate SQL migration and run available simulated-browser/JS tests. -- [ ] Build/test Rust and run the new endpoint/UI against a real server with - representative persisted data. The production acceptance gate remains open. ## 0.33.6 - Stable dashboard chart ranges and plot heights @@ -745,4 +725,4 @@ Future Web Server work must retain the 0.25 split between reusable **Web Service - Race-resistant filesystem confinement for non-Unix targets before enabling local file serving there. - Real PHP-FPM release-host acceptance over Unix socket and loopback TCP. -See [`docs/TODO.md`](docs/TODO.md) for unresolved work and release acceptance, and [`docs/ROADMAP_HISTORY.md`](docs/ROADMAP_HISTORY.md) for completed historical milestones. +See [`docs/ROADMAP_HISTORY.md`](docs/ROADMAP_HISTORY.md) for completed historical milestones. diff --git a/SHA256SUMS b/SHA256SUMS index 0ce5ad2..708e3ae 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,13 +1,13 @@ 61bf40a0572889dc8b0dda9eae4e93e44b0de9364b3c798106c85700ffb37c23 .dockerignore 4140a2956264b44f259e6a37fc00ba02097bbe68b09296e04d3d243c1287129b .env.example 9e6356475bf587091bab72246ec9e325b498f7e48a6220e9a18baa50d0a385a6 .gitignore -46bab1d2c2d6e0a80d57a6b232b1ba4c220884b95cd45ddccbafce2aa160cddb AGENTS.md -3e46648fd8c91548ea77dcac4f6cbdbc5741c64b33240c44a50c138597880e99 CHANGELOG.md +5135a996f12151e481942bf3b4c46c42b675c1cf7086bd44733f3be260177d8c AGENTS.md +e7bea33973a805fdd1fd3af967555004b3509ef9b81576819c707c3dd5c47497 CHANGELOG.md ca201e3cd3f9842eae0006d0b40a0a72bcd3d4da6e4e5ecaf200d1b9a1390db2 Cargo.lock fec5f1bae9a59ef1fabd27c5e45d31448dd8f811736eebfa8826b3806a7e3349 Cargo.toml 25c7f3c8162cbf5cc156439774118a81dae9fdc52b611ec7e40c230fc99a772c Dockerfile -d48bcdcda6d9261d7b358c39b8bdf93d39b722b3751af3b5ebe192e16bd8b4e6 README.md -7bba42713fa944908f5f3611841bf6b36377240ae4b77ec692ff48ea662b44f9 ROADMAP.md +f0d89d95166d47b6ac2033cc8f5eaf2246c0c70139437394c98e10a95ab3d0dc README.md +b1d6174f509c743622cb27c92e047b724ac9d44bbf1c0987178da8a3bbd74b01 ROADMAP.md da8311e3db945290666324a2f24bf4521b365272d7ac3f862ded1be993eea646 UPDATE_CHECK.toml f645f218dd062bf6cb54a8ee634290562cad364a4cccf09f16e3c8303582af8c build.rs ea8d091843f86f4c66a5c76b20268b2feb397070851b6e05c5bc7d7a8d9157eb deploy/README.md @@ -27,12 +27,12 @@ e52b31d0abf65ae3034f6cb367cca113633014ce8d1d799ad4dd5624d2c7b8bb docker-agent/n 5220d669171ec3ad65f50e0bc8de5178dadaa4904def8e1a4f61002aa9e9a55b docker-agent/server.go 875faa3969df9409d79f050a84b4183496f557edda3d34aaec19fe556cf8c471 docs/ACME_PROVIDERS.md ec15163f5f51c9212003b7edd1fc380321311c2496fbaa9b45c838281b9af475 docs/ACME_RFC2136.md -00aef5eea37a8f56d7ea95e63ddcfdf39bac041168af29582b5ab2ab62d4c81e docs/API.md +70588369169c06e71925d6f45f98258153626cccbd168a77eb882a4a7ded7702 docs/API.md f27053f8a93d7883025a04922d015e0b697cad653e714be632b757e167297de1 docs/API_ACCESS.md d30551a52ffd34c5963094d2826ba39795210a4e6a4e4d7333066dae7456fc50 docs/ARCHITECTURE.md -fb99fac0a490af262a0ecdad593ffe5164004c2166fe7ddd607476bcc56a1623 docs/CACHE.md +7f61bc6e5c82b6519b50525eb2516298a5e28dbb28537d31eaf2f56bb7d5ff59 docs/CACHE.md 9113f3abbfe81bafc4855d8680df8aa2a381ecc358cc3e1a0f78b6b756421117 docs/CACHE_DIAGNOSTICS.md -a8fd061f8bdd920158fa42972f9d26418cbfe9d2cd8757ba1d9d1d12e5c908ee docs/CACHE_HOTPATH_0.34.0.md +de75fae9e387b2e8b3d478677963f8d67a124f3f02b746de285ca8b697b3816b docs/CACHE_HOTPATH_0.34.0.md 3822d72e12f82c37a9e70ab40c7e89e89145ad37f29914e4702db4bf024a5501 docs/CHANGELOG_TECHNICAL_ARCHIVE.md c8e69e492179ce39958cc63615cf46c4ac08cfdb7f3483aaab66868a3abcd687 docs/CONTAINER.md 833ddb12f23dcac796223521388b19c6ec3ea04c5550cf1c5f4ea4eb7c58f3a0 docs/CUSTOM_LOCATIONS.md @@ -56,33 +56,22 @@ be53a58afbeddbd7deb4b5f6447f2dc74db0327839bd83407c20915df98dcfa0 docs/NOT_FOUND e68d5a5cf0991de9308f78afb0264e02979f7463a5f395eb7a6e9ec2bd3a3d54 docs/PHP_FPM.md bc068a176056f5ea4d3b637071e644ed58a2140494c233ef22aa66e9700e3299 docs/PHP_FPM_ISOLATION.md ad6e3f003b820a4f45e30e39577d120a69d167c70910bda4891971759458c3dd docs/PROMETHEUS.md -401ab1bb2c7f3961742195d81f5daf5c9cf81609a4b175bea1d10d2a4b727082 docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md -f13ee0005d57d2576ccc970745f17069096df4afa23d712ea0f748da85433f7d docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md -c9bbdd1cd8df02b2a19016b7bbeee1cf55917614ac672f1f94e0a05db7d309ef docs/PROXY_1KB_OPT3_DEV_0.34.0.md -27002c4d4e19bb7d58521b8ba4cd5faf180d2917b6ac200dd7fc3664c2060145 docs/PROXY_1KB_OPT4_DEV_0.34.0.md -8c26aa9ebebf54d912749e3336e548b59d805997366be2324157c3e56b887f5f docs/PROXY_1KB_OPT5_DEV_0.34.0.md -deea89aed410c0619bdbdb12437edc7aff9dca2fec7fdc059e918c649fc36b51 docs/PROXY_HOTPATH_0.34.0.md -a7d6189b0e8cb854918456e0063101ee77bfb2419a9ba0430eb0e3f8678c0f09 docs/PROXY_RPS_0.34.0.md -887d1fdcc3b546b024afcd18b1716b8e3a74a3800012bf4bda7a122d2eabe233 docs/README.md +df6dc1070f94eeee7d4273251b93ab03b34e6cf0b02789d5b8f538a37814b382 docs/README.md 6e46a137b9c88dac8cb9e3fdca877430092392de9913d746278dbff8652460c2 docs/REDIRECTS.md 0dfd0d431c5aedc67abac1ce5de040ffa790ec9578fd7d6f91a987c333d73358 docs/REFERENCES.md -540fa46716d562bfc846d2f7bac0515380831ddbdcaa01ff4ac86be4ff3d3aa0 docs/RELEASE_0.34.0_CACHE_FIX_PL.md d85c5567fc6a7714b96ad7cfe4ee9ff2085f84c4f31534a6484340d03a73e0f7 docs/RENEWAL.md -cd4ed3af26eb51c9ff6ef3dda9d52ff5120bcfc78c842bce34a639cf85b6aea9 docs/ROADMAP_HISTORY.md +82bbcd1c3a3bb6a44065f96f160f8633b2f2450ab4723412570cc885a6de459b docs/ROADMAP_HISTORY.md 759bf2987f2b602488829d1856fa33d57c2a3b23bc681ef5a931b9633af3a533 docs/SECRET_STORE_BACKUP.md ec7e32fd19e6ba6ed1732f1f3dd92ac0860609e8c0bbd3c1998840cdfb24bf54 docs/SECURITY.md 620b69e566d27af04f0604f4ca3f88cf95cdfea7fbc6ae9dd92446d2b2b5d9be docs/SECURITY_AUDIT.md -007537ee971463072b2482dfa4fdcf1955f837ced0b711d4eef3b6267d16d4f6 docs/SECURITY_AUDIT_0.31.0.md -4dcf3243a13d9ae1827b5cefdbb9e27bd6310321c50d4d5934527d44437df6ef docs/SETTINGS.md +3434a638ef0198d659ad77e0a70d0540cf4f203e10e147713e6c2530ba84324a docs/SETTINGS.md 8fc112cb45a2f35ef5b0dd4371aa6f43aca4df85db62c6ae02e56d74a3b702ae docs/STREAMS.md 95148bfa1d2f8ff52b591c9acefc6c8992dadc4ebda43a195101d9b31d67a576 docs/TEST_REPORT.md -47e3cf646b2884db18684385ca7ff18d929b6de6a5206ed5e7ad27a71d350052 docs/TEST_REPORT_0.34.0.md -cfb452d8496839e832b5fbacdd4e52525b0ecc3112fbb62340c1e580bd5c1717 docs/TODO.md 751c9b13b47d0959a8575d6e82bdecba988b950a374ba0b084fbba08b4b65ebe docs/TRANSFORM.md e6d270894cafe96f8aa9cf58c9609223d03c9bcf6f872adfcbfb767a7b313f0a docs/TRANSPORT.md -811390a8c4ad6439dda4d532e35edc389db70f185ec728604ab35e22033f7e07 docs/VHOSTS.md +915da874d0b24e39b359fb70cc557730f7053a8334ced9ed406b6fa933c0dfaf docs/VHOSTS.md 09b60fb7b40db4d0139fa2a9e69a2aa9f28d4e61b8643521cfc7e936d7df8774 docs/VHOST_ACCESS.md -ca5d1d1d7e8aa372dcd3e2bd56bb7a9b8977953482ece560cfef58a19314242e docs/WEB_SERVER.md +cdf208dc6438d11c8119b6de81cad7a838eaeeb9cbbb5417e691253760d69ddf docs/WEB_SERVER.md c336f998027330c58d5afbbbb3a3a682fd44ae3997643882fe342633822f8bb3 docs/screenshots/README.md 5502e8ce44276213803b66c58e3a4d804a1354f5ed5d83f47ee805e8c44ab088 examples/acl-headers.json 2431dd88a2d9ad2cf3cd4b8fcdfa602563ab1013c612d8917fddbe7febfb2dc3 examples/cache-dashboard.json @@ -136,73 +125,13 @@ a641bd35400c56f3b68b0a92e97b602d06090f03972ca8f01fa744e90dd97403 scripts/deploy 2cc658faaa610b21b48942c59b955525539ad4503fd60f2a49c7e626a22faaff scripts/migration/migrate_npm.py a459d95bbb0076f33ade8b273b813308167f9d8d45242b309a2d18320ca0093c scripts/migration/migrate_npm_api.py 31ccad3ec3c880119f3b7002f0c6652f8b43634edad8ab3763cb0e4497d0bb1f scripts/migration/migrate_npm_core.py -bf934d59052988a528f3b7e7e3957b5a1ff1cf17db0d543e165cf76600e2137a scripts/performance/README.md -939287dadf7309689d29d2171757c2d80e91d898c3f9112454b943d08bd611fb scripts/performance/ab_1kb_dev.py -0380406a059ee9ec27473f8a46435f638f73a6dc729382ae1561c08dbf7ccb53 scripts/performance/benchmark.py -9ac220163256e08513597c397c016a48345b114d7876b439259cc310b36c83fc scripts/performance/checks_cache.py -f09125d081d0eed9a44fdca75de14e6bbf09dcb00fe1f027ba2270f89e3ff192 scripts/performance/checks_policy.py -96e01db050987af431a4662fce34752842bb2575237fd3239bcfc10ae55def6b scripts/performance/checks_proxy.py -c1d8c184811ae8fb1960d1ada08d8a4a95405c3b83dd18e1a27d711b596140be scripts/performance/checks_static.py -1044eb8d225e2fa117d5435697abdf0a014aa2f9063d4b41566e74ef47d50a34 scripts/performance/common.py -0f1fa62ee3429509dc21fe1cbfa7159e38d260ada98bfc6376e8078c9840ef48 scripts/performance/concurrency_regression.py -044da66a00a04f558a6b6e6157db0a7656dd5ee22adf0b0a47482c7f951380e9 scripts/performance/lb_regression.py -1b601d68d15909d786ac63890731f0592bd9057a68993a1d90bc70d6618e6f82 scripts/performance/peer.py -830a360609e2b8ebbb1ef6f3d89e3e6151e8fefb36c3673d20df68ff8932eabb scripts/performance/profile_syscalls.py -ceb26650fc8b63d0107eb9100ccd572e5c70f7a0ce5cc533a94083e20485095d scripts/performance/regression.py -d6480962a23f55d19a54aba27648ca1b28b50f03afe2decbfa272efd142df756 scripts/performance/resources.py -2dcd4c7e39dd58fd77c3c713d9bdee5025cace7191ca5f12bd6e7fe28f333358 scripts/performance/simulate_cache_034.py -4a5af2d3870d9517b4ad9c008789490cfb07affe98ffd629e51f7674ff54a642 scripts/performance/simulate_nginx_gap_034.py -f85dd0c29d5a96ad59a71e035ac4303c7b6a70a8d61280a15cdb966b32a218c7 scripts/performance/soak.py -fe6ff5bfd76ac7a2752a8a2e1e45c63e549c4a61c753008671e478df3ebff304 scripts/performance/telemetry.py -7c1ba46c623e1ade47b872fcdf4526a48ca85c56515dffeac626188b0357d23c scripts/performance/transport_clients.py -bd19431c89d7dcfe6743dffff9c4b16260a976d1936bdc4c11d4fcf6b3a7dda6 scripts/performance/transport_regression.py -1d747278da3fca093ed91c96d4a9480d0f711d0773d673906dbb56081635603c scripts/performance/wrk_report.lua -b5b3135449f6be4e97088b76ef7cf2428e2de3c7944c9cbbcb1edd2016c0fd4a scripts/preflight.py +e380bf369dcfd16e0e82d243d072bf8739e005d66b049c9b849a0d8b2d34162c scripts/preflight.py 3d6a0503afc25beab79ddc86864bcbb15248ded7368d33aa64321b8cdff59933 scripts/resolve_dependencies_0312.sh 20124173452b4514bad8850c1637569a9a191882478451e549bd16b432605fc1 scripts/runtime/acme_provider.py e69813fb1d3a4cdcfb7243964cc323fc21a603087bf2e84188fa42b9d3cee4b5 scripts/runtime/acme_rfc2136.py 174c8059625e3315a3d9fefe74c17f78ed850ff25ee15bfdabc148107dd344eb scripts/start_stream_backends.sh 046b4282550f5cb4e53906665c91c7442be70fc68fd0743466fd832582f7c979 scripts/test_stream_tcp.py f0b48ec0cc4411755cc0b4128eebce710375afa99344cb76198cf31074f8eb63 scripts/test_stream_udp.py -fafd577ccbb58485c47de35fc125e5c64a3a7628ae40d01d4cd924fce2f414db scripts/tests/cache_age_policy_034_model.py -b75a381d1b41bfb2ea7d2ce09728249b5e1465f76d57e2c3f151af72a5ae4d03 scripts/tests/cache_age_transform_034_live.py -dbab8d5457546a1ffe8233514ee94225a73974fe5d6f3b1f5fadd06885fa228c scripts/tests/cache_hot_034_static_test.py -6cb79535f742d1a41685f486688dc2f1330ea362e3da1a15d4fa32561094282f scripts/tests/cache_scope_034_static_test.py -efb7a1fde2c74f30cd4a1abb88ae07c1cac7378bb33685dd09d52bb30849daac scripts/tests/dashboard_chart_headers_0335_browser_test.py -eb2c9e5c968938b6fbe4675b547ba1f537519878b9095e1fea133f4d83f96fc8 scripts/tests/dashboard_chart_typography_0337_browser_test.py -6a908fd434d2a8a80adfd06a66d4b83c4174e448d453a8c10b1812475d036677 scripts/tests/dashboard_chart_width_0333_browser_test.py -252b401c29e2db50e105d7fb044bcae14752748261a67da1a4025d71bfa2608c scripts/tests/dashboard_cpu_autoscale_0335_browser_test.py -41a2b3765006c138324ae1312f5c1edfed6ce48051230f85779e8d2aad34d2a3 scripts/tests/dashboard_cpu_fixed_scale_0335_browser_test.py -de583adde771e0d6f64ea0d5aed45b1c942a7c78b7193598eeeb77fff00c2803 scripts/tests/dashboard_layout_0331_browser_test.py -1c813c78dfe96ba4d2c9470c93dd24e81be06e6485ebdc39cbfa30f8d32aa0fe scripts/tests/dashboard_metric_visuals_ui_test.mjs -f14626c5ec006477062b496bdc0025caa3366c7037ee598e15cd55e29041d58f scripts/tests/dashboard_modal_0333_browser_test.py -f934f5ce66105a1394b93761a6081a51133837ac038f5faf67a236f676b15c1b scripts/tests/dashboard_overview_labels_0337_browser_test.py -df87420f16c418aa17a8ba8808e506095681446d32a7d76ae0244222ed37928a scripts/tests/dashboard_presets_sql_test.py -df1457797241aab114a5671ce3f0229815dd29d06ab6f6254c0a1048576af6e1 scripts/tests/dashboard_resizing_0333_browser_test.py -149910e6e8d569443a724fd21701e2fd8f6b5e360991d4f710f5d21364402bce scripts/tests/dashboard_resume_rate_charts_ui_test.mjs -3868dc7b241f1cf15bc297104e29449ec3e35f75cb66078439bc4b5c4ee11040 scripts/tests/dashboard_stable_scales_0336_browser_test.py -fa42ec0475e4398d647b3c094309b4332cd2b5649db223cae0b8e821d6b44a28 scripts/tests/dashboard_title_color_0337_browser_test.py -3c1b2bda6410d3348b0f5a05c70494cca9636339bed8217fc4abc0a30981faae scripts/tests/overview_samples_0330_ui_test.mjs -72185f59f31bf04a4e88d53fe2fca52509f3663cb5f40b1d3bc319a5f945ef23 scripts/tests/panel_html.mjs -b0136544af717b75df2cb107900ea65c3f945fdd0040369dbb93919008fb954c scripts/tests/panel_html.py -8d1903e2b17b3d59cd0a5bec3e298e468b3d88ed2dda8b724239cc45d8353c59 scripts/tests/progressbar_0332_browser_test.py -7fc86a34a47c6e1f0f9fe2748d0ea7f59e455f8bf4e5ad081e185356e43721b0 scripts/tests/proxy_1kb_opt3_static_test.py -ec05272ad8c14c7a0265a84cc7b6ebc0dca0651413106d655681681873b88fc8 scripts/tests/proxy_1kb_opt4_static_test.py -37306b175190aa72b143d263060fa6441244eb40b2bcb54c82071cc87d73b54d scripts/tests/proxy_1kb_opt5_static_test.py -2a6e75c05f3860acf65ea8132f0083904478b690145115ec017c120d70f26cca scripts/tests/proxy_hotpath_034_static_test.py -0c50def966cbc95c22339d1c506763fab4d9e5c176a161205ab88d79af5be920 scripts/tests/rps_hotpath_034_static_test.py -5be5cdc60324f4f0f6638640000b888a5babb0a9bbf29511c824a0ba38878c2a scripts/tests/sendfile_034_panel_browser_test.py -978789bfe89d43034dfebddba49b9b17daedb1be031732734f95375e56c78306 scripts/tests/sendfile_034_static_test.py -1bb99b148d0297235b707f16865ca077329246fc19ecbb20a143bdeb56a23f37 scripts/tests/sidebar_nav_0333_browser_test.py -12f2b18b0ed2171c49eb8cde2c2b88fc627054c2f574c04bd9847821b00cd4c1 scripts/tests/system_ram_meter_0332_browser_test.py -046b4282550f5cb4e53906665c91c7442be70fc68fd0743466fd832582f7c979 scripts/tests/test_stream_tcp.py -f0b48ec0cc4411755cc0b4128eebce710375afa99344cb76198cf31074f8eb63 scripts/tests/test_stream_udp.py -6fa1667a844b7bc2adaa78738339b58ac6c94a6d6e7be34573433dcd323ae809 scripts/tests/upgrade_performance_test.py -f0a2e3daa0a49aac54723cb5a281360e49a97c076281372c5cca2a85e096c02c scripts/tests/vhost_backend_popup_0337_browser_test.py -23cb1f790cd5cee1b76ecd85f6a2a44ef2e16b0840bf06beaa2ae13048063b2f scripts/tests/web_observability_0330_browser_test.py -3babdd0fd6e2998755ae88615112f6ea056f5c808245abb6de60fbda555594d1 scripts/tests/web_observability_0330_source_test.py -f55efa99245bbdbc98fe3202e52952d224ac3200612561580d52f6e251810049 scripts/tests/web_server_0330_ui_test.py -0dd1cef06a1c3e1546e2fb9b085bf369dfbed02107b9ab01b4fb4f3b894099c0 scripts/tests/web_server_0334_browser_test.py 22e880e98c7b9d700c7a4c02b9b1d543e5dbd1f4ecdca42813a1083165b32a19 scripts/version.py 9bf4e4a61824f82b773b4e8ffa0909b9cf0a42bfc14d695bd64da240cc634a61 src/acl.rs 63efab1e95d41134fc30cc57320c32810bbae9f81a640bdb24b1be2059e63d8a src/acl/dynamic.rs @@ -222,7 +151,7 @@ b4a2a068acfa1143b0032b9cd072c19cf3fdda2580f0b932d11d6368cd2ab3ad src/admin/cert 51f46a2cb3167c4e3a784dfbb59b8fe0e9d5d3d51365b26699498e31d0afb6a9 src/admin/certificates.rs b5f6d49f1add583cfc8cd4aa36ffe4db56dac4de3b151fe8239fe42f507019cc src/admin/dashboard_presets.rs 5e150d4ef0413a2f92db5b6631669933b84d4b4796633d01476c4b39188cd05c src/admin/documentation.rs -958546c43c05bf4bed8121b24db3b3096f9243f3c56a71e1762592b755684013 src/admin/documentation/documents.rs +3bc1425c0dd142471a48391474ab2551de7d6f2b788280ed4788ab94a435c712 src/admin/documentation/documents.rs 67b901485d5858d5b5bf78a6260035db32241531d99481754c3533fdf404facb src/admin/documentation/tests.rs 1ce779b84aa8f47bdb3c34f83593c1f2f06d2235f0319847096cfcd035434168 src/admin/file_log_search.rs ab18acf6fb4ac86fb22635b9219a12dac76d24952c8f15538486267a9b4b7855 src/admin/file_logs.rs @@ -246,7 +175,7 @@ a40adfbaa908da3ea726d7aa3d45394a70a6a55b6fc0063270ec4d2a2f2026a3 src/admin/sess 4b8a0483ede651d08489a4d6e5bb616f06423094bbbefcee70b8755d65c04398 src/admin/status_live.rs c97881b17b7a42511868b0e48e2f25aab6fb5b613d538f5e1f1967508454e486 src/admin/status_system.rs cadd7dca29b9aad8c77c7bc23dd249ea37a15adfa4d636a8ef00273a91ce35dc src/admin/tail.rs -ccea3b72782aa9d36e41b89f40a755289d285692155c27cc81a0968f8780fe2a src/admin/tests.rs +58c9dbabfcfe37ce41d18e38777ead17781da91b3fc97bff19073fa09b262bd2 src/admin/tests.rs 48fcd818c522fe80f2c7b2e218f87de6a7772ffd0a24e93502780b03ec78e335 src/admin/web/acme.js c0d08c5b8fc13dec9bf53fd842d224b9df3874fc3bdf31413e3ab182a8cec2aa src/admin/web/app/backend-dashboard.js a085ee336e6669229f3ad2f74536f19bd088412ca79bcaff30dff293f1e1baf9 src/admin/web/app/chart-scales.js @@ -416,11 +345,11 @@ f2a8c2a98f4ccfb708f9d75a48ec7b351232be9b8a4364b7e63ba14d28eead7c src/http3/sett a31db7eac12b578b1eb5a74fae98bbe36312da95b7d7173135dbc41fc454da21 src/http3/tests.rs 68fb5ab6a889995a66ad12cb6acf88c5d04f0aca572e2ae15f1a9cd171a5f24b src/lib.rs 2551189a3e5701371026c9abd2c5d44d37495f28fb67e220180421bfd315e3dd src/logging.rs -18c8d74c21a47f234be93551d7264b1f8f8c00d1bbfbe555d91618c58cea0740 src/logging/error_runtime.rs -bc3873c6f22d1ee7fa76ae586547c9867e6eeaec8266e3ed1f529c87108df2b6 src/logging/format.rs +21ddb01b8f9c7fab60840d18103ef8de4716ff0c890cc1525b472b69af7916ec src/logging/error_runtime.rs +42b88cefef603fafa9eec3ba83a53069f01f571e98cf923210a6fe605e008e90 src/logging/format.rs f5815d778cc1785723faed4abb15f9880f22b6524b5289fe1e7158905aac0986 src/logging/model.rs -42fef80af50735cd566c6801dad86ba81be98cf4a8ec7d45c2ea26f53f556a22 src/logging/runtime.rs -7ee1eb1d3b52b4c14303f8ca14a1924d4d6bbe75b2e43ce3ca47d7958f792c7d src/logging/tests.rs +b90ee5c7badacd0141fba99a8109fced785a11abe11675c8687fa35e68f5e7b1 src/logging/runtime.rs +c83a7b84a9d7226c200394c351c4b3d9c417869828c01a35d4b1297670ab50f2 src/logging/tests.rs 3836ed14b6710e183f048ee4d7807a35366c27a27338057fe4a2811129b63302 src/main.rs fc8e99eac83c083d65101f6402aacbcc8392d66508c31566a50de3ebcb12e7da src/main/admin_runtime.rs 42a9197475823f67c1ca762951ede5d7a2f10124bd6ed4d1a3365f4297be894c src/main/bootstrap.rs @@ -466,7 +395,7 @@ cc7fe83f42acb19d79abe83107427cdc6b0ed14083702f4fe66af22f7cdfc4cc src/proxy/head 55133ceace2e5262d9c54fa41bea099a27b817d1e09c1cd0e2493fd9cee4da38 src/proxy/redirect_helpers.rs 2280841494c3e14ccc5e3f02b026fe3c4328a3dab04ab449b9286bb69e39c028 src/proxy/request_id.rs b9a2307c6aa7f16613d935508a61574dfee964020b1b5be86293df2d684615d8 src/proxy/request_id/tests.rs -e8d1210a6b8e8c6a388d016d22e7c18255cf737c5583124b35cd50bd2592f23c src/proxy/tests/authority_tests.rs +3db4188e2cc6d69fdff1b817cc15d0e6b18bda18d4e4729d37c05736c7d8cad6 src/proxy/tests/authority_tests.rs 1aa9bbbfb2659facdaeaf185e11221b1c8cd5fe6e23c7c1dacd3aa55e3db4fcb src/proxy/tests/cache_fast_path_tests.rs 0f2eacadf1447db8aa1c1a78005aa262e0f747096bfe7e0a0e41a2cd31a0d17f src/proxy/tests/h3_navigation_tests.rs d134b0eb617f82f9971b3f3461379ea37c57bd0079bf54748d0c434b2ef1b475 src/proxy/tests/http_invalidation_tests.rs diff --git a/docs/API.md b/docs/API.md index e7b49ad..c244ddc 100644 --- a/docs/API.md +++ b/docs/API.md @@ -21,7 +21,7 @@ additional requests receive 429. Ambiguous duplicate control headers and request bodies are rejected. See [CACHE](CACHE.md#http-ban--purge-from-authorized-client-ips) for request examples, scoping, confirmation and reverse-proxy requirements, and -[0.31.0 security review](SECURITY_AUDIT_0.31.0.md) for audit findings. The authenticated +[security review](SECURITY_AUDIT.md) for audit findings. The authenticated `/api/cache/invalidate` endpoint remains unchanged. ## Browser documentation and anonymous surface diff --git a/docs/CACHE.md b/docs/CACHE.md index b8fb4d2..b4c5513 100644 --- a/docs/CACHE.md +++ b/docs/CACHE.md @@ -232,7 +232,7 @@ separate from the administrator-only selective invalidation API. If an application changes a GET resource after POST/PUT/PATCH/DELETE, use origin freshness/validators or one of the explicit invalidation paths; RustProxy does not guess the affected URLs. For exposure review and test scenarios, see -[0.31.0 security audit](SECURITY_AUDIT_0.31.0.md). +[security audit](SECURITY_AUDIT.md). ## Eligibility and freshness diff --git a/docs/CACHE_HOTPATH_0.34.0.md b/docs/CACHE_HOTPATH_0.34.0.md index 448f6eb..e19a504 100644 --- a/docs/CACHE_HOTPATH_0.34.0.md +++ b/docs/CACHE_HOTPATH_0.34.0.md @@ -54,15 +54,12 @@ Native Rust test file: `src/cache/tests/hot_hit_034_tests.rs`. ./dev.sh check CARGO_INCREMENTAL=0 ./dev.sh test CARGO_INCREMENTAL=0 cargo build --release --locked -python3 scripts/tests/cache_hot_034_static_test.py cargo test --release --locked --lib manual_hot_lookup_microbenchmark -- --ignored --nocapture ``` -For real HTTP RPS use the supplied process regression and same-host nginx -benchmark (`scripts/performance/README.md`) with **identical** configs. The -standard nginx comparator benchmark explicitly disables the cache; it cannot -measure this HOT-specific improvement. Compare two compiled RustProxy 0.34.0 -binaries with the same cache-enabled, cache-warmed vhost and fixed no-query URL. +For real HTTP RPS, compare two compiled RustProxy 0.34.0 binaries +with the same cache-enabled, cache-warmed vhost and fixed no-query URL. +A cache-disabled nginx comparison cannot measure this HOT-specific improvement. Check `X-Cache: HIT`, body SHA-256, exact `Cache-Status`, transform marker, Age, ETag, p95/p99, CPU/request, memory and warm-up/eviction counts while measuring. diff --git a/docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md b/docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md deleted file mode 100644 index ba6f1e5..0000000 --- a/docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md +++ /dev/null @@ -1,147 +0,0 @@ -# RustProxy 0.34.0 DEV — 1 KiB lock contention follow-up - -## Goal and baseline - -**Priority 1: reverse proxy, HTTP/1.1 keep-alive, uncompressed 1,040-byte -responses, cache OFF.** Aiming for approximately 100,000 RPS and at least 80% -of nginx under the same pinned-CPU benchmark. Priority 2: local static WWW, -measured separately. No feature removals and **no version bump**. - -The user supplied a **new executable** (SHA-256 -`1a84c58b20945bba9fdd1bc1a44d9e9d6ac7659d714343a46a1a8511a78331fa`, -`rust-proxy 0.34.0`), apparently built from the prior DEV revision. It passed -151 actual black-box HTTP tests with 0 failures and 1 skipped optional zstd -decoder. This runtime coverage belongs to the **supplied executable only**, -not to the freshly edited source below. Neither `cargo` nor `rustc` is -available in the editing environment; the new revision has NOT been compiled. - -## Modified source - -### `Cache::record` and `Cache::record_reason` - -Before, every HIT, MISS and BYPASS acquired the same `Store` mutex used by the -HOT object store, exact LRU and cache inventory. With cache disabled, a 1 KiB -request still contested that mutex to increment `cache_disabled`. - -`src/cache/stats.rs` now holds bounded per-(scope,domain) statistics separately: - -- An `ArcSwap`-published immutable, borrowed-lookup BTreeMap of domain handles. - **The normal path does not lock or allocate**, even for BYPASS. New domains - alone use a registration mutex and a copy-on-first-visit registry update. -- Atomic counts for HIT, MISS, BYPASS, stored objects and saved body bytes. - The common static reasons (`cache_disabled`, `cache_hit`, `cache_miss`, - `request_cookie`) use direct atomic counters. Other reason names use another - immutable published index and atomic counters; they are registered once. -- Hard cardinality remains **4096 (scope, domain) pairs**. When full, new - unregistered domains are skipped as before. Scope purge preserves stats. - Only a revision-bearing global purge resets the per-domain registry. -- The dashboard still exposes the same `DomainCounters`, reason strings, - saved byte counts, ratio and per-vhost/object views. Counter snapshots are - quiescent-exact, but under concurrent requests individual fields can reflect - slightly different instants, consistent with other atomic metrics. -- HOT object and COLD representation/LRU access, barriers, memory accounting, - cache statuses, hit policy, retry and backend security paths are unchanged. - -Trade-offs: a first-visit registration copies the bounded domain index and -allocates one counter set; this is not an adaptive, unbounded cache. The hot -path's ArcSwap guard and lookup still cost CPU. Measure that cost in profiles; -compare under high-cardinality traffic to identify any excessive registration -cost before production acceptance. - -### Traffic analytics queue saturation - -Previously `Exchange::drop` cloned the vhost, route, method, path, IP, user -agent and country before trying to enqueue the `Sample`. If the bounded -2,048-element MPSC queue was full, all those clones were discarded. - -`Analytics::push_lazy` now acquires the same bounded Tokio queue permit with -`try_reserve` first and creates the sample only after a successful reservation. -Rejections increment `queue_dropped` exactly as before. Neither the analytics -worker, counters/top lists, reset behavior nor request log path is removed. -This improves allocation costs **on overflow** but does not turn dropped -analytics into delivered samples; monitor the existing drop counter. - -### Per-vhost duration histogram atomics - -Previously `VhostMetrics::complete` incremented every cumulative latency bucket -whose threshold covered the observation and the `+Inf` bucket. For an ordinary -short response this meant **11 shared atomic RMWs** for the histogram alone, -on top of existing exchange/status/byte/latency metrics. - -It now increments **exactly one exclusive bucket** chosen from the same 10 -latency boundaries. Both the regular vhost and Docker-vhost Prometheus exporters -compute the original cumulative `le` buckets, `+Inf` and `_count` from the 11 -exclusive buckets **at scrape time**. The metric names, labels, boundaries, -body-lifetime timing and measurement collection remain intact. The cumulative -snapshot may be concurrent with request completions, as before. Added boundary -unit tests for exact thresholds and long-duration `+Inf`. - -This is source-only: neither the throughput benefit nor compiled Rust behavior -has been verified in the editing environment. Review exporter parity and -under-load histogram correctness on the newly compiled executable. - -## Verification and untested claims - -- PASS: source preflight, cache/HOT/LB/RPS/histogram/static source guards and <=500-line - production Rust modules. -- PASS: SQLite migration + preset constraints, upgrade/lock and source - replacement tests. -- PASS: compiled user-supplied binary: 151 real HTTP checks, 0 failed, - 1 optional zstd skip. This binary predates the source changes above. -- NOT RUN: Rust compile, Rust unit tests, native regressions or RPS after - rebuilding the new source. No numerical speedup attributed to this patch. -- Current diagnostic A/B on the user-supplied binary may use - `scripts/performance/ab_1kb_dev.py` and must remain labelled *old vs already - supplied binary*, **not** old vs newly edited source. The `wrk` 30 s x 7 - pinned-CPU benchmark remains the production acceptance gate. - -## Repeat on a compiler-enabled benchmark host - -```bash -./dev.sh doctor -./dev.sh check -CARGO_INCREMENTAL=0 ./dev.sh test -CARGO_INCREMENTAL=0 cargo build --release --locked -python3 scripts/performance/regression.py \ - --binary previous=/absolute/path/to/supplied-rust-proxy \ - --binary edited="$(realpath target/release/rust-proxy)" \ - --out /tmp/rustproxy034-counter-http -python3 scripts/performance/lb_regression.py \ - --binary previous=/absolute/path/to/supplied-rust-proxy \ - --binary edited="$(realpath target/release/rust-proxy)" \ - --out /tmp/rustproxy034-counter-lb -python3 scripts/performance/transport_regression.py \ - --binary previous=/absolute/path/to/supplied-rust-proxy \ - --binary edited="$(realpath target/release/rust-proxy)" \ - --out /tmp/rustproxy034-counter-transport -python3 scripts/performance/benchmark.py \ - --binary previous=/absolute/path/to/supplied-rust-proxy \ - --binary edited="$(realpath target/release/rust-proxy)" \ - --nginx /usr/sbin/nginx --wrk /absolute/path/to/wrk \ - --out /tmp/rustproxy034-counter-benchmark \ - --modes proxy --sizes small --concurrency 16,64,128 \ - --duration 30 --warmup 5 --repeats 7 -``` - -Explicitly set `max_idle_per_host=64` on **both** before/after vhosts, with the -same TLS/H1/H2/health/cache/logging/settings. Verify 1,040 bytes and SHA-256, -no failures, latency percentiles, per-request CPU, RSS, file descriptors, -backend connects and all log/traffic drop counters. Test concurrent global -purges and per-vhost purges during traffic and verify the stats reset behavior -and cache HIT/MISS/BYPASS/reason distribution. Run long soaks and both cold/HOT -cache tests. The 100k RPS / nginx 80% goal is still **OPEN**. - -## Build fix after first compiler feedback (OPT2-FIX1) - -The first user build of OPT2 failed with six Rust `E0753` diagnostics because -`src/cache/stats.rs` starts with `//!` *inner module* documentation but is -injected via `include!("stats.rs")` after other items in `src/cache/mod.rs`. -The six documentation lines are now ordinary `//` comments; there is no -behavior change. Three unnecessary `mut` bindings in -`src/cache/dashboard.rs` were also removed (warnings only). Version remains -0.34.0; no performance or feature semantics have changed in this fix. - -The compiler-enabled host should rerun `./dev.sh check`, -`CARGO_INCREMENTAL=0 ./dev.sh test` and an optimized release build, then -performance benchmarks. This editing environment still lacks `cargo` and -`rustc`, so the fix has been statically checked, **not compiled** here. diff --git a/docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md b/docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md deleted file mode 100644 index 96428e7..0000000 --- a/docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md +++ /dev/null @@ -1,137 +0,0 @@ -# RustProxy 0.34.0 DEV: 1 KiB reverse-proxy throughput audit - -## Goal and evidence boundaries - -Priority 1 is short (~1 KiB) HTTP reverse-proxy responses: aim for approximately -100,000 requests/s, and eventually at least 80% of nginx under the **same** -controlled benchmark, without disabling authentication, ACLs, caching semantics, -rate limits, traffic accounting, request IDs, logs, body streaming or protocol -support. Priority 2 is local WWW serving, to be tuned and measured separately. -This is a development revision; Cargo/package version remains **0.34.0**. - -**This source is not compiled in the current environment.** Neither `cargo` -nor `rustc` is available. Only the user's pre-existing executable (SHA-256 -`453757748455d7b9b50a6140e322ee4d4cf4a030eea29318686f2e28c594b644`) -was run. Live configuration A/B tests therefore substantiate a **configuration -bottleneck**, not the throughput of this edited source. - -## Real baseline experiment (existing executable only) - -Disposable loopback nginx upstream, 1,040-byte body, HTTP/1.1 persistent -frontend connections, caching disabled, identity response encoding, four Tokio -workers, ApacheBench (`ab -k -n 12000`), 16/64 concurrent frontend clients. -Checks outside the timed intervals verified HTTP 200, exact 1,040 bytes and -SHA-256 `62a08d75cdde0416c883d72458385b5e56ae8c14466934ad784587ddb93a548b`; -0 failed requests in the shown samples. The host has shared CPU resources; -results vary and are not a pinned-CPU or production acceptance benchmark. - -| Existing executable configuration | 16 clients, RPS | 64 clients, RPS | -|---|---:|---:| -| max_idle_per_host = 4 | 18,226 | 17,935 | -| max_idle_per_host = 64 | 30,906 | 24,872 | -| Relative change (configuration only) | +69.6% | +38.7% | - -A separate repeat experiment at 64 clients (two runs per variant, in order) -produced: `idle4` 14,028 / 14,834; `idle16` 22,205 / 24,711; -`idle32` 25,580 / 29,088; `idle64` 36,521 / 33,847; -`idle64 + recycle 250,000` 31,168 / 29,753; -`idle32_again` 34,798 / 29,194; `idle64_again` 34,677 / 29,965; -`idle4_again` 23,503 / 23,514 RPS. The order drift warns against -reporting a single percentage as a robust speedup. Raising generation -recycling from 25,000 to 250,000 did not establish any improvement; its -original limit is retained. All shown variants reported zero HTTP failures. - -A preliminary nginx comparison gave ~75k/117k RPS at 16/64 clients, but later -nginx samples were ~50k/93k on the shared host. Do not compute a definitive -"% nginx" from these non-isolated runs. The 100k/80% goals remain **OPEN**. - -## Source modifications - -| Location | Change | Preserved behavior / caveat | -|---|---|---| -| `src/balance/model.rs` | New vhost default `max_idle_per_host`: **4 -> 64** | Configured values **1..64** remain validated; existing explicitly saved `4` stays `4`. Disabled keepalive, 30 s idle timeout, 25k-request and 3,600 s recycling rules are unchanged. Increased idle sockets use more file descriptors and memory. | -| `src/admin/web/vhosts/editor.js` | Match the new `64` default when the field is absent | Existing values, limits and all editor settings remain unchanged. | -| `src/proxy/body.rs` | At `Exchange::drop`, use exclusive `Mutex::get_mut()` rather than lock/unlock; move owned `route`, `client_ip`, `country`, `upstream_error` into final observation where possible | Exactly the same log fields, analytics sample, counters, permit lifetime, errors and websocket/H3 fields. Analytics still gets a clone when a vhost sample is queued; history still receives the original. | -| `src/proxy/headers.rs` | Skip reserved cookie sanitization immediately if the corresponding `Cookie`/`Set-Cookie` header is absent | Full sanitization, admin-cookie separation and restoration restrictions still apply when the header is present. | -| `src/balance/tests.rs`, `src/proxy/body/tests.rs`, `src/proxy/headers/tests.rs` | Add compatibility, completion and cookie regression assertions | These Rust tests **require** a compiler before they count as run. | -| `scripts/tests/*hotpath*` | Update source-only guards for the new **bounded** default | Static assertions do not prove functional or runtime performance equivalence. | - -**Persistence note:** old vhosts may store `keepalive.max_idle_per_host=4` -explicitly; the new code default does not rewrite them. To apply the measured -configuration improvement before rebuilding, change that option to `64` in the -relevant vhost and verify actual idle socket/FD usage. Where hundreds of origins -share a host, test `16` or `32` if the FD/memory trade-off is preferable. - -## Bottlenecks to profile next, not silently disable - -1. `src/cache/dashboard.rs::Cache::record`: obtains a **shared cache-store - mutex on every outcome**, including `BYPASS` when cache is disabled through - `src/proxy/forward/cache_lookup.rs`. This may compete with HOT cache - operations and admin dashboard reads; profile `pthread_mutex`/futex and - lock hold times. Candidate: separate bounded, sharded domain/reason - counters from the HOT object store while preserving exact dashboard totals, - reason labels and reset semantics. -2. `src/logging/format.rs::Logs::push`: per-request bounded history - `try_lock`, potentially causing recorded log drops under high concurrency. - Preserve *existing* drops/accounting, live subscribers and external sinks. - Candidate: worker-local bounded batches or sharded history with deterministic - bounded merge; validate log order, recent history, retention and drops. -3. `src/proxy/body.rs::Exchange::drop` + `src/traffic.rs::Analytics::push`: - request-sized log strings, sampling queue sends, atomics and lifecycle - observations. Profile allocations and CPU per completed request. Candidate: - compact shared immutable identity fields or conditional cloning solely - where consumers require ownership, without removing logs or analytics. -4. `src/proxy/forward/route_cache.rs`: request-target path copies, - header policy and several route/cache eligibility steps. Measure with - sampling `perf record -g` before extracting additional fast paths; preserve - query handling, canonical host, ACL, authentication, retries and WebSockets. -5. `src/upstream/*`: profile H1 pool misses, TCP connect rate, Hyper - connection reuse and generation swaps. Increasing recycle thresholds or - changing HTTP/2/H2C modes without an isolated performance case is not - recommended. -6. **Separate WWW priority:** benchmark short static objects independently; - existing `sendfile: auto` applies to eligible cleartext H1 objects >=64 KiB - and cannot accelerate 1 KiB bodies. - -No lock removal, tracing/log disabling, caching-policy weakening, bypass of -security checks, or transformation of complete bodies into buffered copies is -proposed without a test proving semantic equivalence. - -## Acceptance gate on a machine with Rust toolchain and isolated load generator - -```bash -./dev.sh doctor -./dev.sh check -CARGO_INCREMENTAL=0 ./dev.sh test -CARGO_INCREMENTAL=0 cargo build --release --locked -python3 scripts/performance/regression.py \ - --binary after="$(realpath target/release/rust-proxy)" \ - --out /tmp/rustproxy034-1kb-regression -python3 scripts/performance/lb_regression.py \ - --binary before=/absolute/path/to/old-rust-proxy \ - --binary after="$(realpath target/release/rust-proxy)" \ - --out /tmp/rustproxy034-1kb-lb -python3 scripts/performance/transport_regression.py \ - --binary before=/absolute/path/to/old-rust-proxy \ - --binary after="$(realpath target/release/rust-proxy)" \ - --out /tmp/rustproxy034-1kb-transport -python3 scripts/performance/benchmark.py \ - --binary before=/absolute/path/to/old-rust-proxy \ - --binary after="$(realpath target/release/rust-proxy)" \ - --nginx /usr/sbin/nginx --wrk /absolute/path/to/wrk \ - --out /tmp/rustproxy034-1kb-benchmark \ - --modes proxy --sizes small --concurrency 16,64,128 \ - --duration 30 --warmup 5 --repeats 7 -``` - -Use the **same explicit** `max_idle_per_host=64` and all other vhost settings -for the *source A/B* test. Run 4 vs 64 as a separate *configuration A/B* test; -never attribute a settings difference to code. Verify full content checksums, -HTTP 2xx/4xx/5xx, p50/p95/p99, drop counters, backend connections, -FD/RSS trends, CPU/request and packet/TCP retransmit rates. Test H1/H2/H3, -TLS, WebSockets, sticky/rate/ACL, cache MISS/HIT/BYPASS, admin sessions, active -failover, cancellation, idle timeouts and a sustained memory/connection soak. -Profile the freshly **built** executable separately from timed RPS runs. - -Until compilation and those gates pass, this is a **development source-only -candidate**, not a claim of reaching 100k RPS or 80% nginx. diff --git a/docs/PROXY_1KB_OPT3_DEV_0.34.0.md b/docs/PROXY_1KB_OPT3_DEV_0.34.0.md deleted file mode 100644 index 9c58672..0000000 --- a/docs/PROXY_1KB_OPT3_DEV_0.34.0.md +++ /dev/null @@ -1,88 +0,0 @@ -# RustProxy 0.34.0 DEV — 1,040-byte response, OPT3 - -## Scope - -Primary: uncached reverse proxy, HTTP/1.1 keep-alive, 1,040-byte body, -no compression, ACL/security/logging/analytics/counters fully operational. -Goal: ~100,000 RPS and >=80% of nginx on the **same isolated host**. -Secondary: local Web Server/static files, measured independently. -**Version remains exactly 0.34.0**; existing configuration and defaults are -unchanged; no functions, checks or logs are disabled. - -## New code optimizations (relative to OPT2-FIX1) - -1. `src/acl.rs` / `src/proxy/cache_helpers.rs`: borrow ordinary validated - routing paths using `Cow` during immutable route selection. Calls - with escaped (`%xx`) or repeated separators still use the original strict - allocator/decoder. Invalid control bytes, dot segments, UTF-8 and ambiguous - encodings remain rejected; the original wire path is not rewritten. -2. `src/proxy/forward_entry.rs` plus cache/Web Service call sites: retain the - original `http::Uri` value so normal paths can remain borrowed during the - entire mutable forwarding/rewrite operation. Cache-refresh tasks that need - ownership still allocate explicitly and hold their own `String`. -3. `src/forwarding.rs`: a **direct, untrusted** TCP peer now obtains the same - seven forwarding headers without allocating hop-chain `Vec` structures. - Trust chains and the configured `trust_forwarded_proto` logic use the - unchanged full parser. IPv4/IPv6 RFC 7239 formatting and true listener - port/scheme are preserved; untrusted spoofed values are still ignored. -4. `src/proxy/forward_entry.rs`: when effective client IP is identical to - the transport peer, reuse the IP stored by observation. No redundant - `IpAddr::to_string()` and no per-exchange mutex write. Different/trusted or - IPv4-mapped addresses are still stored in canonical form. -5. `src/proxy/headers.rs`: plain single-valued `Connection: keep-alive` - avoids the temporary `Vec` while all standard hop-by-hop fields - are removed; comma-separated, repeated and complex Connection fields keep - the original removal parser to prevent nominated-header leaks. -6. `src/proxy/request_id.rs`: preserve SHA-256, CSPRNG, per-process sequence, - unpredictable 64-char hexadecimal IDs and wire headers, but write the hex - digits into one preallocated byte buffer before UTF-8 verification. -7. `src/cache/stats.rs`: select the most frequent `cache_disabled` reason - counter directly. The same exact atomic totals, reason labels, 4096-domain - limit and reset behavior are retained. - -The changes avoid repeated work in common cases; no numerical throughput -benefit has been measured for this new source. In particular, no cache, log, -metrics, retry or auth feature was switched off for a synthetic benchmark. - -## Evidence and known limits - -- Provided newly compiled *OPT2-FIX1* executable SHA-256: - `6037e98d059672b716f2e1411976bf305cac927901a6f6662d8b703829ff8196`. -- PASS **89** HTTP proxy/policy/cache black-box assertions, zero failures, - testing that supplied executable, **not** the new OPT3 code. -- Source checks, migration/upgrade checks and ZIP checks are separate - static/offline verification; they are not equivalent to cargo/test/live. -- A single interrupted diagnostic A/B sample at c=64, 3 s, same idle=64, - gave older binary 33,752 RPS and OPT2-FIX1 binary 33,556 RPS, zero failed - requests; do **not** infer a true performance difference from one sample. -- NOT RUN here: `cargo check`, `cargo test`, rebuilt OPT3 integration tests, - post-build A/B RPS, long soaking, real TLS/H2/H3 compatibility and 80%-of-nginx - acceptance. No Rust toolchain is available in this editing container. -- Resource/latency trade-offs (including log/analytics drops, RSS/FDs) require - the same measured and pinned server/peer/generator CPU sets for before/after. - -## Rebuild and measure - -From the unpacked `RustProxy/` directory: - -```bash -./dev.sh check -CARGO_INCREMENTAL=0 ./dev.sh test -bash dev.sh -python3 scripts/performance/regression.py \ - --binary new=/ABSOLUTE/PATH/NEW/rust-proxy --out /tmp/rustproxy-opt3-http -python3 scripts/performance/ab_1kb_dev.py \ - --binary before=/ABSOLUTE/PATH/OPT2FIX1/rust-proxy \ - --binary after=/ABSOLUTE/PATH/OPT3/rust-proxy \ - --nginx /usr/sbin/nginx --ab /usr/bin/ab \ - --out /tmp/rustproxy-opt3-1kb --concurrency 16,64 \ - --duration 5 --repeats 3 --workers 4 --max-idle 64 -``` - -For an acceptance-grade evaluation, run the `scripts/performance/benchmark.py` -paired before/after/nginx test with separate pinned CPU sets, a verified origin -and 30-second runs repeated 7 times. Report median and p95/p99 latency, -errors, CPU, RSS, file descriptors, dropped log/analytics events, and all HTTP -status/body hashes. Retest TLS/H2/H3/WebSocket, auth, cache semantics and -connection/backpressure limits. Compare the **rebuilt** OPT3 binary against -OPT2-FIX1 at exactly the same listener/backend settings and compiler profile. diff --git a/docs/PROXY_1KB_OPT4_DEV_0.34.0.md b/docs/PROXY_1KB_OPT4_DEV_0.34.0.md deleted file mode 100644 index d40ce15..0000000 --- a/docs/PROXY_1KB_OPT4_DEV_0.34.0.md +++ /dev/null @@ -1,97 +0,0 @@ -# RustProxy 0.34.0 DEV / OPT4: HTTP/1.1 small-body follow-up - -## Objective and validity - -Primary goal: 1,040-byte reverse-proxy response, HTTP/1.1 keep-alive, -cache disabled; target about 100k RPS and at least 80% of nginx on the **same -isolated host**. Secondary goal: local static Web Server, benchmarked separately. -No version bump or feature, metric, logging, ACL, authentication, TLS, -HTTP/2/3, stream, retry or cache-policy bypass is introduced. - -The **provided and compiled OPT3 binary** (SHA-256 -`2efe273340f9cf92c5900c85c0bd1b775c87a9d1eaff1a10ec7635d268b080b1`) -passed 151 black-box HTTP assertions: 0 failures and 1 optional Zstandard -case skipped. This is evidence for the **input** binary, not for the new -OPT4 sources. OPT4 still requires a native Rust build, cargo tests and a -post-build benchmark. The diagnostic same-host CPU shares resources and is -not an acceptance benchmark. - -## Code changes relative to OPT3 - -1. `src/proxy/forward/route_cache.rs`: store normal public path/query as - `Cow<'_, str>` referencing the independently retained, immutable original - URI. Without a path rewrite both public/origin suffixes stay borrowed; - strict encoded/rewrite handling remains an owned validated string. - Cache refresh tasks explicitly call `.to_string()` only when they actually - must retain an owned target across spawned tasks. -2. `src/proxy/upstream_helpers.rs`: skip allocating a request-body metering - adapter **only** when `Body::is_end_stream()` establishes that it contains - no DATA frames or trailers. Every nonempty/streaming request retains the - original metering, timeout, backend lease accounting and cancellation. - The response adapter still retains the exchange/inflight permit and handles - completion/response bytes for empty requests. -3. `src/proxy/types.rs`, `src/proxy/forward/upstream.rs`: consume owned - `http::Uri` when preparing the upstream target. HTTP/1.1 returns it directly; - H2 takes the same URI into parts for strict authority rewriting. Remove an - unnecessary intermediate URI clone; replay still builds independent URIs. -4. `src/forwarding.rs`: reuse the formatted IPv4 peer address when creating - RFC 7239 `Forwarded` as well as X-Real-IP/X-Forwarded-For. Preserve quoted - IPv6 bracket and proxy-chain logic for trusted and untrusted peers. -5. `src/proxy/upstream_helpers.rs`: fast-path byte-safe truncation of short - or ASCII host/path log values, keeping the original max-*characters* - semantics for non-ASCII UTF-8. -6. `src/proxy/headers.rs`: avoid nominated-field token parsing for absent, - lone `keep-alive`, or lone `close` Connection values. A `close` field on - the wire still forces the general parser, so a nominated `Close` header - is removed exactly as before. Multi-value, comma lists and malformed tokens - retain the previous defensive parser. -7. `src/proxy/headers.rs`: similarly avoid the generic token parser when - checking HTTP upgrade on plain keep-alive, close, or absent Connection. - Complex multi-valued requests and explicit Upgrade remain unchanged. - -## Tests and limits - -- New Rust cases validate empty vs nonempty request-body permits and bytes, - H1 URI/authority/query, ASCII/Unicode truncation, and Connection variants - including a literal nominated `Close` field. Existing regression coverage - still includes sticky backends, cache, ACL and trusted forwarded identity. -- Source, SQLite migration and upgrade tests are necessary but **not** Rust - compilation or performance evidence. The compile check must run before use. -- Run `./dev.sh check && CARGO_INCREMENTAL=0 ./dev.sh test`, followed by - the native black-box acceptance suite (proxy, policy, cache, transform, - compression, static serving), plus TLS/H2/H3 and websocket tests. - -## Measurements - -An initial diagnostic same-host `ab` sample at concurrency 16 gave -OPT2 33,042.55 RPS and compiled OPT3 36,962.11 RPS, without HTTP errors; -this single sample is not proof of a reliable uplift. A separate complete -12-sample run (two 3-second repetitions each at 16/64 clients) produced: - -| Concurrency | OPT2 median RPS | OPT3 median RPS | nginx median RPS | OPT3/nginx | -| --- | ---: | ---: | ---: | ---: | -| 16 | 21,608 | 29,906 | 92,586 | 32.3% | -| 64 | 38,060 | 34,781 | 123,809 | 28.1% | - -All 12 samples had valid 200 responses and correct 1040-byte body checks. -OPT3 was slower than OPT2 at concurrency 64; the variability is too large to -claim a sustained uplift or regression based on two short same-host samples. -Raw evidence includes run order, response checksum and nginx comparator. -No result here measures OPT4, which was not built in this environment. - -## Next evaluation commands - -```bash -./dev.sh check -CARGO_INCREMENTAL=0 ./dev.sh test -bash dev.sh -python3 scripts/performance/regression.py --binary opt4=/absolute/path/target/debug/rust-proxy --out /tmp/opt4-http -python3 scripts/performance/ab_1kb_dev.py --binary opt3=/absolute/path/opt3/rust-proxy --binary opt4=/absolute/path/target/debug/rust-proxy --nginx /usr/sbin/nginx --ab /usr/bin/ab --out /tmp/opt4-ab --duration 5 --repeats 3 --concurrency 16,64 --workers 4 --max-idle 64 -``` - -For authoritative acceptance: pin client/server/origin to nonoverlapping CPU -sets, align build profile and all flags, compare 30s x7 runs, record median, -p95/p99, correct 1040 B response checksum and status, CPU, RSS, FDs, cache -stats, analytic/sink queue drops, errors, and both HTTP/1.1 and optional -HTTP/2 endpoints. Reject changes on new 4xx/5xx, body mismatches, regressions -of access controls or material memory/fd increases. diff --git a/docs/PROXY_1KB_OPT5_DEV_0.34.0.md b/docs/PROXY_1KB_OPT5_DEV_0.34.0.md deleted file mode 100644 index aaaf77f..0000000 --- a/docs/PROXY_1KB_OPT5_DEV_0.34.0.md +++ /dev/null @@ -1,93 +0,0 @@ -# RustProxy 0.34.0 DEV — OPT5, 1,040-byte reverse-proxy optimization - -## Acceptance boundary - -The source package preserves **0.34.0** and all existing configurable services, -logs, dashboards, cache/ACL/auth policies, streaming, TLS/H2/H3/WebSocket, -failover, retry and bounded-resource behavior. OPT5 is a **source-only** candidate: -no Rust compiler is available in the editing environment. The new code's -compilation, Rust unit tests, binary throughput and full-protocol interoperability -are unverified until the developer builds it. The compiled OPT4 executable is -the known test input, **not** a benchmark of OPT5. - -## Compiled OPT4 input verification - -Input: `/mnt/data/rust-proxy(4)`, SHA-256 -`1c4c701f11b57b1293020061c8af21d5a9df0c86f03290018a90edc007667b24`. - -- The real HTTP regression runner completed **151 passing**, **0 failing** cases; - **1 skipped** because Python `zstandard` is not installed. Protocol tests for - TLS/H2/H3 are a separate acceptance gate and were **not** passed by this run. -- Same-host ApacheBench HTTP/1.1 keep-alive, 1,040-byte body verified by exact - SHA-256, cache OFF, 4 Tokio workers, 64 idle HTTP1 upstream connections, - 3-second warm-and-measure samples, two repeats per variant/concurrency: - -| RPS, median of 2 samples | 16 clients | 64 clients | -| --- | ---: | ---: | -| OPT3 | 37,017 | 40,668 | -| OPT4 | 39,054 | 36,425 | -| nginx | 99,808 | 110,783 | -| OPT4 vs OPT3 | +5.5% | -10.4% | -| OPT4 / nginx | 39.1% | 32.9% | - -There were zero HTTP failures and the response body was verified before/after -measurements. Shared CPU and only two short samples are diagnostic, NOT a stable -speedup or production proof. The 100k RPS and 80%-of-nginx goals remain unmet. - -## OPT5 changes (8 coherent hot-path optimizations) - -1. `src/proxy/request_id.rs`: eliminate per-request SHA-256. Keep the existing - OS-seeded/reseeded thread-local CSPRNG and 256-bit ID; XOR a monotonic - sequence into the first 8 nonce bytes. Output remains uniformly random, - 64 lowercase hex characters and the protected `X-Request-ID` correlation - remains. Random IDs continue to be unique with overwhelming probability. -2. `src/forwarding.rs` and `src/proxy/forward_entry.rs`: only construct - `Context.original_url` for a frontend with configured Tinyauth. This is the - *only* runtime consumer of that absolute URL. Public `context(...)` still - constructs it by default. Trusted/untrusted forwarding and all validation - still execute identically; IPv4/IPv6 and `x-original-url` for Tinyauth remain. -3. `src/balance/lease_runtime.rs`: borrow `"ok"` for successful backend health - observations instead of allocating a `String`. 5xx failures still record the - exact original `status=NNN` detail and slow/passive health counters are kept. -4. `src/traffic.rs`: receive 64 samples at a time (Tokio `recv_many`) and apply - them under a single dashboard-state mutex. Order within the existing bounded - 2,048-sample queue is preserved; waiting is outside the mutex and producer - backpressure and drop accounting are unchanged. -5. `src/traffic.rs`: reuse map entries with borrowed vhost names. New keys are - cloned only when a domain first appears, preserving the max 64-vhost bound. -6. `src/traffic.rs`: classify status 1xx–5xx using static labels; preserve - unknown extension status labels such as `7xx`. -7. `src/traffic.rs`: replace per-sample lowercase allocation for User-Agent - family detection with ASCII-insensitive byte-window comparisons, and avoid - copying all TopTable maps when displaying a single vhost. -8. `src/proxy/upstream_helpers.rs`: skip resolving inherited/vhost/web-service - compression settings when no encoding is accepted even under the global - codec ceiling (for example `Accept-Encoding: identity`). Existing compression - handling is retained for every accepted codec and request method. - -The caller's front-end policies remain evaluated on every request. No log, -metric, access control or authentication feature was disabled or made optional. - -## Verification and outstanding gates - -Static source, SQL migration and managed-upgrade tests are executed and their -outcomes supplied in the external evidence archive. New Rust test fixtures cover -request IDs, forwarding/trusted IPv6, compression, healthy/5xx backend detail, -analytics batching, top classes, country-independent User-Agent matching, -reset and the bounded queue. - -To verify the new build on the developer's system: - -```bash -./dev.sh check -CARGO_INCREMENTAL=0 ./dev.sh test -./dev.sh build -``` - -After building, repeat the actual HTTP regression, LB regression and TLS/H2/H3 -transport tests and collect at least seven 30-second CPU-isolated, warmed, -interleaved OPT4/OPT5/nginx measurements with `wrk`, p95/p99 and CPU/RSS. -Treat the non-tested binaries, short shared-host tests and simulated Python -checks as distinct types of evidence. Measure the secondary Web Server path -separately. Do not alter the version or silently relax cache, auth or timeout -policies to reach a RPS target. diff --git a/docs/PROXY_HOTPATH_0.34.0.md b/docs/PROXY_HOTPATH_0.34.0.md deleted file mode 100644 index 165206c..0000000 --- a/docs/PROXY_HOTPATH_0.34.0.md +++ /dev/null @@ -1,84 +0,0 @@ -# RustProxy 0.34.0 — reverse-proxy hot-path candidate - -## What was changed - -1. **Routes:** `Snapshot.routes` and `fallbacks` now hold - `Arc` handles. The same compiled location is shared across - hostname aliases. `observe` resolves vhost/location once from the accepted - listener/TLS-state and stores a request-local handle. `forward_request` - consumes the same immutable snapshot/route; internal error fallbacks still - resolve the explicitly configured named location independently. -2. **Host matching:** wildcard candidate strings are generated lazily only - after an exact host failed, without changing longest-specific-suffix order. -3. **LB:** single-backend non-cookie pools use a direct admitted lease without - temporary candidate vectors or scheduler mutex. Multi-backend weighted - round-robin retains its ordering and mutex; least-connection/traffic, - adaptive, IP hash and cookie-affinity retain their behavior. IP hash avoids - repeated canonicalization/temporary `format!` strings for each candidate. -4. **Health:** a `healthy_cached` atomic allows checking healthy backends - without the health mutex; all state transitions remain inside the existing - state machine, including passive quarantine and half-open recovery. - Resolved active/passive per-member policies are held in shared immutable - `Arc` handles so a new request lease does not deep-clone health paths, - headers or optional host strings. -5. **Fallback/retry:** request body+HeaderMap replay templates are made only - for configured error fallbacks. Retry headers are cloned only for eligible - safe replay requests where another member or H2 graceful close makes retry - possible; writes, streaming bodies and WebSockets are not replayed. -6. **Telemetry:** three per-request mutex-backed scalar tags are now atomic; - their original event strings and `Option` log fields are decoded at - exchange completion. Request ID, method, host and path are moved into the - terminal log record instead of deep-cloned in Drop; the traffic sample - still receives its own required copies. Logs avoid a broadcast clone if - there are no subscribers, preserving history, sink and counters. - -## Preserved boundaries - -All vhost, ACL, GeoIP, Basic Auth, Tinyauth, WebSocket, retry, cache, -transform, compression, log, metrics, `sendfile`, PHP-FPM, HTTPS/H2/H3 and -admin panel configuration paths are intended to remain compatible. Defaults -and SQLite schema are unchanged; no new performance toggle was added. The -optimization reuses route objects within one immutable snapshot but does not -allow stale snapshot cache fills (existing revision ticket barrier remains). -Sticky cookies deliberately retain the general selection path, and upstream -request-count recycling is unchanged (default 25,000 per generation). - -## Locally executed checks vs required checks - -- **Executed locally:** scoped-cache/sendfile/hot-path *source-only* guards, - Python syntax, SQLite migration and upgrade preservation tests, selected JS - panel scripts and simulated editor tests. These do **not** prove Rust builds. -- **Prepared, NOT executed:** Rust unit tests for shared routing, single-member - LB capacity/health, passive ejection/recovery, atomic log fields, retained - sendfile, cache and protocol behavior. -- **Unmeasured:** 0.34.0 requests/sec, latency, CPU cost, memory and nginx gap. - Do not infer a throughput gain from fewer source-code locks/allocations. - -## Build and acceptance - -Run on the intended Linux builder with exactly the same compiler, dependencies, -release flags and workload as the 0.33.8 reference: - -```bash -./dev.sh doctor -./dev.sh check -CARGO_INCREMENTAL=0 ./dev.sh test -CARGO_INCREMENTAL=0 cargo build --release --locked -python3 scripts/tests/proxy_hotpath_034_static_test.py -python3 scripts/tests/cache_scope_034_static_test.py -python3 scripts/tests/sendfile_034_static_test.py -python3 scripts/tests/dashboard_presets_sql_test.py -python3 scripts/tests/upgrade_performance_test.py -``` - -Next run `scripts/performance/regression.py`, `lb_regression.py`, -`transport_regression.py` and `concurrency_regression.py` against both binary -versions. After functional acceptance, run the paired -`scripts/performance/benchmark.py` suite against the unmodified baseline, -this candidate and nginx; use the instructions in -`scripts/performance/README.md` including separate CPU affinities. - -Measure RPS/p95/p99, CPU microseconds per request, RSS, FDs, pool rotations, -health transitions, cache HIT ratio and logger drops. For each payload report -`(0.34.0 RPS / 0.33.8 RPS - 1)*100` and `0.34.0 RPS/nginx RPS*100` from real -medians. A source-only revision is **not production-accepted**. diff --git a/docs/PROXY_RPS_0.34.0.md b/docs/PROXY_RPS_0.34.0.md deleted file mode 100644 index 68d8a61..0000000 --- a/docs/PROXY_RPS_0.34.0.md +++ /dev/null @@ -1,78 +0,0 @@ -# RustProxy 0.34.0: hot-path throughput revision - -**Status: uncompiled source candidate.** This revision does not change the -application version, public API, dashboard controls, cache policy or tuning -defaults. It must not be labelled production-ready until the new Rust binary -passes compile, unit tests, live regressions and paired performance comparisons. - -## Code changes and intended cost reductions - -| Focus | Change | Kept behavior / risk to verify | -|---|---|---| -| Per-request identity | Draw a 256-bit nonce from the OS-seeded/reseeded thread-local CSPRNG instead of asking `getrandom` on every request. Preserve SHA-256 of nonce+monotonic sequence and the existing 64-hex ID format. | IDs remain non-predictable. Check ID uniqueness and raw upstream headers. | -| Upstream HTTP request target | Compile the validated scheme/authority `http::Uri` once with each immutable backend; parse only the request path+query and rebuild using typed `Uri::from_parts`. | Never accept a client-supplied authority, bypass DNS/address policies or change SNI. Test H1, H2, IPv6, query and percent-encoding. | -| Load balancing | Do not clone the member ID string for an existing smooth-WRR scoreboard entry. Continue the previously introduced no-mutex single-member non-cookie path. | Sticky-cookie modes, capacity limits, retry exclusions, health and drain remain unchanged. | -| Active health | Use already-compiled `Arc` when dispatching probes, avoiding a clone of its path/headers for each scheduled probe. | Same 250-ms supervisor, timeout, enabled/claim gate and health state transitions. | -| Per-frame streaming | Capture the leased backend once at first streamed request DATA (response already captured at wrapper construction). Do not take `Exchange.backend` mutex on every upload chunk. | Request/response byte counters, mid-stream failures, idle timeouts and lease lifetime remain active. | -| Pool recycling | Preserve exact request-count and elapsed-time recycling checks per request, with the existing independent 250-ms maintenance cycle. The pool factory/connection policy is unchanged. | Exact lifecycle semantics are more important than saving one cheap clock read; tune idle/age limits only after profiling. | -| Log history | Move completed `LogEntry` directly into bounded history when there are no live viewers or enabled external sinks, instead of deeply cloning its strings. Continue cloning and delivering to live viewers/sinks when enabled. | Same JSON fields, history size, drop counters, file/Loki/Logstash forwarding. | -| Route/HTTP instrumentation | Remove redundant route-ID mutex/write in forwarding and collapse H1/H2 version tallying into one branch. | Route is recorded once before forwarding; H1/H2 counters remain. | - -These historical changes build on the previous 0.34.0 route-Arc, single-backend LB, -conditional retry/fallback-header cloning, atomic exchange flags and cache HOT -LRU changes. There are no new UI toggles. This **earlier** revision retained `max_idle_per_host` -4 and `pool_recycle_requests` 25,000. The **subsequent DEV 1 KiB revision** -changes only the default idle limit to 64, based on live configuration A/B -with the old binary (not source A/B); see `docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md`. `sendfile: auto` remains opt-in and applies only to eligible Linux -plaintext H1 static WWW responses, not reverse proxy or TLS/H2/H3. - -## Cache transform/query policy - -An enabled `bypass_query` is a deliberate policy: `/theme.css?v=...` is a -`BYPASS` even when the origin sends `Cache-Control: public`. Turning the bypass -off permits a transformed CSS object to follow `MISS -> HIT` and appear in the -cache dashboard. An existing transformed HOT row with hit count and TTL is -positive evidence of this. This throughput revision does not modify the -cache eligibility logic or discard Age/ETag/Cache-Control headers. - -## Verification already possible here - -- **Old/reproducible reference executable only:** 110 passed, 0 failed, 1 skipped - (`compression.codec.zstd` skipped because the optional decoder was missing), - from live proxy/policy/cache/transform loopback regression. This does **not** - exercise the newly modified Rust source. -- **Old/reproducible reference executable only:** query bypass enabled gives - `BYPASS -> BYPASS`, disabled gives `MISS -> HIT` and a transformed representation; - the older Age regression is also reproducible on this old executable. -- Source-only invariants: `scripts/tests/rps_hotpath_034_static_test.py` plus - existing route/cache/sendfile checks. SQL migration and upgrade scripts pass. -- **Not performed:** Rust compilation or newly added Rust unit tests, live - behavior of the optimized binary, HTTP/2 live TLS, comparative RPS, CPU - profiling and long-duration memory/health soak. The environment does not - include `cargo`, `rustc` or `wrk`. No post-change RPS gain is claimed. - -## Acceptance commands on a Rust-capable Linux host - -```bash -./dev.sh check -CARGO_INCREMENTAL=0 ./dev.sh test -CARGO_INCREMENTAL=0 cargo build --release --locked -python3 scripts/tests/rps_hotpath_034_static_test.py -python3 scripts/tests/cache_age_transform_034_live.py \ - --binary "$(realpath target/release/rust-proxy)" \ - --output /tmp/rustproxy034-cache-after -python3 scripts/performance/regression.py \ - --binary optimized="$(realpath target/release/rust-proxy)" \ - --out /tmp/rustproxy034-http-regression -``` - -Run `scripts/performance/benchmark.py` from the documented reproducible -before/after/nginx command in `scripts/performance/README.md`. Compare the -unchanged old 0.34.0 binary with the newly compiled source, not a baseline -with different feature flags. Record RPS at 1,040 B, 64 KiB, 1 MiB, p95/p99, -CPU/request, RSS, connection counts, errors and exact response-body hashes. -Use `perf record -g -p ` during a stable request interval and inspect -`perf report --stdio` to identify the next true bottleneck. - -Do not interpret Python origin self-tests, source inspections, or speculative -Amdahl models as measured RustProxy throughput. diff --git a/docs/README.md b/docs/README.md index 42bfdea..54821ee 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,7 +7,6 @@ This is the canonical index for the current project documentation. The authentic - [Project README](../README.md) - product overview, installation entry points and current release notes. - [ROADMAP](../ROADMAP.md) - planned and completed release scope. - [CHANGELOG](../CHANGELOG.md) - user/operator-visible changes by release. -- [TODO](TODO.md) - open work only; completed work is not retained as TODO. ## Functional documentation @@ -42,7 +41,6 @@ This is the canonical index for the current project documentation. The authentic - [MOCK_SERVER](MOCK_SERVER.md) - local Python mock origin used by development/integration checks. - [MOCK_AUTH](MOCK_AUTH.md) - local fake Tinyauth peer for development only. - [MEMORY_STRESS](MEMORY_STRESS.md) - sustained-memory audit, offline load simulations, real RustProxy staging stress and RSS measurement procedure. -- [TEST_REPORT_0.34.0](TEST_REPORT_0.34.0.md) - 0.34.0 source-stage test evidence, simulated cache retention and required acceptance gates. ## Technical documentation @@ -61,7 +59,7 @@ Completed roadmap history is retained separately as [ROADMAP_HISTORY](ROADMAP_HI ### Retained audits and historical context -- [SECURITY_AUDIT](SECURITY_AUDIT.md) and [SECURITY_AUDIT_0.31.0](SECURITY_AUDIT_0.31.0.md) - current security audit material and the 0.31.0 audit. +- [SECURITY_AUDIT](SECURITY_AUDIT.md) - security audit material. - [CHANGELOG_TECHNICAL_ARCHIVE](CHANGELOG_TECHNICAL_ARCHIVE.md) and [ROADMAP_HISTORY](ROADMAP_HISTORY.md) - retained long-form technical history and completed milestones. - [ACME_RFC2136](ACME_RFC2136.md) - historical DNS provider details. - [INTEGRATIONS_040](INTEGRATIONS_040.md) and [MANAGEMENT](MANAGEMENT.md) - retained design and integration notes. @@ -72,7 +70,7 @@ The old per-version implementation notes and audits removed from this trimmed so ## Maintenance rules - Put user/operator-visible release changes in `CHANGELOG.md`. -- Put future release scope in `ROADMAP.md` and unresolved work in `docs/TODO.md`. +- Put future release scope and unresolved work in `ROADMAP.md`. - Keep detailed implementation evidence in technical documents or version audits, not in CHANGELOG. - When functionality changes, update the functional document, this index, Reader coverage and cross-links in the same change. - Remove obsolete or duplicated documents when they no longer provide historical or technical value; retain useful audits with an explicit historical label. diff --git a/docs/RELEASE_0.34.0_CACHE_FIX_PL.md b/docs/RELEASE_0.34.0_CACHE_FIX_PL.md deleted file mode 100644 index 3f3c1e4..0000000 --- a/docs/RELEASE_0.34.0_CACHE_FIX_PL.md +++ /dev/null @@ -1,93 +0,0 @@ -# RustProxy 0.34.0 — poprawki cache, transformacji i kontrola RPS - -Status: **zmodyfikowane źródła, niekompilowane lokalnie** (brak `cargo` i `rustc`). Numer wersji: 0.34.0. - -## Co odtworzono na binarce użytkownika - -Uruchomiono binarkę Linux ELF w izolowanej konfiguracji z lokalnym origin HTTP, z włączonym cache, transformacją CSS, `max_ttl_seconds=86400`, diagnostyką i API administracyjnym. HTTP/1.1 localhost, pełne GET, brak kontaktu z witryną produkcyjną. - -| Odpowiedź | Pierwszy GET | Drugi GET | Liczba pobrań z origin | -|---|---|---|---| -| `/normal.css` | MISS | HIT | 1 | -| `/aged.css` | MISS | MISS | 2 | -| `/br.css` | MISS | HIT | 1 | -| `/aged-br.css` | MISS | MISS | 2 | -| `/private.css` | MISS | MISS | 2 | -| `/no-store.css` | MISS | MISS | 2 | - -Zwykły CSS i skompresowany Brotli dostają HIT. Warianty z `Age: 1553483` mają `MISS → MISS` mimo `Cache-Control: public, max-age=2592000`. Przypadki `private` / `no-store` słusznie nie trafiają do cache. Zmiana keep-alive vhosta nie usuwała poprawnie zachowanego obiektu. `expose_only_hits=true` celowo ukrywa nagłówki na MISS. - -## Główna przyczyna i poprawka - -W `src/cache/policy/response.rs` obowiązywał wzór `ttl = min(origin_max_age, local_max_ttl)` oraz `remaining = ttl - corrected_age`. To jest sprzeczne z zadaniem lokalnego limitu: świeżość w origin nadal trwa, ale CDN `Age > local_max_ttl` dawał sztuczne 0 sekund lokalnej świeżości. - -Poprawiony wzór: - -```text -corrected_initial_age = max(apparent_age, upstream_age + response_delay) -remaining_origin_freshness = max(0, origin_max_age - corrected_initial_age) -remaining_local_freshness = min(remaining_origin_freshness, local_max_ttl) -entry_fresh_until_age = corrected_initial_age + remaining_local_freshness -``` - -Przykład z przesłanych nagłówków: wiek upstream 1 553 483 s = prawie 18 dni. Pierwotna świeżość 2 592 000 s (30 dni), więc pozostało około 12 dni w origin. Po naprawie RustProxy będzie mógł przechować świeżą odpowiedź przez lokalny limit (domyślnie 86 400 s = 1 dzień). Nagłówek Age nadal będzie około 1,55 mln sekund i będzie rósł na HIT — to poprawne. - -`Age` nie wynika z `Last-Modified: 2016`. Najczęściej jest przekazany przez upstream/CDN, a stara data `Date` też mogłaby podnieść apparent_age. Rozróżnienie: `Age` = HTTP wiek całkowity; `resident_age_seconds` = pobyt w HOT/COLD. - -## HOT RPS i panel - -1. HOT HIT nie wyjmuje już i nie wkłada od nowa wpisu oraz indeksu `Vary`, tylko aktualizuje licznik HIT i kolejność LRU w miejscu. Zmiana ma zmniejszyć alokacje i krytyczną sekcję; **zysk RPS jeszcze niezmierzony**. -2. Panel Cache ma osobne kolumny **Age** i **Czas w RustProxy**. API `dashboard.objects[]` dodaje `resident_age_seconds` także dla COLD. Pozostałe nazwy pól nie zostały zmienione. -3. Licznik `x-cache-hits` dotyczy konkretnego aktualnego wpisu. Gdy obiekt zostanie wyparte z HOT (limit wpisów/pamięć), usunięty lub zastąpiony po MISS, licznik restartuje. W próbce użytkownika TTL wynosił około 604 tys. s (7 dni), więc późniejszy MISS po ~4 minutach **nie wynikał z normalnego wygaśnięcia TTL**. Konkretnej przyczyny epizodu nie można ustalić bez trendów `evictions`, `purges`, `hot_demotions`, `cold_hits`, kluczy/Vary oraz zmian konfiguracji. -4. Brak `x-cache` na MISS jest oczekiwany, gdy włączono **nagłówki diagnostyczne tylko na HIT**. Brak wpisu w API może mieć inne przyczyny (np. cache-control `private`, `no-store`, brak polityki origin, limit obiektu) i nie należy ich obchodzić na siłę. -5. Nie zmieniano uprawnień, `private`/`no-store`, kluczy Vary, TTL po stronie origin ani zasad selektywnej invalidacji. - -## Wynik symulacji TTL - -Wykonano **100 000 deterministycznych scenariuszy**. Dla każdego sprawdzono: po naprawie lokalna świeżość nie przekracza `local_max_ttl` ani świeżości pozostałej w origin. - -| Scenariusz | Stary pozostały TTL | Po poprawce (symulacja) | -|---|---:|---:| -| Age = 1 553 483, local cap 1 dzień | 0 s | 86 400 s | -| Age = 1 553 483, local cap 7 dni | 0 s | 604 800 s | -| Origin kończy świeżość za 10 s | 0 s | 10 s | -| Origin jest już nieświeży | 0 s | 0 s | - -W symulacji 33 387 / 100 000 losowych przypadków stara polityka błędnie dawała zero zamiast dodatniego TTL. **To nie jest procent regresji w realnym ruchu**, a wynik losowania parametrów testowych. - -## Nginx — rzeczywiste pomiary starej binarki - -Archiwalne wyniki samej binarki `rust-proxy 0.34.0` na tym środowisku (Linux, 4 vCPU, HTTP/1.1 loopback, ApacheBench, 64 połączenia, cache OFF, 3 powtórzenia), z nginx 1.26.3. Brak miarodajnego benchmarku **nowych źródeł** przed ponowną kompilacją. - -| Endpoint | RustProxy RPS (mediana) | nginx RPS (mediana) | RustProxy / nginx | -|---|---:|---:|---:| -| proxy 1 KiB | 10,544 | 58,709 | 18.0% | -| proxy 64 KiB | 7,893 | 17,358 | 45.5% | -| proxy 1 MiB | 1,010 | 1,666 | 60.6% | -| www 1 KiB | 11,984 | 71,318 | 16.8% | -| www 64 KiB | 9,224 | 26,600 | 34.7% | -| www 1 MiB | 1,264 | 1,098 | 115.2% | - -Ograniczenia porównania: ApacheBench może limitować klienta; wyniki nie reprezentują TLS/H2/H3 ani ruchu produkcyjnego. **Nie wolno tych RPS przypisywać nowej rewizji.** - -## Zestaw testów dla nowej kompilacji - -```bash -CARGO_INCREMENTAL=0 cargo test --locked --lib -CARGO_INCREMENTAL=0 cargo build --release --locked -python3 scripts/tests/cache_age_policy_034_model.py -python3 scripts/tests/cache_age_transform_034_live.py --binary target/release/rust-proxy --output /tmp/cache-test-034 -python3 scripts/tests/cache_scope_034_static_test.py -python3 scripts/tests/proxy_hotpath_034_static_test.py -python3 scripts/tests/sendfile_034_static_test.py -``` - -Do testów jednostkowych dodano Age, transformowane CSS/HOT, democję do COLD i odczyt COLD po restarcie, logikę LRU/HIT oraz kontrolę zakazów `private`/`no-store`. Test lokalny sprawdza CSS/Brotli, origin count, API, keep-alive config i ukryte nagłówki. Test po kompilacji bez przełącznika `--expect-old-age-bug` ma wykazać `MISS → HIT` dla Age/Brotli. Na starej binarce odtwarzanie regresji przechodzi z `--expect-old-age-bug`. - -Kontrole wykonane na źródłach: źródłowe testy cache scope, proxy hotpath, sendfile, składnia JS, parsowanie tłumaczeń. **Nie uruchomiono testów Rust ani benchmarków poprawionej binarki** — środowisko nie ma `cargo/rustc`. - -## Następny pomiar po kompilacji - -Porównać tę rewizję 0.34.0 z poprzednią 0.34.0 oraz nginx dla 1 KiB / 64 KiB / 1 MiB; osobno cache OFF, HOT HIT i transformacja/Brotli. Raportować RPS, p95/p99, procesor, RSS, nowe TCP, `cache.evictions` i `cache.purges`; nie modyfikować ustawień cache podczas benchmarku. - -Uwaga: `curl -I -X GET` miesza zachowanie HEAD i GET; do testów reprezentacji zalecane `curl -sS -D - -o /dev/null URL`. diff --git a/docs/ROADMAP_HISTORY.md b/docs/ROADMAP_HISTORY.md index 6ad8aa7..729dcd6 100644 --- a/docs/ROADMAP_HISTORY.md +++ b/docs/ROADMAP_HISTORY.md @@ -1,6 +1,6 @@ # Historical roadmap archive -> Archived during the RustProxy 0.25 documentation cleanup. This file preserves completed historical roadmap entries. The active roadmap is [`../ROADMAP.md`](../ROADMAP.md); unresolved work is tracked in [`TODO.md`](TODO.md). +> Archived during the RustProxy 0.25 documentation cleanup. This file preserves completed historical roadmap entries. The active roadmap is [`../ROADMAP.md`](../ROADMAP.md). # RustProxy roadmap diff --git a/docs/SECURITY_AUDIT_0.31.0.md b/docs/SECURITY_AUDIT_0.31.0.md deleted file mode 100644 index 1a026ca..0000000 --- a/docs/SECURITY_AUDIT_0.31.0.md +++ /dev/null @@ -1,148 +0,0 @@ -# RustProxy 0.31.0 - API, panel and HTTP cache-maintenance security review - -Scope: source snapshot supplied for the v0.31.0 maintenance follow-up, reviewed -2026-10-10. This is a **code/contract review with offline regressions**, not a -penetration test or proof of production safety. The supplied archive has no -`.git` directory; the review does not establish parity with remote `master`. - -## Interfaces and trust boundaries - -| Surface | Exposure | Authentication and mutation rules | Assessment | -|---|---|---|---| -| Public administration bootstrap (`/`, login JS/CSS, `/api/auth/status`, setup/login POST) | Login/public administration listener | Host/origin validation; request marker for unsafe bootstrap actions; one-time setup capability and login throttling | Source-reviewed; live browser flow pending | -| Private panel HTML, JS, documentation, OpenAPI | Administration listener | Administrator session; `Cache-Control: no-store`; CSP, frame and MIME protections | Source and anonymous-response test reviewed; runtime pending | -| `/api/status`, `/api/system-info`, `/api/traffic`, `/api/cache`, logs, config, preferences, audit | Administration listener | Administrator cookie session; sensitive reads not on anonymous allowlist | Source and regression tests added; runtime pending | -| `/api/events`, live-log SSE | Administration listener | Session on subscribe and repeated validity checks; bounded viewers; heartbeat/disconnect cleanup | Source-reviewed; reconnect/load pending | -| State-changing admin APIs (including cache/config/preferences) | Administration listener | Valid session, Origin/Host, `X-RustProxy-Request: 1`, CSRF token; audit on cache commands | Source and regression tests added; runtime pending | -| Certificate API (`/api/certificates/{id}/reload`, deploy) | Administration listener | Narrow-scoped Key ID + Bearer grant; no general read access; HTTPS required remotely; origin validation when supplied | Source-reviewed; token rotation/revocation live tests pending | -| Public `BAN`/`PURGE` | Configured public vhost listener | **No admin session**: separate explicit IP/CIDR allowlist (empty by default), matched listener/vhost/ACL and optional rate limit | Hardened below; reverse/HTTP tests pending | -| Separate Prometheus metrics exporter | Distinct optional listener | Configured IP CIDRs and bearer-token policy; not the admin session | Unchanged; deployment exposure audit pending | - -## Findings addressed in this follow-up - -1. **E0716 / compilation defect (high functional impact).** `error()` accepts - `&'static str`; a reference to `format!(...)` was a temporary. Invalid - selector errors now log the detailed reason server-side and return a fixed, - non-sensitive message with status 422. This removes the reported lifetime - violation; the Rust compiler must still verify the entire crate. -2. **Cache protocol-scope confusion (medium).** Path-only `BAN`/`PURGE` - selectors could affect representations of the same host/path across both - HTTP and HTTPS. Targets are now built with the trusted external scheme, - validated to `http`/`https`, and the matched Host/vhost. IPv6 literal Hosts - are bracketed for the URL parser. Other tenants, schemes and hosts remain - outside the selector. -3. **Unbounded blocking maintenance work (medium availability risk).** Cache - invalidation may scan hot/cold entries. Both public HTTP and authenticated - admin cache purge/selective invalidation share two maintenance slots and - execute scans with `spawn_blocking`. Bursts receive 429 instead of piling - up unbounded blocking work on Tokio core threads. The permit stays with the - worker after a client disconnects. This is a concurrency bound, **not** a - per-client rate quota or a hard time limit for disk work. -4. **Ambiguous maintenance input (medium).** Duplicate TTL/confirmation or - `Content-Length` headers, transfer-encoded bodies, nonempty bodies and - upgrade requests are rejected; `Content-Length: 0` remains compatible with - CLI clients. TTL is limited to 1..604800 s. Broad `BAN /` requires the - explicit confirmation header; `PURGE` cannot use BAN-only controls. -5. **Audit ordering (low/medium).** Removal of an administrator-created BAN - rule now records the audit entry before mutating state. Unavailable audit - storage blocks the mutation rather than allowing an unaudited removal. -6. **Operator safety/usability (low).** An EN/PL help popover beside the HTTP - allowlist gives exact `curl` syntax, IP-only authentication semantics and - reverse-proxy identity requirements. No new global config variable is added. - -## Existing protections checked in source - -- Admin router's exact anonymous method/path allowlist contains only the - expected authentication bootstrap APIs; private docs and assets remain - session-protected. Request guard rejects duplicate Host/Origin and checks - browser `Sec-Fetch-Site` cross-site signals. -- Mutation guard uses an Origin/Host match, per-session CSRF comparison and - `X-RustProxy-Request` marker; cookie sessions are HttpOnly/SameSite=Strict - and use Secure under HTTPS/trusted forwarded protocol. -- Secure response policy includes CSP, `X-Frame-Options: DENY`, - `X-Content-Type-Options: nosniff`, referrer restrictions and no-store for - private material. A static scan checked 27 first-party browser JS files for - direct unsafe HTML injection and dynamic `eval`/`Function` sinks; this scan - **cannot establish that the entire DOM is XSS-free**. -- IP trust comes from the transport peer or a validated forwarding chain only - when the peer is configured trusted. An untrusted client-supplied XFF does - not independently grant BAN/PURGE permission. The cache allowlist rejects - `/0`, invalid CIDRs, duplicates and more than 32 entries. Requests are never - forwarded upstream. -- Certificate API tokens have explicit operation/ID grants and cannot use the - general admin read API; Bearer grant operations are intentionally separate - from cookie + CSRF authentication. - -## HTTP maintenance CLI (allowlisted client only) - -```sh -# One URL, including the query string: 200 with JSON result. -curl -i -X PURGE 'https://site.example/assets/app.js?v=3' - -# Lazily ban previously cached objects under this path: 202 with JSON result. -curl -i -X BAN 'https://site.example/assets/' - -# Entire host: explicit opt-in confirmation header. -curl -i -X BAN 'https://site.example/' -H 'X-RustProxy-Confirm-Wide: 1' - -# Optional BAN expiration (30 seconds). -curl -i -X BAN 'https://site.example/assets/' -H 'X-RustProxy-Ban-Ttl: 30' -``` - -Use **Settings -> Cache -> HTTP BAN / PURGE IP allowlist** to grant the -caller IP/CIDR. This HTTP interface does not accept admin cookies/tokens as an -alternative to the allowlist. The domain, Host header, external scheme and -listener must select the intended vhost. Behind a reverse proxy, forward the -non-standard methods untouched and **replace**, rather than append untrusted, -XFF/Forwarded headers at the edge. Never add the entire reverse-proxy subnet -as a maintenance client merely because it is trusted for forwarding. - -## Release security test plan (requires compiled RustProxy) - -1. On the deployment host, run `cargo fmt --all -- --check`, - `./dev.sh check`, `./dev.sh test`, and `./dev.sh release`. Address any - compiler/regression errors before installation. `cargo audit` or an - equivalent advisory scan requires an up-to-date advisory DB and was not run - in this offline environment. -2. Start a disposable instance with a populated cache under both HTTP and - HTTPS for the same vhost/URL. Confirm `PURGE` returns 200 and drops the - exact query variant only; `BAN` returns 202 and invalidates only **older** - entries on the URL prefix. Confirm the *other* scheme and vhost stay hot. -3. From a denied IP, expect 403 on `BAN` and `PURGE`. With an empty allowlist, - all clients must be denied. Send duplicate TTL/confirmation/Content-Length, - TE/body and `BAN /` without confirmation: expect 400. On a trusted-proxy - hop with missing real client identity, expect 403. Test both IPv4 and IPv6. -4. From an allowed IP, send 10 concurrent broad maintenance requests; confirm - that only two workers run simultaneously and excess return 429. Verify - server CPU/IO and audit entries while the cold tier is populated. -5. Send GET requests without admin cookies to `/api/config`, `/api/status`, - `/api/system-info`, `/api/events`, `/api/logs`, `/docs/openapi.json` and - `/assets/app.js`; expect 401 and protective headers. With valid cookie but - missing/bad Origin, marker or CSRF on PUT/POST/DELETE, expect 403. Test - logout and password/session invalidation, live SSE reconnect/expiry. -6. Test the Certificate API with no grant, a valid certificate-specific grant, - revoked token, HTTP remote peer, and misuse against `/api/config`/logs. - Check errors for token disclosure and audit coverage. -7. Test actual browser startup, CSP console, output escaping and form save - with malicious HTML strings; inspect reverse-proxy Host/Origin forwarding. - Prefer HTTPS administration with a strict `allowed_hosts` list. - -## Residual risk / non-claims - -- **Not executed in this environment:** Cargo format/check/tests/release, - actual Rust listener requests, HTTP/2/3, fuzzing, penetration tests, - session revocation under load, proxy deployment configuration, dependency - CVE scanning and long-running cold-cache resource measurements. -- Proxy trust configuration is security-critical. A misconfigured edge that - accepts forged forwarded client addresses can let external users impersonate - an allowlisted IP. Restrict direct listener reachability and sanitize headers. -- A public IP allowlist is not user authentication; anyone using an allowed - IP/network can issue destructive maintenance commands. Use narrow CIDRs and - optionally enforce additional controls on the edge (mTLS, firewall). -- The two-worker concurrency bound mitigates bursts, not all denial-of-service - scenarios: configured cache size and storage latency still determine scan - cost. Per-IP quotas are not provided here. -- Proxy protocol path, certificate verification, arbitrary upstream code, - all application content and third-party packages are **not certified safe** - by this audit. Production acceptance remains open until the runtime test - plan is completed. diff --git a/docs/SETTINGS.md b/docs/SETTINGS.md index 2a91c86..c9c18d0 100644 --- a/docs/SETTINGS.md +++ b/docs/SETTINGS.md @@ -350,7 +350,7 @@ The adjacent **info (`i`) popover** in Settings → Cache shows ready-to-copy `curl -X PURGE` and `curl -X BAN` examples in English and Polish, including the explicit `BAN /` confirmation. These public methods authorize by the source IP/CIDR only; they do not accept a browser admin session as a bypass. -See [0.31.0 security audit](SECURITY_AUDIT_0.31.0.md) for reverse-proxy trust and tests. +See [security audit](SECURITY_AUDIT.md) for reverse-proxy trust and tests. ## Static-response transformation diff --git a/docs/TEST_REPORT_0.34.0.md b/docs/TEST_REPORT_0.34.0.md deleted file mode 100644 index 541745a..0000000 --- a/docs/TEST_REPORT_0.34.0.md +++ /dev/null @@ -1,236 +0,0 @@ -# RustProxy 0.34.0 verification report - -## Acceptance status - -**Source candidate, not verified production release.** Rust (`cargo`, `rustc`) is -not installed in this execution environment, and the runtime has no access to -its dependency registry. Consequently the modified Rust source was **not -compiled**, the new Rust tests were **not executed**, and no 0.34.0 RustProxy -binary or live throughput figures are supplied. Source inspection and simulated -UI/cache tests do not close the runtime acceptance gate. - -## DEV 1 KiB counters follow-up (source remains uncompiled) - -The newly supplied binary `rust-proxy(1)` identifies as 0.34.0 and has -SHA-256 `1a84c58b20945bba9fdd1bc1a44d9e9d6ac7659d714343a46a1a8511a78331fa`. -It completed **151 passing real HTTP tests, 0 failures, 1 optional zstd -skip**. These tests were run on the supplied binary, not on the subsequently -modified source. The latest source moves per-domain cache statistics outside -HOT store mutexes and lazily allocates analytics samples only after a queue -permit is available. Histogram completion now increments one exclusive -latency bin instead of up to 11 cumulative bins; both Prometheus exporters -convert them to the same cumulative format at scrape time. Static -source/SQL/upgrade checks pass; **Rust compile, -Rust unit tests and live new-source RPS remain OPEN**. See -`docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md` and the separate evidence archive. - -## DEV 1 KiB reverse-proxy follow-up (October 2026; source only) - -The supplied **old executable** passed loopback HTTP/1.1 1,040-byte content -checks and ApacheBench 12,000-request samples. The configuration trial -`max_idle_per_host=4 -> 64` increased short-run RPS, but shared-CPU noise -precludes a stable nginx-parity percentage. The edited sources now use 64 by -default for absent/new keepalive fields and reduce exclusive-Drop mutex/clone -and absent-cookie overhead. See `docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md`. -**No newly modified binary was built or run.** Rust unit tests, runtime -regressions, long soaks and post-edit code A/B are still required. - -Current DEV source checks performed on the modified tree: - -| Check | Result | Scope | -|---|---|---| -| `./dev.sh doctor` | PASS | Source preflight; reports `cargo` and `rustc` absent | -| `scripts/tests/rps_hotpath_034_static_test.py` | PASS | Guardrails, defaults, retained routing/security and source-size limits only | -| `scripts/tests/proxy_hotpath_034_static_test.py` | PASS | Routing and telemetry source guards only | -| `scripts/tests/cache_scope_034_static_test.py`, `cache_hot_034_static_test.py`, `sendfile_034_static_test.py` | PASS | Cache and WWW source guards only | -| `scripts/tests/dashboard_presets_sql_test.py` | PASS | SQLite migrations and preset constraints | -| `scripts/tests/upgrade_performance_test.py` | PASS | Disposable upgrade/preservation/locking test | -| `scripts/tests/sendfile_034_panel_browser_test.py` | PASS, 2 widths | Simulated Chromium admin API, not live Rust | -| `scripts/tests/vhost_backend_popup_0337_browser_test.py` | PASS, 4 widths | Simulated Chromium admin API, not live Rust | -| Node `--check` (vhost editor), Python `compileall`, JSON parser (OpenAPI) | PASS | Syntax only | -| `./dev.sh check`, `./dev.sh test` | BLOCKED, exit 127 | Rust toolchain missing; no Rust unit/build acceptance | -| Old supplied binary, `ab`/nginx configuration A/B | COMPLETED, DIAGNOSTIC | 1,040-byte H1 payload; shared-CPU and no new code exercised | - - -## Later 0.34.0 reverse-proxy optimization revision - -The refreshed source archive additionally reuses immutable route handles, -limits lazy host wildcard allocation, fast-paths one-backend non-cookie LB, -atomically caches healthy state and exchange scalar log fields, shares -compiled per-member health policies across leases, and avoids -unneeded fallback/retry header clones and empty-subscriber broadcast clones. -The existing cache/sendfile work and all panel controls remain. Added Rust -regression cases and the source-only hot-path guard are **not a substitute** -for compilation and live HTTP, health, capacity, sticky, and H2 retry tests. -No RPS improvement, corrected nginx gap, or full functional parity has been -measured for this new revision. Details: `docs/PROXY_HOTPATH_0.34.0.md`. - -## RPS-focused follow-up source revision (not yet compiled) - -See `docs/PROXY_RPS_0.34.0.md` for detailed changes in request IDs, validated -upstream URI construction, LB scheduling, probes, streaming byte accounting, -unchanged pool generation lifetimes and log-history ownership. Source-only invariants pass. -A live test of the PREVIOUS executable (not this source revision) completed -110 passing cases, 0 failing cases and 1 skipped optional zstd decoder in the -proxy/policy/cache/transform harness. The old executable also confirms query- -string bypass is deliberate and reproduces the older Age/TTL bug. The new -binary has NOT been built or tested and its RPS gain is unknown. - -## HOT HIT follow-up source revision (not yet compiled) - -See `docs/CACHE_HOTPATH_0.34.0.md`. Response header cloning has moved outside -the global HOT lock, and redundant bounded stats key allocation, empty BAN -checks and full HOT TTL sweeps were removed. A new Rust concurrency/regression -suite and an ignored microbenchmark have been added. Existing COLD and Vary -behavior, exact LRU and cache semantics remain. This source has not been built -or run under Rust in this environment; no RPS gain is claimed. - -## Changes and correctness boundaries - -- Operational per-vhost saves (upstream keepalive/recycling, connection limits, - load-balancing/health policy, timeouts, request gate and diagnostics) retain - HOT and COLD entries. Backend origin, routing, content, cache and transform - changes retain conservative invalidation. Vhost-specific changes purge only - that vhost; global cache, transform, forwarding, headers and Web Service or - PHP-FPM profile changes may conservatively purge all tiers. -- Scope-specific cold-cache records persist an origin/content compatibility - fingerprint. COLD startup rejects stale scope records and deletes their cache - artifacts. Legacy COLD metadata defaults its fingerprint to zero; production - vhosts have a nonzero SHA-derived fingerprint. A one-time cold refill can - occur on the 0.33.8 -> 0.34.0 upgrade because the global identity changed. -- A separate administrative configuration revision acts as a cache-ticket - publication barrier: a request bound to a pre-save routing snapshot cannot - create a new cache ticket after the save. The data revision stays stable for - operational-only saves, so previously cached objects remain available. -- Reverse-proxy response-body byte metering captures the backend lease once at - wrapper creation instead of acquiring a backend-selection mutex per DATA - frame. Request-body accounting, error paths and exchange ownership remain. -- An **opt-in `sendfile: auto`** setting is wired through Web Service serde, - compiled config, the Delivery UI and documented API. The Linux HTTP/1.1 - TCP transport adapter calls `libc::sendfile` for eligible static files; - cache-enabled, ranged, compressed, transformed and TLS/H2/H3 paths retain - existing streaming. It is **disabled by default** and must pass native - compilation, correctness, transport and load tests before use. No persisted - SQLite schema migration or LB UI change is required. - -## Local verification - -| Check | Outcome | Coverage / caveat | -|---|---|---| -| `./dev.sh doctor` | PASS | Project source layout and tool inventory | -| SQL migration/preset test | PASS | All ten migrations re-applied, preset access and limits | -| Upgrade test | PASS | Stop/lock guards, SQLite/.env/language preservation, symlink rejection | -| Python compileall | PASS | Python syntax only | -| Vhost LB/Cache Chromium test | PASS | Simulated browser quick-view, 320/375/768/1440px | -| Other Chromium scripts | 15 PASS | Simulated API/CSS/JS, not a live Rust server | -| Sidebar navigation Chromium test | INCOMPLETE | Execution time budget; not counted as a pass | -| Three Node dashboard scripts | PASS | Simulated UI logic | -| `./dev.sh check` and `./dev.sh test` | BLOCKED | `cargo`/`rustc` unavailable | -| Rust unit/integration tests (including new scoped COLD regression) | NOT RUN | Requires Rust toolchain | -| Live HTTP/protocol/LB regressions | NOT RUN | Requires built 0.34.0 binary | -| 0.33.8 / 0.34.0 / nginx paired RPS benchmarks | NOT RUN | No verified 0.34.0 binary | -| Zero-copy sendfile WWW setting | SOURCE IMPLEMENTED / NOT RUNTIME VERIFIED | Linux plain H1 static GET only, `off` by default | -| Sendfile editor Chromium save/reload | PASS | 375/1440px, simulated admin API, off -> auto -> off | -| Sendfile Rust adapter + fallback test code | ADDED / NOT RUN | TCP marker replacement, persistent socket, HEAD, range, conditional; compiler unavailable | -| Amdahl sendfile/nginx sensitivity study | PASS MODEL | No runtime evidence for simulated improvements | - -The 0.33.8 report in the supplied evidence archive recorded **706 passing -Rust tests**, **152 HTTP assertions**, **111 transport assertions** and -**17 LB assertions**. Those are *prior-version* results and are **not evidence -that 0.34.0 passes**. This report never transfers previous pass counts. - -## Synthetic cache-retention scenario (not an RPS measurement) - -Run: - -```bash -python3 scripts/performance/simulate_cache_034.py -``` - -Assume 20 vhosts, 100 HOT + 100 COLD objects each, all public/cacheable with -infinite TTL and adequate cache capacity; 1,000 configuration saves (800 -operational-only, 200 content changes to one vhost), followed by 500 uniformly -random GETs per save. A miss immediately reloads an object into its original -tier. No networking, filesystem, CPU timing, origin delay or concurrency is -modeled. Results with seed 3400: - -| Synthetic metric | Old global purge | 0.34.0 scoped retention | -|---|---:|---:| -| Object invalidations | 473,540 | 37,300 | -| GET misses | 470,006 | 37,069 | -| GET hits | 29,994 | 462,931 | -| HIT ratio | 5.999% | 92.586% | - -The modeled change yields **92.123% fewer invalidations**, **92.113% fewer -misses**, and **+86.587 percentage points** of synthetic cache HIT ratio. -**Measured 0.34.0 requests/s gain: unknown.** These values must not be -presented as a real speedup or a projection against nginx. - -## Previously measured comparison (0.33.8, not this candidate) - -From the attached 0.33.8 evidence, five paired benchmark runs with cache OFF, -median RPS: - -| Mode / payload | 0.33.8 RPS | nginx RPS | 0.33.8 / nginx | -|---|---:|---:|---:| -| Reverse proxy 1,040 B | 29,334.12 | 170,610.16 | 17.19% | -| Reverse proxy 64 KiB | 20,735.07 | 35,393.29 | 58.58% | -| Reverse proxy 1 MiB | 4,461.19 | 2,730.23 | 163.40% | -| WWW 1,040 B | 24,906.20 | 351,605.13 | 7.08% | -| WWW 64 KiB | 20,856.67 | 203,709.91 | 10.24% | -| WWW 1 MiB | 2,833.71 | 12,611.49 | 22.47% | - -The 0.33.8 1 MiB proxy path was **2.77% slower than rebuilt 0.33.7**. -Removing a per-response-frame mutex is a performance candidate but is **not** -a measured correction of that regression. No throughput gains are attributed -here to the new source changes. - -## Hypothetical 0.34.0/nginx gap sensitivity (NOT MEASURED) - -Run `python3 scripts/performance/simulate_nginx_gap_034.py`. Using the **real -0.33.8 measured medians** above, assume zero-copy removes 50% of the -read/copy part of per-request cost; that part hypothetically accounts for -20/40/60% of *total* cost for eligible static files. Model **1/3/5%** total -service-time savings from removed proxy-frame mutexes, independently. -Amdahl: `RPS_sim = RPS_0.33.8 / (1 - hypothetical_fraction_saved)`. The -result is a sensitivity calculation, **not a prediction or 0.34.0 benchmark**. - -| Workload | Measured 0.33.8/nginx | Modeled low | Modeled middle | Modeled high | -|---|---:|---:|---:|---:| -| Proxy 1040 B | 17.19% | 17.37% | 17.73% | 18.10% | -| Proxy 64 KiB | 58.58% | 59.18% | 60.40% | 61.67% | -| Proxy 1 MiB | 163.40% | 165.05% | 168.45% | 172.00% | -| WWW 1040 B | 7.08% | 7.08% | 7.08% | 7.08% | -| WWW 64 KiB | 10.24% | 11.38% | 12.80% | 14.63% | -| WWW 1 MiB | 22.47% | 24.97% | 28.09% | 32.10% | - -These ratios apply only to eligible Linux plain HTTP/1.1 static workloads -with cache OFF. TLS, HTTP/2, HTTP/3, ranged, compressed and object-cached -responses are **not** accelerated by this bridge. Bandwidth, contention, -queueing, realistic filesystem and actual CPU profiles are not modeled. -Even the favorable modeled 1 MiB case remains ~3.1x slower than nginx WWW; -small WWW remains ~14.1x slower. Proxy 1 MiB already exceeds nginx in the -**prior baseline**, not as a demonstrated new 0.34.0 improvement. - -## Required rollout gates - -1. On a trusted Linux build worker with the pinned Rust toolchain, run - `./dev.sh check`, `CARGO_INCREMENTAL=0 ./dev.sh test`, and a locked release - build. Resolve any compiler/test issues before a deployment. -2. Follow `scripts/performance/README.md` to run `regression.py`, - `transport_regression.py`, `lb_regression.py`, and - `concurrency_regression.py` against verified 0.33.8 and 0.34.0 binaries. -3. Exercise administration saves during simultaneous requests: save keepalive - values and confirm unchanged HOT/COLD entries and hashes; change one backend - origin and confirm only its scope is purged, both before/after restart. Check - that old snapshots cannot recreate invalidated objects. -4. Run the five-repeat, 15-second paired benchmark against 0.33.8, 0.34.0 and - nginx on the *same host and affinity*, including large proxy responses, CPU, - RSS, resource limits and response correctness. -5. Compile the source-level Linux `sendfile` bridge and validate on a running - RustProxy process: plain HTTP/1 sendfile path, headers/body exactness, - keepalive/pipelining, slow clients, cancellation, HEAD, range, cache, - compression, TLS/H2/H3 fallback, symlink security and observability. Do - **not** activate Auto on production before that gate passes. -6. Complete a mixed-traffic soak and staged rollback rehearsal. Until then, - do not describe 0.34.0 as production-accepted. diff --git a/docs/TODO.md b/docs/TODO.md deleted file mode 100644 index 5c4420d..0000000 --- a/docs/TODO.md +++ /dev/null @@ -1,19 +0,0 @@ -# RustProxy TODO - -This file contains open work that remains after the current RustProxy 0.25 Web Server implementation. - -## Web Server follow-up - -- Multipart byte-range responses if required; current single-range/If-Range support remains the default scope. -- Named locations only if they can reuse the existing vhost/location router rather than introduce a second router. -- Optional PHP `PATH_INFO` script/path splitting after a confinement-safe rule is defined. -- Additional FastCGI observability (bounded active/error/latency counters) without per-script labels. -- A race-resistant non-Unix filesystem confinement implementation before enabling the Web Server on non-Unix targets. - -## Release acceptance - -- Run `./dev.sh fmt`, `./dev.sh check` and `./dev.sh test` on a host with the stable Rust toolchain. -- Run live static-file acceptance over HTTP/1.1, HTTP/2 and HTTP/3, including ACL/auth/custom-error/header behavior. -- Run a real PHP-FPM acceptance matrix over a Unix socket and loopback TCP, including `index.php`, front-controller `try_files`, POST bodies, timeouts and source-disclosure failure tests. - -Completed release work belongs in `CHANGELOG.md`; planned version work belongs in `ROADMAP.md`; implementation/audit evidence belongs in the matching audit document. diff --git a/docs/VHOSTS.md b/docs/VHOSTS.md index 03aef56..487db42 100644 --- a/docs/VHOSTS.md +++ b/docs/VHOSTS.md @@ -234,7 +234,7 @@ origin do not expose a selected backend or stale origin timing. | `pool_recycle_seconds` | 3600 | 0 (off) or 60..604800 | | `http1_max_buffer_bytes` | 131072 | 8192..1048576 | -The new default of 64 applies when this field is absent in a newly created or deserialized vhost; existing saved values (including 4) remain unchanged. Each idle upstream connection consumes a file descriptor and memory, so use a smaller value for resource-constrained deployments with many origins. See `docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md` for the live configuration A/B evidence. +The new default of 64 applies when this field is absent in a newly created or deserialized vhost; existing saved values (including 4) remain unchanged. Each idle upstream connection consumes a file descriptor and memory, so use a smaller value for resource-constrained deployments with many origins. These configure HTTP idle reuse and TCP keepalive; HTTP/2 also uses pings for active connections, with a 10-second ping timeout and no pinging solely to keep idle diff --git a/docs/WEB_SERVER.md b/docs/WEB_SERVER.md index cebb2f9..0f2a3ca 100644 --- a/docs/WEB_SERVER.md +++ b/docs/WEB_SERVER.md @@ -134,9 +134,7 @@ does not change the listener protocol configuration. tested against a running 0.34.0 RustProxy binary in the current environment. The included Rust unit/integration test and real-network test cases must pass on a Linux build worker before enabling in production. Sendfile cannot by -itself close the small-response gap to nginx. See -`docs/TEST_REPORT_0.34.0.md` and -`scripts/performance/simulate_nginx_gap_034.py`. +itself close the small-response gap to nginx. Live protocol and performance acceptance is still required. ### Intercept application HTTP errors (0.29.7) diff --git a/scripts/performance/README.md b/scripts/performance/README.md deleted file mode 100644 index 1090d21..0000000 --- a/scripts/performance/README.md +++ /dev/null @@ -1,354 +0,0 @@ -# Live performance and regression harness - -## DEV OPT3 / 1 KiB response: uncompiled source - -`docs/PROXY_1KB_OPT3_DEV_0.34.0.md` describes 7 optimized hot-path sites -and how to benchmark their **newly rebuilt** executable. Baseline -`rust-proxy(2)` passed 89 real proxy/policy/cache checks. This runtime result -is not evidence that uncompiled OPT3 changes work, nor a measured OPT3 RPS gain. - -These scripts exercise actual RustProxy executables through loopback sockets. -They do not simulate Rust request handling. The Python server in `peer.py` is a -controllable test **upstream**, used to verify streaming, non-replayable writes, -headers, cache policy and WebSocket frames. Performance measurements use nginx -as the shared upstream instead of Python. - -## DEV 1 KiB histogram writes (source candidate) - -`VhostMetrics::complete` now writes one exclusive histogram bin for each -completed exchange instead of up to 11 cumulative atomics. Both Prometheus -exporters reconstruct `le` and `+Inf` at scrape time. Unit tests cover boundary -semantics. This change has not been compiled or benchmarked: see -`docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md`. - -## DEV 1 KiB cache counter follow-up - -See `docs/PROXY_1KB_COUNTERS_DEV_0.34.0.md`. Cache domain metrics no longer -lock the HOT store on BYPASS; analytics cloning is deferred until the queue -has capacity. The new **source** remains uncompiled in the editing environment. -For quick 1,040-byte loopback diagnostics with ApacheBench (not a production -acceptance benchmark), use `python3 scripts/performance/ab_1kb_dev.py --help`. -The authoritative long-run comparison remains `benchmark.py` with pinned -CPU sets, an explicitly equal upstream idle pool and multiple `wrk` repeats. - -## DEV 1 KiB follow-up on 0.34.0 (newer source) - -See `docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md`. The previous executable's 4-vs-64 -idle-connection tuning experiment is real loopback evidence, but the *edited* -source has not been compiled. Its new default is 64 for new vhosts, whereas -explicitly saved `4` remains `4`. Keep the idle limit equal between before/after -*code* benchmarks; measure configuration tuning in a separate labelled series. -The source release is not eligible for a claim of 100k RPS or 80% nginx yet. - -## Reverse-proxy optimization revision 0.34.0 - -After compiling the refreshed candidate, measure the same unmodified 0.33.8 -binary and nginx reference as before, without turning off logging, request -IDs, ACL, cache-policy enforcement or LB health. Profile request-size 1,040 B -first. Before RPS runs execute full proxy, LB, transport and concurrency -regressions (including single-member capacity, recovery, sticky cookies, -replays on safe methods, rejected non-replayable writes and H2 connection -shutdown). Treat `--max-idle` tuning and recycles as separately labelled -experiments rather than mixing them into the primary comparison. There are no -new measured RPS figures in the source candidate. - -## Follow-up RPS candidate on 0.34.0 - -`docs/PROXY_RPS_0.34.0.md` tracks the code changes and their regressions. -Compile before/after binaries from the two 0.34.0 source revisions and use -the paired benchmark below; a live run against the older binary cannot -establish a measured RPS improvement for the uncompiled candidate. Keep -`max_idle_per_host`, recycling, HTTP1/H2 settings and all application security -and logging features equal between runs. Treat any configuration tuning as a -separate experiment. Profile the resulting binary with `perf` only after -correctness checks pass. - -## Offline cache-retention model for 0.34.0 - -`python3 scripts/performance/simulate_cache_034.py` compares cache invalidation -policies under a deterministic synthetic sequence of vhost saves and uniform -cacheable GETs. The model contains **no RustProxy server**, no timing and no -network traffic; its percentages are **not measured RPS gains**. See -`docs/TEST_REPORT_0.34.0.md` for assumptions and the live acceptance gap. - -## Requirements - -- Linux with readable `/proc`, Python 3.10+, executable RustProxy binaries. -- nginx and wrk for performance comparisons. Both paths are explicit; no - system-wide installation is required. -- `taskset` from util-linux for the benchmark's explicit CPU affinity and for - optional affinity settings in the other scripts. -- Optional Python `brotli` and `zstandard` decoders increase compression - acceptance coverage. A missing optional decoder is reported as **skipped**. -- Permission to bind unprivileged loopback sockets and start local processes. - In a sandbox without sockets or `/proc`, run in the approved local execution - environment. Do not replace real runtime results with a peer self-test. - -Each run creates a new private temporary data directory and initializes its own -administrator with a random password supplied only over stdin. Configurations -are applied through RustProxy's existing authenticated CLI/API and listener -changes receive a restart. No historical SQLite database is copied or edited. -TLS keys, databases and passwords are never written to the evidence directory. -Temporary state is outside the evidence tree and is removed on normal exit. -`fixture-cleanup.json` records the cleanup check. Interrupted runs can leave an -unrelated `/tmp/rustproxy-perf-*` directory; do not include it in a release. - -Before timed benchmark, soak or syscall-profile work, `wait_quiescent()` waits -for the existing authenticated ACME runtime API to report `ready` or -`unavailable`. A fresh RustProxy data directory can start a Python/Certbot -environment bootstrap even when no certificates have an issuer. The gate -requires no configured issuers, jobs or renewal observations, the same live Rust -PID and start time, and no child processes for five continuous seconds. It polls -every 0.5 seconds and fails after 180 seconds instead of measuring installation -work. Selected status, process count and timing evidence is saved in -`