From b6a3e122258ae341492a0335a1217589a85a34d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Gruszczy=C5=84ski?= Date: Sun, 18 Oct 2026 07:48:59 +0200 Subject: [PATCH] v0.34.0_7 --- CHANGELOG.md | 22 ++++- README.md | 21 ++++- ROADMAP.md | 15 +++- SHA256SUMS | 38 ++++---- docs/API.md | 2 +- docs/LIVE_LOGS.md | 2 +- docs/MIGRATION.md | 2 +- docs/PROXY_1KB_OPT5_DEV_0.34.0.md | 93 ++++++++++++++++++++ scripts/tests/proxy_1kb_opt5_static_test.py | 58 ++++++++++++ scripts/tests/rps_hotpath_034_static_test.py | 10 ++- src/balance/lease_runtime.rs | 11 +-- src/balance/tests.rs | 15 ++++ src/compression/tests.rs | 13 +++ src/forwarding.rs | 28 +++++- src/forwarding/tests.rs | 35 ++++++++ src/proxy/forward_entry.rs | 3 +- src/proxy/request_id.rs | 19 ++-- src/proxy/request_id/tests.rs | 9 ++ src/proxy/upstream_helpers.rs | 30 ++++--- src/traffic.rs | 65 +++++++++----- src/traffic/tests.rs | 64 ++++++++++++++ 21 files changed, 471 insertions(+), 84 deletions(-) create mode 100644 docs/PROXY_1KB_OPT5_DEV_0.34.0.md create mode 100755 scripts/tests/proxy_1kb_opt5_static_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0aea037..b132b94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ ## 0.34.0 +- DEV 1 KiB OPT5 multi-site follow-up: use direct CSPRNG 256-bit request ID + bytes mixed with the existing per-process sequence rather than SHA-256 per + request, retaining 64 lowercase hex digits and protected `X-Request-ID`. + Construct the absolute original URL only when the frontend enables Tinyauth, + without changing trusted/untrusted forwarding or IPv6 validation. Borrow the + healthy LB response detail instead of allocating the string `ok`. +- Keep traffic analytics non-blocking and bounded at 2,048 queued samples; + process up to 64 samples per dashboard mutex acquisition in arrival order, + allocate vhost keys only on first observation, classify User-Agents without + a lowercase string, reuse static common status-family labels and render + one-vhost views without cloning every top table. Skip response compression + policy resolution when no codec is accepted by the global ceiling (e.g. + `identity`), retaining all enabled codec, scope and exclusion behavior. +- Source-only until OPT5 compilation and Rust tests. The compiled OPT4 input + passed 151 live HTTP checks (0 failed, 1 optional Zstandard skipped). + Brief shared-host OPT4/OPT3/nginx tests are mixed at 16/64 connections; + see `docs/PROXY_1KB_OPT5_DEV_0.34.0.md`. No version increment. + - DEV 1 KiB follow-up (OPT4): keep ordinary unmodified public request targets borrowed through URI rewriting, avoid creating a metering body adapter for request bodies with no frames/trailers, and consume already-validated upstream @@ -15,8 +33,8 @@ refresh owned targets, streamed upload metering, full hop-by-hop filtering for complex values, TLS/H2/H3 authority handling and request limits. - OPT3 compiled binary passed 151 live HTTP checks, 0 failed, 1 skipped - (optional Zstandard). OPT4 Rust sources must still be compiled and benchmarked; - see `docs/PROXY_1KB_OPT4_DEV_0.34.0.md`. + (optional Zstandard). OPT4 was subsequently compiled and tested similarly; + see `docs/PROXY_1KB_OPT4_DEV_0.34.0.md` and newer OPT5 analysis. - DEV 1 KiB throughput follow-up (OPT3): keep normalized safe routing diff --git a/README.md b/README.md index b4d4948..57ffb10 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,25 @@ # RustProxy 0.34.0 -## DEV: OPT4 small-response proxy hot path (uncompiled sources) +## DEV: OPT5 multi-hot-path tuning (source candidate; uncompiled) -The compiled OPT3 binary passed 151 real HTTP acceptance checks (1 optional +OPT5 builds on the **compiled OPT4** (151/151 executed live HTTP checks passed; +1 optional Zstandard check skipped). The shared-host 1,040-byte HTTP/1.1 +comparison shows OPT4 at about 39,054 RPS with 16 connections and 36,425 RPS +with 64; mixed results versus OPT3 mean there is no proven broad RPS gain. +OPT5 removes unnecessary ID hashing and unauthenticated original-URL +construction, avoids per-success health-detail allocation, batches bounded +traffic analytics, avoids routine status/User-Agent allocations, and fast-paths +unoffered response compression. All policies, observability, caches and +protocol support remain enabled; no version bump. **OPT5 has not been built +or measured.** See [`docs/PROXY_1KB_OPT5_DEV_0.34.0.md`](docs/PROXY_1KB_OPT5_DEV_0.34.0.md). + +## DEV: OPT4 small-response proxy hot path (subsequently compiled) + +The compiled OPT4 binary passed 151 real HTTP acceptance checks (1 optional Zstandard case skipped). OPT4 removes avoidable allocations and string work for ordinary uncached HTTP/1.1 requests, with regressions for empty request -bodies, URI/Host/Connection semantics and Unicode log truncation. OPT4 has -**not been compiled or RPS-measured**. No settings or functionality disabled; +bodies, URI/Host/Connection semantics and Unicode log truncation. Its brief +shared-host RPS comparison is mixed, so no stable improvement is claimed. No settings or functionality disabled; version remains 0.34.0. Details: [`docs/PROXY_1KB_OPT4_DEV_0.34.0.md`](docs/PROXY_1KB_OPT4_DEV_0.34.0.md). diff --git a/ROADMAP.md b/ROADMAP.md index 2f8979d..afe356e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,14 +2,25 @@ ## 0.34.0 - Scoped cache correctness and proxy body hot path +- [x] Compile supplied OPT4 binary; run real HTTP regression (151 passing, + 0 failed, 1 skipped) and a short OPT3/OPT4/nginx 1,040-byte diagnostic run. +- [x] OPT5 source: remove per-request SHA-256 while retaining CSPRNG entropy, + skip Tinyauth-only original URL work when Tinyauth is absent, avoid a pooled + backend success-detail allocation, batch analytics processing, use borrowed + status/UA labels, avoid scoped compression resolution for unaccepted codecs. +- [x] Preserve exact forwarding and analytics semantics with new Rust tests; + retain 0.34.0, TLS/H2/H3, WebSocket, cache, ACL, logs and bounded queues. +- [ ] Compile and test OPT5; run full real-protocol regression and >=7 + 30-second CPU-isolated A/B/nginx runs before accepting a throughput gain. + - [x] OPT4 source candidate: borrow original no-rewrite public target, bypass the request body adapter only for bodies reporting end-of-stream, avoid redundant upstream URI refcount operations, and simplify common forwarding/header/log branches without weakening validation. - [x] Add OPT4 regression tests for body accounting and permits, Unicode, URI identity, and safe Connection header filtering. -- [ ] Compile and run OPT4 Rust tests, re-test ACL/TLS/H2/H3/WebSocket/cache; - collect A/B-nginx RPS and profiling evidence before accepting a gain. +- [ ] Complete OPT4/OPT5 Rust unit suites and extended TLS/H2/H3/WS/LB + acceptance; the short OPT4 RPS comparison did not prove a stable gain. - [x] DEV 1 KiB: remove shared HOT store mutex from cache outcome/reason diff --git a/SHA256SUMS b/SHA256SUMS index 50605c0..0ce5ad2 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -2,12 +2,12 @@ 4140a2956264b44f259e6a37fc00ba02097bbe68b09296e04d3d243c1287129b .env.example 9e6356475bf587091bab72246ec9e325b498f7e48a6220e9a18baa50d0a385a6 .gitignore 46bab1d2c2d6e0a80d57a6b232b1ba4c220884b95cd45ddccbafce2aa160cddb AGENTS.md -16b657a20824fd310c1eab6498b0b0db6f250a56b09c556d9c149ce1f61f0c0a CHANGELOG.md +3e46648fd8c91548ea77dcac4f6cbdbc5741c64b33240c44a50c138597880e99 CHANGELOG.md ca201e3cd3f9842eae0006d0b40a0a72bcd3d4da6e4e5ecaf200d1b9a1390db2 Cargo.lock fec5f1bae9a59ef1fabd27c5e45d31448dd8f811736eebfa8826b3806a7e3349 Cargo.toml 25c7f3c8162cbf5cc156439774118a81dae9fdc52b611ec7e40c230fc99a772c Dockerfile -73194a4738e42a3c36623e750022bd2ef6dc9483c287c583a0a172d815962454 README.md -2c387b854dc5433c3dd4a8cd1e70272f2fd8c5d5a483cd9f7fe9b4ceeed95c36 ROADMAP.md +d48bcdcda6d9261d7b358c39b8bdf93d39b722b3751af3b5ebe192e16bd8b4e6 README.md +7bba42713fa944908f5f3611841bf6b36377240ae4b77ec692ff48ea662b44f9 ROADMAP.md da8311e3db945290666324a2f24bf4521b365272d7ac3f862ded1be993eea646 UPDATE_CHECK.toml f645f218dd062bf6cb54a8ee634290562cad364a4cccf09f16e3c8303582af8c build.rs ea8d091843f86f4c66a5c76b20268b2feb397070851b6e05c5bc7d7a8d9157eb deploy/README.md @@ -27,7 +27,7 @@ e52b31d0abf65ae3034f6cb367cca113633014ce8d1d799ad4dd5624d2c7b8bb docker-agent/n 5220d669171ec3ad65f50e0bc8de5178dadaa4904def8e1a4f61002aa9e9a55b docker-agent/server.go 875faa3969df9409d79f050a84b4183496f557edda3d34aaec19fe556cf8c471 docs/ACME_PROVIDERS.md ec15163f5f51c9212003b7edd1fc380321311c2496fbaa9b45c838281b9af475 docs/ACME_RFC2136.md -e21a5c3cea78386fcb3ebd6516a66eed2ecd1966481ae0d3b2f57fa01010542e docs/API.md +00aef5eea37a8f56d7ea95e63ddcfdf39bac041168af29582b5ab2ab62d4c81e docs/API.md f27053f8a93d7883025a04922d015e0b697cad653e714be632b757e167297de1 docs/API_ACCESS.md d30551a52ffd34c5963094d2826ba39795210a4e6a4e4d7333066dae7456fc50 docs/ARCHITECTURE.md fb99fac0a490af262a0ecdad593ffe5164004c2166fe7ddd607476bcc56a1623 docs/CACHE.md @@ -43,10 +43,10 @@ f749f7bd2eb4ae9c484f77a3cd8cd5ff21445319a3938e0384a3414134fe713a docs/FILE_LOGS 36599e0b59164e662442f3e8203c16b3123d09b7684d8ecdbffce42d22cc221b docs/HTTP3.md a00ec3c1d3b28ba10677435128e031e5f18fe8c5cef2ac90eb0f6779cefeb434 docs/INSTALL.md 3b5c31c805e5aee0dc2100b81246a9b631b5abb996faa598ae94f87ba39305f3 docs/INTEGRATIONS_040.md -4bb56add544f15d54695cf797cd16a51c09024360cdeef803a88988f4a54b3a5 docs/LIVE_LOGS.md +bd85d4040fb7d213d7983596ad6f207cecbb03bfcfc399a4333f1e3fdf9211b9 docs/LIVE_LOGS.md 7c35c4cdbe7bbb81d94f3bf6aeb22b8a866af3e7f0d62bbedc299e4d6f264e0d docs/MANAGEMENT.md d8829cc68d982c933d04207cf7bc78cc918044c72e024e7ecad5220953bb3a79 docs/MEMORY_STRESS.md -e36247774051aa9c83c1a2fda91c1f7857991dcaa4fa90320ca5b492e31dbcf8 docs/MIGRATION.md +5551dc403f1b450a5d4bfd48b1130971751ec0ec51e4fc9b2f5cff1971a212ab docs/MIGRATION.md 5f090a190000c6badab2a74e5d4968e70c34de43b59c1452f5b05503b5c5c18a docs/MOCK_AUTH.md 1f85fac0e06e432ef6f744f0b5f9c1b7a27a1e9e1ad30dc1a218eab96b1caffe docs/MOCK_SERVER.md 6cd54c6e2c3e5c23ae0104e09e88ab6d93bde26bd92b23c1a078fe7c4ee04b55 docs/NGINX_GAP_0.34.0.json @@ -60,6 +60,7 @@ ad6e3f003b820a4f45e30e39577d120a69d167c70910bda4891971759458c3dd docs/PROMETHEU f13ee0005d57d2576ccc970745f17069096df4afa23d712ea0f748da85433f7d docs/PROXY_1KB_HOTPATH_DEV_0.34.0.md c9bbdd1cd8df02b2a19016b7bbeee1cf55917614ac672f1f94e0a05db7d309ef docs/PROXY_1KB_OPT3_DEV_0.34.0.md 27002c4d4e19bb7d58521b8ba4cd5faf180d2917b6ac200dd7fc3664c2060145 docs/PROXY_1KB_OPT4_DEV_0.34.0.md +8c26aa9ebebf54d912749e3336e548b59d805997366be2324157c3e56b887f5f docs/PROXY_1KB_OPT5_DEV_0.34.0.md deea89aed410c0619bdbdb12437edc7aff9dca2fec7fdc059e918c649fc36b51 docs/PROXY_HOTPATH_0.34.0.md a7d6189b0e8cb854918456e0063101ee77bfb2419a9ba0430eb0e3f8678c0f09 docs/PROXY_RPS_0.34.0.md 887d1fdcc3b546b024afcd18b1716b8e3a74a3800012bf4bda7a122d2eabe233 docs/README.md @@ -187,8 +188,9 @@ b0136544af717b75df2cb107900ea65c3f945fdd0040369dbb93919008fb954c scripts/tests/ 8d1903e2b17b3d59cd0a5bec3e298e468b3d88ed2dda8b724239cc45d8353c59 scripts/tests/progressbar_0332_browser_test.py 7fc86a34a47c6e1f0f9fe2748d0ea7f59e455f8bf4e5ad081e185356e43721b0 scripts/tests/proxy_1kb_opt3_static_test.py ec05272ad8c14c7a0265a84cc7b6ebc0dca0651413106d655681681873b88fc8 scripts/tests/proxy_1kb_opt4_static_test.py +37306b175190aa72b143d263060fa6441244eb40b2bcb54c82071cc87d73b54d scripts/tests/proxy_1kb_opt5_static_test.py 2a6e75c05f3860acf65ea8132f0083904478b690145115ec017c120d70f26cca scripts/tests/proxy_hotpath_034_static_test.py -6fa2f22816996e989dad9dc7884adbf24f02f0ca6a6634b06e219eb08526c7fe scripts/tests/rps_hotpath_034_static_test.py +0c50def966cbc95c22339d1c506763fab4d9e5c176a161205ab88d79af5be920 scripts/tests/rps_hotpath_034_static_test.py 5be5cdc60324f4f0f6638640000b888a5babb0a9bbf29511c824a0ba38878c2a scripts/tests/sendfile_034_panel_browser_test.py 978789bfe89d43034dfebddba49b9b17daedb1be031732734f95375e56c78306 scripts/tests/sendfile_034_static_test.py 1bb99b148d0297235b707f16865ca077329246fc19ecbb20a143bdeb56a23f37 scripts/tests/sidebar_nav_0333_browser_test.py @@ -326,12 +328,12 @@ c96a87e82ff0c1c9ce3d0cd5135903bb53af78d1c4fb6ebfe5fea503bf5644ea src/auth/store f5afae218f3a6f91bc2f1f5a90094630ce5b7ca574de61807b5a317b13937ec2 src/backup/validation.rs 426a946b2c9e244f09f6cbc5f9664b8ad0737529a9bb5a3d549d27196a1f8043 src/balance.rs ca1e03b56a872dca37ed9d05ead41c1538a808e80b09fb9494823c443cd51c61 src/balance/health.rs -109c55f9af8918d4f314f0796e19de06f3129eefee3ef405770b99fc368f2aee src/balance/lease_runtime.rs +70c44291960b835c7a456cf066469b0002e1e35ba497663361d659d452a52e5e src/balance/lease_runtime.rs aba0f4ae46f05b07bfc434b7174553ce20e5187e1b40a6cf946eb2aefbe1840b src/balance/model.rs 8550e5d64aa6fee442cea3e8e243fc4cfc1e8aa39726bc5cc6b2afa9307e3d05 src/balance/pool_core.rs 477132dd381e23f1be197c32ff58a0925f41d3e6aa63fe77c10973cefb2b76f6 src/balance/pool_selection.rs 8abb6048303ee8c022546a528d5693167766e7450efd8e20f37ec6fd0fa9ef5e src/balance/pool_view.rs -2ae82db4e3ff2d8df1f6fb3786b54ff61ed4a0e8d725d50fbb080b4fa0e641c6 src/balance/tests.rs +814be4263c35b84de2b3c3bb3c60d7ead9f593154ac85aa6c8ae35700ee591c7 src/balance/tests.rs 5049d2bd01a9c866f3bad2288d8d594df96bba6045dfe15cd22b4173562b7388 src/basic_auth.rs 9a8858b7ff461a4d961ba701ca3ed6eaec15c3f7f05eef8e0e30edc63617605a src/basic_auth/tests.rs 7def03ace07dc38a4806277d75b6ce673d33ddaaf1c4855fa0a3372be1cf9489 src/cache/cold.rs @@ -374,7 +376,7 @@ d4429cf4f35fd8d0082108fb62602fde72c9c2fc6edfadfe3268c6a8214a8f89 src/cache/test af28534625859ff66011e10c54cff8f82c2cfa1c173c4d99b89c27c1437f0659 src/cache/tests/v022_cache_tests.rs f76b41a2b0ffa5699a456cc2cfa0b063098c90552fe01e0d05e6ee91a01cbe57 src/cache/tickets.rs e0146bf5d2e2e33aedcccd9205e5bd457559c9d77dc7ce266f8c165d1b867f0f src/compression.rs -4bdbde197d135fee8be20e93427eba8ca7b60b801717ed7ca734319bedb6ef05 src/compression/tests.rs +2bb1ea392dc18553a03a6ae1c53f52712ae86571ea31b1bcd93c8fdec2cb60ec src/compression/tests.rs 7bc8285cb2012506fd1a38a71dffa561f885f4ab65a9b2a7c8344319e3cd1cf6 src/config.rs 56b3d37e7447995bec2fdda1c17dcc74b60958a63ca88b88ff57e56347d8b0b9 src/config/backend.rs beb831a208c4e5f7ebe8327686b1e0edc66608c7032f895cb4fc3366b189aae0 src/config/compile.rs @@ -395,8 +397,8 @@ b45d6040810c93ae2acb04ef6b0a59c189751edf9008cd81a4521c9e04e7f07d src/docker.rs 4c2c988191a97f81568a8df64af772463d010844f1abf327871901cdb32d4336 src/docker/model.rs 5f4610c59b2d4ff6a205a2d9ae11bc84be689b2a68302459f546b3f43e53593c src/docker/reconcile.rs a12c47df4b90b6fc36c978a42477e6ddffbbbf69aed7252101c2fd829709bd6e src/docker/tests.rs -2cb42ac0a5444fdcfea34c616dd6fe5f5ea1d06c84bcd64714ddf096830ec99f src/forwarding.rs -b25fbb00de0e37ad5fa1a049839265e5f7e9f83b6214c40a95f57ea4807685bc src/forwarding/tests.rs +ae46f4a971119d300d79fa506ed116054313334484546ba2f411e893e4e7586c src/forwarding.rs +4622bf34a316fca5dd524d229652a1fef8b1828f34e2c3e9bc0e35fe03b6a393 src/forwarding/tests.rs af7a4e91c32ffbe4701c8c06e48ba3f8bbbe55e8b9e877aecb30ccdf858e9e8f src/geoip.rs 80850f7acd1ed4c0048b065e9e81740869a8efbb0d9b5d2e83d8ec7fbc494703 src/geoip/mmdb.rs 06aa056d789742cc11c8826c338d985e6a6fb22da6c3530937eec38e35478034 src/geoip/mmdb_tests.rs @@ -456,20 +458,20 @@ cfa941b60b5d167ec9d8e2566ea3342b588d1a9078a6ca711ad6062ddf00fec9 src/proxy/forw 0a921474ef2b93eabdaf3888651cf29130eace5ccb1e0f5d6bc25e6aa69280be src/proxy/forward/success_prepare.rs 3f21439bbe4966fff43b3ebce8b59fa76ecce376197b4c56a2dce3a6d98f5556 src/proxy/forward/upstream.rs bb8c9117e05b0cb4bbfe029ef4d1a031fc755cdce255b15dad072afd1c845737 src/proxy/forward/web_source.rs -2b6ae70b84fd1eba0dba6d8d7e2db62227193dc5b77e4a6111ceab1b219800c2 src/proxy/forward_entry.rs +2c681e48d6be7db784b65cc186bced40f1b6c777d97a4d222c653008b8406a31 src/proxy/forward_entry.rs 9b25f5e3ea4443a63ee6f45d12e1957dd316fe8e85325fc00ea416faaab1025a src/proxy/h2_graceful.rs cc7fe83f42acb19d79abe83107427cdc6b0ed14083702f4fe66af22f7cdfc4cc src/proxy/headers.rs 0f9314d1380febe436def36b5f381731e97cbac90625637575f38f3d72eaaace src/proxy/headers/tests.rs 3f25429418941eb72c3847125340db74684ad316eb278ecf658953026fd41f4f src/proxy/mod.rs 55133ceace2e5262d9c54fa41bea099a27b817d1e09c1cd0e2493fd9cee4da38 src/proxy/redirect_helpers.rs -01b3027cfe92512f71be056a1ae26749b3ad3417d43388f1c05dcf5884d83966 src/proxy/request_id.rs -d2df55b6ccfc7ceb8a3d2c49f664acd183448e3d932d5399d6692e87f79a094b src/proxy/request_id/tests.rs +2280841494c3e14ccc5e3f02b026fe3c4328a3dab04ab449b9286bb69e39c028 src/proxy/request_id.rs +b9a2307c6aa7f16613d935508a61574dfee964020b1b5be86293df2d684615d8 src/proxy/request_id/tests.rs e8d1210a6b8e8c6a388d016d22e7c18255cf737c5583124b35cd50bd2592f23c src/proxy/tests/authority_tests.rs 1aa9bbbfb2659facdaeaf185e11221b1c8cd5fe6e23c7c1dacd3aa55e3db4fcb src/proxy/tests/cache_fast_path_tests.rs 0f2eacadf1447db8aa1c1a78005aa262e0f747096bfe7e0a0e41a2cd31a0d17f src/proxy/tests/h3_navigation_tests.rs d134b0eb617f82f9971b3f3461379ea37c57bd0079bf54748d0c434b2ef1b475 src/proxy/tests/http_invalidation_tests.rs a74815a0b46d1c94ffbf8cc3a8bbbc967daa4aec0981139d2d92624c10553026 src/proxy/types.rs -9a8e38e88bff389e448f54dc07e63fef9652acc72e05833363668c28949daccf src/proxy/upstream_helpers.rs +b45cf5baf45e37915c8cfac43e845976f874f0aa5474f167fa943c0f38decade src/proxy/upstream_helpers.rs 2d9142e3ae95895b5271b4009ab48236a3a2074ce51179c021193b35777560e4 src/proxy/web_cache_revalidation.rs d7b4b809b12ee89293df45e9a64b1c9981fb3f89deaab3762f4a3ad3992dd7b1 src/proxy/web_cache_revalidation/tests.rs 69f775f6746b4c6d729dc10961dbef54168846f4ce9cf7fe453ab5a2b879c9b8 src/proxy/websocket.rs @@ -523,8 +525,8 @@ d9123c623e2a97d09e41edaa29993b4b2c8cba39a3abdf2b48a205aa9a9c2939 src/system_sta 2af2164ecdf2b7fd9eef7c6e358dec053dfac9aedaa501dbb75df742d18638b2 src/tls/tests/acme.rs 93b359e9defd62506729bc90e56c96783e2f14dd477dd67f337ca83149da7168 src/tls/tests/external_paths.rs 10c4a187e3aeb0c72a8c8968ea0cc8bfae4e3cb2f3d834ebd3da1056c4171ec5 src/tls/types.rs -0d2ce0a7f5200437a1f965c62f72c91a681949a0bdec8ef4ccdb837a55626ed7 src/traffic.rs -aa6cea1b22cb5805934e1f819e9c5254a7b4f745025842144f6e6ae671eb170d src/traffic/tests.rs +6ae584944343b83ee3a2772cd93c985ddeaece7a9e6467b7cdbf5da22ee88028 src/traffic.rs +e796efe9d94e2e79220287196fe2d2d9f4a1c7640b9f71652d409478e25d3c09 src/traffic/tests.rs d88c92f5a2a7773bc3e715d24039db35da4df53881f1a0859eec819b96f06d3b src/transform/apply.rs 87f8af5261d79072f8215f1fb8a90715a58e7626544fd8176ce840e3111bdb92 src/transform/compile.rs b2c5236589093454ec60984b785ad6a4af6149d47f6db37d8e08c17fec7a2c1b src/transform/css.rs diff --git a/docs/API.md b/docs/API.md index fa0afa7..e7b49ad 100644 --- a/docs/API.md +++ b/docs/API.md @@ -585,7 +585,7 @@ when enabled. Transformed cached representations add the `transformed` Cache-Sta extension parameter. `X-Cache-Status` is stripped if supplied by an origin but is not emitted by RustProxy. `diagnostic_headers.hide_mock_origin` is removed and old documents containing it are rejected on -export; `request_id` is the only diagnostic-header switch. The generated request ID is a 64-hex-character SHA-256 value and the same protected value is forwarded to the origin as `X-Request-ID`. No revision response-header behavior remains. TLS listener +export; `request_id` is the only diagnostic-header switch. The generated request ID is a cryptographically unpredictable 64-hex-character value and the same protected value is forwarded to the origin as `X-Request-ID`. (The OPT5 DEV implementation replaces the extra SHA-256 digest with CSPRNG output mixed with the per-process sequence, retaining the length and unpredictability.) No revision response-header behavior remains. TLS listener http3 omission is default-on; explicit null is persisted off. Legacy listeners use https_http3 with identical default/null behavior. diff --git a/docs/LIVE_LOGS.md b/docs/LIVE_LOGS.md index 4c8c64a..43bc66f 100644 --- a/docs/LIVE_LOGS.md +++ b/docs/LIVE_LOGS.md @@ -15,7 +15,7 @@ The CLI first probes the loopback administration `/healthz` endpoint over HTTP. This fixes the previous reconnect loop where `--livelogs` treated the TLS listener's expected HTTP-to-HTTPS `308` response as a stream failure. The same local transport selection is used by CLI config and certificate API operations. -The live path is separate from configured file/syslog sinks and from the `/logs` stored-log explorer. Request processing publishes ordinary routed request records into a bounded Tokio broadcast channel without waiting for viewers. Low-level malformed HTTP/TLS/HTTP2/HTTP3 protocol failures introduced in 0.26.0 are intentionally persistent-sink-only and do not enter this Live Logs ring, so scanner noise cannot evict normal live request history. A slow viewer receives an explicit `gap` event with the number of entries lost from that viewer's live queue. The CLI also consults the retained history after connecting/reconnecting and de-duplicates entries by request id. Each request now carries a fresh 64-hex-character SHA-256 `request_id`; the same protected value is forwarded to the backend as `X-Request-ID` for correlation. +The live path is separate from configured file/syslog sinks and from the `/logs` stored-log explorer. Request processing publishes ordinary routed request records into a bounded Tokio broadcast channel without waiting for viewers. Low-level malformed HTTP/TLS/HTTP2/HTTP3 protocol failures introduced in 0.26.0 are intentionally persistent-sink-only and do not enter this Live Logs ring, so scanner noise cannot evict normal live request history. A slow viewer receives an explicit `gap` event with the number of entries lost from that viewer's live queue. The CLI also consults the retained history after connecting/reconnecting and de-duplicates entries by request id. Each request carries a fresh, cryptographically unpredictable 256-bit (64-hex-character) `request_id`, generated by a thread-local CSPRNG and mixed with a per-process sequence; the same protected value is forwarded to the backend as `X-Request-ID` for correlation. From the OPT5 DEV revision onward, this format no longer requires a separate per-request SHA-256 operation. ## User-Agent, country and Traffic analytics diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 632deb6..9a67967 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -70,7 +70,7 @@ There is no SQLite schema or saved-configuration migration. Schema 8 and the exi SQLite schema 7 is upgraded automatically to schema 8. The migration adds `configuration_history`; on first open the current revision is seeded into history, and subsequent successful configuration saves retain the newest 512 revisions for the dashboard revision-history view. No manual SQL is required. -New vhost/location access fields are additive. Missing `method_policy`, location `acl_profile_id`, location country policy and location inline ACL fields retain the prior behavior. RustProxy-generated request IDs are now SHA-256 values and are forwarded to origins as protected `X-Request-ID`; custom request-header rules cannot replace that correlation identifier. +New vhost/location access fields are additive. Missing `method_policy`, location `acl_profile_id`, location country policy and location inline ACL fields retain the prior behavior. In that release RustProxy-generated request IDs became SHA-256 values forwarded to origins as protected `X-Request-ID`; custom request-header rules cannot replace that correlation identifier. OPT5 DEV 0.34.0 retains the same 64-hex-character protected identifier but produces it directly from the CSPRNG mixed with a sequence, without the additional SHA-256 digest. ## 0.25.1 -> 0.25.2 diff --git a/docs/PROXY_1KB_OPT5_DEV_0.34.0.md b/docs/PROXY_1KB_OPT5_DEV_0.34.0.md new file mode 100644 index 0000000..aaaf77f --- /dev/null +++ b/docs/PROXY_1KB_OPT5_DEV_0.34.0.md @@ -0,0 +1,93 @@ +# RustProxy 0.34.0 DEV — OPT5, 1,040-byte reverse-proxy optimization + +## Acceptance boundary + +The source package preserves **0.34.0** and all existing configurable services, +logs, dashboards, cache/ACL/auth policies, streaming, TLS/H2/H3/WebSocket, +failover, retry and bounded-resource behavior. OPT5 is a **source-only** candidate: +no Rust compiler is available in the editing environment. The new code's +compilation, Rust unit tests, binary throughput and full-protocol interoperability +are unverified until the developer builds it. The compiled OPT4 executable is +the known test input, **not** a benchmark of OPT5. + +## Compiled OPT4 input verification + +Input: `/mnt/data/rust-proxy(4)`, SHA-256 +`1c4c701f11b57b1293020061c8af21d5a9df0c86f03290018a90edc007667b24`. + +- The real HTTP regression runner completed **151 passing**, **0 failing** cases; + **1 skipped** because Python `zstandard` is not installed. Protocol tests for + TLS/H2/H3 are a separate acceptance gate and were **not** passed by this run. +- Same-host ApacheBench HTTP/1.1 keep-alive, 1,040-byte body verified by exact + SHA-256, cache OFF, 4 Tokio workers, 64 idle HTTP1 upstream connections, + 3-second warm-and-measure samples, two repeats per variant/concurrency: + +| RPS, median of 2 samples | 16 clients | 64 clients | +| --- | ---: | ---: | +| OPT3 | 37,017 | 40,668 | +| OPT4 | 39,054 | 36,425 | +| nginx | 99,808 | 110,783 | +| OPT4 vs OPT3 | +5.5% | -10.4% | +| OPT4 / nginx | 39.1% | 32.9% | + +There were zero HTTP failures and the response body was verified before/after +measurements. Shared CPU and only two short samples are diagnostic, NOT a stable +speedup or production proof. The 100k RPS and 80%-of-nginx goals remain unmet. + +## OPT5 changes (8 coherent hot-path optimizations) + +1. `src/proxy/request_id.rs`: eliminate per-request SHA-256. Keep the existing + OS-seeded/reseeded thread-local CSPRNG and 256-bit ID; XOR a monotonic + sequence into the first 8 nonce bytes. Output remains uniformly random, + 64 lowercase hex characters and the protected `X-Request-ID` correlation + remains. Random IDs continue to be unique with overwhelming probability. +2. `src/forwarding.rs` and `src/proxy/forward_entry.rs`: only construct + `Context.original_url` for a frontend with configured Tinyauth. This is the + *only* runtime consumer of that absolute URL. Public `context(...)` still + constructs it by default. Trusted/untrusted forwarding and all validation + still execute identically; IPv4/IPv6 and `x-original-url` for Tinyauth remain. +3. `src/balance/lease_runtime.rs`: borrow `"ok"` for successful backend health + observations instead of allocating a `String`. 5xx failures still record the + exact original `status=NNN` detail and slow/passive health counters are kept. +4. `src/traffic.rs`: receive 64 samples at a time (Tokio `recv_many`) and apply + them under a single dashboard-state mutex. Order within the existing bounded + 2,048-sample queue is preserved; waiting is outside the mutex and producer + backpressure and drop accounting are unchanged. +5. `src/traffic.rs`: reuse map entries with borrowed vhost names. New keys are + cloned only when a domain first appears, preserving the max 64-vhost bound. +6. `src/traffic.rs`: classify status 1xx–5xx using static labels; preserve + unknown extension status labels such as `7xx`. +7. `src/traffic.rs`: replace per-sample lowercase allocation for User-Agent + family detection with ASCII-insensitive byte-window comparisons, and avoid + copying all TopTable maps when displaying a single vhost. +8. `src/proxy/upstream_helpers.rs`: skip resolving inherited/vhost/web-service + compression settings when no encoding is accepted even under the global + codec ceiling (for example `Accept-Encoding: identity`). Existing compression + handling is retained for every accepted codec and request method. + +The caller's front-end policies remain evaluated on every request. No log, +metric, access control or authentication feature was disabled or made optional. + +## Verification and outstanding gates + +Static source, SQL migration and managed-upgrade tests are executed and their +outcomes supplied in the external evidence archive. New Rust test fixtures cover +request IDs, forwarding/trusted IPv6, compression, healthy/5xx backend detail, +analytics batching, top classes, country-independent User-Agent matching, +reset and the bounded queue. + +To verify the new build on the developer's system: + +```bash +./dev.sh check +CARGO_INCREMENTAL=0 ./dev.sh test +./dev.sh build +``` + +After building, repeat the actual HTTP regression, LB regression and TLS/H2/H3 +transport tests and collect at least seven 30-second CPU-isolated, warmed, +interleaved OPT4/OPT5/nginx measurements with `wrk`, p95/p99 and CPU/RSS. +Treat the non-tested binaries, short shared-host tests and simulated Python +checks as distinct types of evidence. Measure the secondary Web Server path +separately. Do not alter the version or silently relax cache, auth or timeout +policies to reach a RPS target. diff --git a/scripts/tests/proxy_1kb_opt5_static_test.py b/scripts/tests/proxy_1kb_opt5_static_test.py new file mode 100755 index 0000000..8f71a22 --- /dev/null +++ b/scripts/tests/proxy_1kb_opt5_static_test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""OPT5 invariants, static-only. Does not compile or benchmark Rust.""" +from pathlib import Path + +root = Path(__file__).resolve().parents[2] +def read(path): + return (root / path).read_text(encoding='utf-8') + +assert 'version = "0.34.0"' in read('Cargo.toml') +identifier = read('src/proxy/request_id.rs') +assert 'rand::random()' in identifier +assert 'let mut token: [u8; 32]' in identifier +assert 'sequence.to_be_bytes()' in identifier +assert '*byte ^= sequence_byte;' in identifier +assert 'request_id_hex(&token)' in identifier +assert 'let mut encoded = vec![0u8; digest.len() * 2]' in identifier +assert 'Sha256::new()' not in identifier +forwarding = read('src/forwarding.rs') +assert 'self.context_with_original_url(incoming, peer, local, tls, authority, uri, true)' in forwarding +assert forwarding.count('if include_original_url {') == 2 +assert 'let mut ips = chain(incoming).unwrap_or_default();' in forwarding +assert 'crate::config::request_host(authority_text)?;' in forwarding +assert 'frontend.is_some_and(|frontend| frontend.tinyauth.is_some())' in read('src/proxy/forward_entry.rs') +assert '&context.original_url' in read('src/tinyauth.rs') +lease = read('src/balance/lease_runtime.rs') +assert 'let failure_detail = is_5xx.then(|| format!("status={status}"));' in lease +assert 'let detail = failure_detail.as_deref().unwrap_or(reason);' in lease +assert 'observe_passive_with_detail(' in lease +assert 'lease.response(' in read('src/proxy/forward/upstream.rs') or 'lease.response(' in read('src/proxy/forward/success_prepare.rs') +analytics = read('src/traffic.rs') +for literal in ['mpsc::channel::(QUEUE_CAPACITY)', 'recv_many(&mut batch, 64)', + 'for sample in batch.drain(..)', 'get_mut(sample.vhost.as_str())', + 'state.vhosts.len() < MAX_VHOSTS', 'try_reserve()', + 'self.dropped.fetch_add(1, Ordering::Relaxed)', + 'candidate.eq_ignore_ascii_case(needle.as_bytes())', + '2 => self.statuses.add("2xx", 1)', 'map(Counters::view)']: + assert literal in analytics, literal +assert 'let lower = value.to_ascii_lowercase();' not in analytics +helpers = read('src/proxy/upstream_helpers.rs') +assert 'preferred_encoding(&original_headers, global_compression).is_some()' in helpers +assert 'compression.resolve(' not in helpers +assert 'compression::encode(' in helpers +assert 'wrap_observed_request(body, &exchange' in helpers +for rel, case in [ + ('src/traffic/tests.rs', 'batched_worker_keeps_exact_counts_and_bounded_vhosts'), + ('src/traffic/tests.rs', 'status_top_list_preserves_common_and_extension_family_names'), + ('src/forwarding/tests.rs', 'original_url_is_omitted_only_when_unneeded_and_forwarding_is_identical'), + ('src/compression/tests.rs', 'unaccepted_codecs_cannot_be_reenabled_by_scoped_overrides'), + ('src/balance/tests.rs', 'successful_lease_response_keeps_passive_observability_and_failure_detail'), + ('src/proxy/request_id/tests.rs', 'independent_sequential_ids_keep_full_length_and_unique_values'), +]: + assert case in read(rel), (rel, case) +for path in (root / 'src').rglob('*.rs'): + if path.name == 'tests.rs' or '/tests/' in path.as_posix(): + continue + assert len(path.read_text(encoding='utf-8').splitlines()) <= 500, path +assert not read('src/cache/stats.rs').startswith('//!') +print('PASS OPT5 static source checks; Rust build and runtime acceptance still required') diff --git a/scripts/tests/rps_hotpath_034_static_test.py b/scripts/tests/rps_hotpath_034_static_test.py index a10e9d5..b72d72a 100755 --- a/scripts/tests/rps_hotpath_034_static_test.py +++ b/scripts/tests/rps_hotpath_034_static_test.py @@ -14,12 +14,14 @@ def source(path): return (ROOT / path).read_text(encoding="utf-8") -# Random request identifiers retain CSPRNG entropy, SHA-256, format and sequence. +# Random request identifiers still use the OS-seeded thread-local CSPRNG, +# preserve a 256-bit unpredictable 64-hex token and mix the per-process sequence. +# OPT5 removes the extra SHA-256 operation, not the entropy or correlation ID. identifier = source("src/proxy/request_id.rs") assert "rand::random()" in identifier -assert "hash.update(sequence.to_be_bytes());" in identifier -assert "hash.update(nonce);" in identifier -assert "request_id_hex(&hash.finalize())" in identifier +assert "sequence.to_be_bytes()" in identifier +assert "*byte ^= sequence_byte;" in identifier +assert "request_id_hex(&token)" in identifier assert "SysRng.try_fill_bytes" not in identifier # Compiled/validated target authority survives request URI construction. diff --git a/src/balance/lease_runtime.rs b/src/balance/lease_runtime.rs index 8193eac..c7298ae 100644 --- a/src/balance/lease_runtime.rs +++ b/src/balance/lease_runtime.rs @@ -40,24 +40,21 @@ impl Lease { "ok" }; let before = self.member.state.healthy(); - let failure_detail = if is_5xx { - format!("status={status}") - } else { - reason.into() - }; + let failure_detail = is_5xx.then(|| format!("status={status}")); + let detail = failure_detail.as_deref().unwrap_or(reason); let ejected = self.member.state.observe_passive_with_detail( &self.health, &self.passive_health, reason, reason, - &failure_detail, + detail, elapsed, ); if let Some((first, failures)) = ejected { tracing::warn!(backend = %self.member.definition.id, origin = %self.member.backend.origin, source = "passive", reason, first_error_class = first.class, first_error_at_ms = first.at_ms, first_error_detail = %first.detail, - trigger_error_detail = %failure_detail, failures, + trigger_error_detail = %detail, failures, "backend excluded from traffic"); } let after = self.member.state.healthy(); diff --git a/src/balance/tests.rs b/src/balance/tests.rs index 216f44b..1379a41 100644 --- a/src/balance/tests.rs +++ b/src/balance/tests.rs @@ -807,4 +807,19 @@ mod tests { state.observe_passive(&active, &passive, "ok", Duration::from_millis(2)); assert!(state.healthy()); } + + #[test] + fn successful_lease_response_keeps_passive_observability_and_failure_detail() { + let pool = capacity_pool(0, 0); + let selection = select(&pool); + selection.lease.response(200, Duration::from_millis(5), true); + let state = &selection.lease.member.state; + assert_eq!(state.last_status.load(Ordering::Relaxed), 200); + assert_eq!(lock(&state.health).passive_result, "ok"); + assert!(lock(&state.health).latency_us > 0); + selection.lease.response(503, Duration::from_millis(5), true); + assert_eq!(state.last_status.load(Ordering::Relaxed), 503); + assert_eq!(lock(&state.last_failure).detail, "status=503"); + } + } // mod tests diff --git a/src/compression/tests.rs b/src/compression/tests.rs index 7fadcbf..c323b23 100644 --- a/src/compression/tests.rs +++ b/src/compression/tests.rs @@ -56,6 +56,19 @@ mod tests { assert!(request_headers(&HeaderMap::new()).is_empty()); } + #[test] + fn unaccepted_codecs_cannot_be_reenabled_by_scoped_overrides() { + let global = Settings::default(); + let force_on = Overrides { enabled: Some(true), ..Default::default() }; + for offer in ["identity", "gzip;q=0, br;q=0, zstd;q=0", "deflate", "*;q=0"] { + let mut headers = HeaderMap::new(); + headers.insert("accept-encoding", offer.parse().unwrap()); + assert!(preferred_encoding(&headers, &global).is_none()); + let effective = force_on.resolve(&global, &global); + assert!(preferred_encoding(&headers, &effective).is_none()); + } + } + #[test] fn scoped_codec_overrides_keep_global_codecs_authoritative() { let global = Settings::default(); diff --git a/src/forwarding.rs b/src/forwarding.rs index 28c8775..40a8e00 100644 --- a/src/forwarding.rs +++ b/src/forwarding.rs @@ -214,6 +214,22 @@ impl Compiled { tls: bool, authority: &HeaderValue, uri: &Uri, + ) -> Result { + self.context_with_original_url(incoming, peer, local, tls, authority, uri, true) + } + + /// The absolute original URL is only consumed by the optional Tinyauth + /// verifier. Ordinary proxy requests need the exact forwarding metadata, + /// but do not need to allocate this extra diagnostic URL. + pub fn context_with_original_url( + &self, + incoming: &HeaderMap, + peer: SocketAddr, + local: SocketAddr, + tls: bool, + authority: &HeaderValue, + uri: &Uri, + include_original_url: bool, ) -> Result { let peer_ip = canonical(peer.ip()); let trusted_peer = self.trusted(peer_ip); @@ -222,7 +238,11 @@ impl Compiled { let authority_text = authority.to_str()?; crate::config::request_host(authority_text)?; let target = uri.path_and_query().map(|v| v.as_str()).unwrap_or("/"); - let original_url = format!("{scheme}://{authority_text}{target}"); + let original_url = if include_original_url { + format!("{scheme}://{authority_text}{target}") + } else { + String::new() + }; let peer_text = peer_ip.to_string(); let peer_header = HeaderValue::from_str(&peer_text)?; let peer_node = if peer_ip.is_ipv4() { None } else { Some(node(peer_ip)) }; @@ -269,7 +289,11 @@ impl Compiled { // Validate before inserting authority into Forwarded's quoted host field. crate::config::request_host(authority_text)?; let original_target = uri.path_and_query().map(|v| v.as_str()).unwrap_or("/"); - let original_url = format!("{scheme}://{authority_text}{original_target}"); + let original_url = if include_original_url { + format!("{scheme}://{authority_text}{original_target}") + } else { + String::new() + }; let mut forwarded: Vec<_> = ips.iter().map(|ip| format!("for={}", node(*ip))).collect(); let last = forwarded.last_mut().expect("actual peer is always present"); last.push_str(&format!( diff --git a/src/forwarding/tests.rs b/src/forwarding/tests.rs index 88d64f4..e141b56 100644 --- a/src/forwarding/tests.rs +++ b/src/forwarding/tests.rs @@ -229,4 +229,39 @@ mod tests { assert_eq!(context.values["x-forwarded-port"], "8080"); assert_eq!(context.values["x-forwarded-uri"], "/s?q=1"); } + #[test] + fn original_url_is_omitted_only_when_unneeded_and_forwarding_is_identical() { + for (trusted, peer, local) in [ + (false, "192.0.2.9:5000", "10.0.0.1:8080"), + (true, "10.0.0.4:5000", "10.0.0.1:8080"), + (false, "[2001:db8::a]:5010", "[2001:db8::b]:8080"), + ] { + let settings = Settings { + trusted_proxies: if trusted { vec!["10.0.0.0/8".into()] } else { vec![] }, + trust_forwarded_proto: true, + }.compile().unwrap(); + let mut incoming = HeaderMap::new(); + incoming.insert("x-forwarded-for", "192.0.2.15".parse().unwrap()); + incoming.insert("x-forwarded-proto", "http".parse().unwrap()); + let peer: SocketAddr = peer.parse().unwrap(); + let local: SocketAddr = local.parse().unwrap(); + let host: HeaderValue = "app.example:8443".parse().unwrap(); + let uri: Uri = "/route?q=hello".parse().unwrap(); + let full = settings.context(&incoming, peer, local, true, &host, &uri).unwrap(); + let compact = settings.context_with_original_url( + &incoming, peer, local, true, &host, &uri, false, + ).unwrap(); + assert!(!full.original_url.is_empty()); + assert!(compact.original_url.is_empty()); + assert_eq!(full.client_ip, compact.client_ip); + assert_eq!(full.scheme, compact.scheme); + assert_eq!(full.values, compact.values); + let mut full_headers = incoming.clone(); + let mut compact_headers = incoming.clone(); + full.apply(&mut full_headers); + compact.apply(&mut compact_headers); + assert_eq!(full_headers, compact_headers); + } + } + } diff --git a/src/proxy/forward_entry.rs b/src/proxy/forward_entry.rs index 978d896..7c70e1b 100644 --- a/src/proxy/forward_entry.rs +++ b/src/proxy/forward_entry.rs @@ -193,13 +193,14 @@ async fn forward_request( } } } - let mut context = match snapshot.forwarding.context( + let mut context = match snapshot.forwarding.context_with_original_url( request.headers(), peer.0, local.0, tls, &original_host, request.uri(), + frontend.is_some_and(|frontend| frontend.tinyauth.is_some()), ) { Ok(context) => context, Err(_) => { diff --git a/src/proxy/request_id.rs b/src/proxy/request_id.rs index 75db0f0..5db56dc 100644 --- a/src/proxy/request_id.rs +++ b/src/proxy/request_id.rs @@ -1,14 +1,13 @@ fn request_id_hash(sequence: u64) -> String { - use sha2::{Digest, Sha256}; - - // Thread-local CSPRNG is seeded (and periodically reseeded) from the OS. - // Do not make a getrandom syscall on the hot path for every HTTP request. - // Keep hashing the per-process sequence as before for the same ID format. - let nonce: [u8; 32] = rand::random(); - let mut hash = Sha256::new(); - hash.update(sequence.to_be_bytes()); - hash.update(nonce); - request_id_hex(&hash.finalize()) + // Same thread-local CSPRNG as before, now with no per-request SHA-256. + // XORing an increasing sequence into independent uniformly random bytes + // preserves the full 256-bit unpredictability and 64-hex-character shape. + // The sequence is still mixed in; this is not a predictable counter ID. + let mut token: [u8; 32] = rand::random(); + for (byte, sequence_byte) in token[..8].iter_mut().zip(sequence.to_be_bytes()) { + *byte ^= sequence_byte; + } + request_id_hex(&token) } fn request_id_hex(digest: &[u8]) -> String { diff --git a/src/proxy/request_id/tests.rs b/src/proxy/request_id/tests.rs index 701d2f5..c436707 100644 --- a/src/proxy/request_id/tests.rs +++ b/src/proxy/request_id/tests.rs @@ -30,6 +30,15 @@ mod request_id_tests { } } + #[test] + fn independent_sequential_ids_keep_full_length_and_unique_values() { + let ids: std::collections::HashSet<_> = (0..4096u64) + .map(request_id_hash) + .collect(); + assert_eq!(ids.len(), 4096); + assert!(ids.iter().all(|id| id.len() == 64)); + } + #[test] #[ignore = "explicit release-mode microbenchmark; not an acceptance threshold"] fn benchmark_request_id_formatting() { diff --git a/src/proxy/upstream_helpers.rs b/src/proxy/upstream_helpers.rs index 3c105a6..d17ca09 100644 --- a/src/proxy/upstream_helpers.rs +++ b/src/proxy/upstream_helpers.rs @@ -312,21 +312,29 @@ async fn observe(State(state): State>, mut request: Request, next: Next response.headers_mut().remove("alt-svc"); } let global_compression = &snapshot.envelope.config.compression; - let mut compression = global_compression.clone(); - if let Some(frontend) = frontend_context.as_ref() { - compression = frontend - .compression - .resolve(global_compression, &compression); - } - if let Some(route) = effective_route_context { - if let crate::config::CompiledSource::WebServer(service) = &route.source { - compression = service + // All scoped policies have the global codec ceiling. With no accepted + // global encoding (e.g. Accept-Encoding: identity), encode is a no-op: + // avoid even constructing/resolving the per-request settings. + if method != Method::HEAD + && crate::compression::preferred_encoding(&original_headers, global_compression).is_some() + { + let mut compression = global_compression.clone(); + if let Some(frontend) = frontend_context.as_ref() { + compression = frontend .compression .resolve(global_compression, &compression); } + if let Some(route) = effective_route_context { + if let crate::config::CompiledSource::WebServer(service) = &route.source { + compression = service + .compression + .resolve(global_compression, &compression); + } + } + response = crate::compression::encode( + response, &original_headers, &method, &compression, + ).await; } - response = - crate::compression::encode(response, &original_headers, &method, &compression).await; if let Some(encoding) = response .extensions() .get::() diff --git a/src/traffic.rs b/src/traffic.rs index 1d5a0ab..c160e9d 100644 --- a/src/traffic.rs +++ b/src/traffic.rs @@ -122,7 +122,16 @@ impl Counters { self.methods.add(&sample.method, 1); self.method_families.add(method_family(&sample.method), 1); self.sources.add(sample.source, 1); - self.statuses.add(&format!("{}xx", sample.status / 100), 1); + // Standard HTTP status families are static; the unusual 6xx-9xx + // extension codes still keep their exact original dashboard label. + match sample.status / 100 { + 1 => self.statuses.add("1xx", 1), + 2 => self.statuses.add("2xx", 1), + 3 => self.statuses.add("3xx", 1), + 4 => self.statuses.add("4xx", 1), + 5 => self.statuses.add("5xx", 1), + family => self.statuses.add(&format!("{family}xx"), 1), + } self.routes.add(sample.route.as_deref().unwrap_or("(main)"), 1); self.paths.add(&sample.path, 1); self.cache.add(sample.cache, 1); @@ -225,16 +234,27 @@ impl Analytics { let (tx, mut rx) = mpsc::channel::(QUEUE_CAPACITY); let worker_state = state.clone(); tokio::spawn(async move { + // One bounded receiver and one ordered aggregator as before. + // Batch up to 64 ready messages under one dashboard-state lock; + // never hold the lock while waiting for new messages. + let mut batch = Vec::with_capacity(64); loop { tokio::select! { _ = shutdown.cancelled() => break, - sample = rx.recv() => { - let Some(sample) = sample else { break; }; + count = rx.recv_many(&mut batch, 64) => { + if count == 0 { break; } let mut state = worker_state.lock().unwrap_or_else(|e| e.into_inner()); - if !state.vhosts.contains_key(&sample.vhost) && state.vhosts.len() >= MAX_VHOSTS { - continue; + for sample in batch.drain(..) { + if let Some(counters) = state.vhosts.get_mut(sample.vhost.as_str()) { + counters.record(sample); + } else if state.vhosts.len() < MAX_VHOSTS { + // Allocate the map key only for a previously unseen vhost. + let name = sample.vhost.clone(); + let mut counters = Counters::default(); + counters.record(sample); + state.vhosts.insert(name, counters); + } } - state.vhosts.entry(sample.vhost.clone()).or_default().record(sample); } } } @@ -268,7 +288,9 @@ impl Analytics { } total.view() } else { - state.vhosts.get(selected).cloned().unwrap_or_default().view() + // No need to copy a dozen bounded top lists just to render one + // vhost's read-only dashboard view while holding the state lock. + state.vhosts.get(selected).map(Counters::view).unwrap_or_else(|| Counters::default().view()) }; View { scope: selected.to_owned(), @@ -320,30 +342,33 @@ fn method_family(method: &str) -> &'static str { } fn user_agent_family(value: &str) -> &'static str { - let lower = value.to_ascii_lowercase(); - if lower.contains("bot") || lower.contains("spider") || lower.contains("crawler") || lower.contains("slurp") { + // ASCII-insensitive classification without copying and lowercasing a + // potentially 512-character User-Agent for every sampled request. + let has = |needle: &str| value.as_bytes().windows(needle.len()) + .any(|candidate| candidate.eq_ignore_ascii_case(needle.as_bytes())); + if has("bot") || has("spider") || has("crawler") || has("slurp") { "bot" - } else if lower.contains("curl/") { + } else if has("curl/") { "curl" - } else if lower.contains("wget/") { + } else if has("wget/") { "wget" - } else if lower.contains("python-requests") || lower.contains("python/") || lower.contains("aiohttp/") { + } else if has("python-requests") || has("python/") || has("aiohttp/") { "python" - } else if lower.contains("go-http-client") { + } else if has("go-http-client") { "go" - } else if lower.contains("java/") || lower.contains("okhttp/") { + } else if has("java/") || has("okhttp/") { "java" - } else if lower.contains("postmanruntime/") { + } else if has("postmanruntime/") { "postman" - } else if lower.contains("edg/") || lower.contains("edge/") { + } else if has("edg/") || has("edge/") { "edge" - } else if lower.contains("opr/") || lower.contains("opera") { + } else if has("opr/") || has("opera") { "opera" - } else if lower.contains("chrome/") || lower.contains("crios/") { + } else if has("chrome/") || has("crios/") { "chrome" - } else if lower.contains("firefox/") || lower.contains("fxios/") { + } else if has("firefox/") || has("fxios/") { "firefox" - } else if lower.contains("safari/") { + } else if has("safari/") { "safari" } else { "other" diff --git a/src/traffic/tests.rs b/src/traffic/tests.rs index b280612..d2c6c76 100644 --- a/src/traffic/tests.rs +++ b/src/traffic/tests.rs @@ -75,4 +75,68 @@ mod tests { assert_eq!(user_agent_family("curl/8.12.1"), "curl"); assert_eq!(user_agent_family("ExampleBot/1.0"), "bot"); } + #[test] + fn status_top_list_preserves_common_and_extension_family_names() { + let mut counters = Counters::default(); + for status in [100, 200, 302, 404, 503, 701] { + counters.record(Sample { + vhost: "test".into(), route: None, method: "GET".into(), + path: "/".into(), client_ip: None, status, + duration_ms: 1, request_bytes: 0, response_bytes: 1, + cache: "BYPASS", source: "reverse_proxy", + user_agent: None, country: None, + }); + } + let labels: std::collections::HashSet<_> = counters.view().statuses.into_iter() + .map(|entry| entry.value).collect(); + for label in ["1xx", "2xx", "3xx", "4xx", "5xx", "7xx"] { + assert!(labels.contains(label)); + } + } + + #[test] + fn user_agent_matching_is_ascii_case_insensitive_even_with_unicode() { + for (agent, expected) in [ + ("MOZILLA/5.0 ChRoMe/120.0", "chrome"), + ("żółw / uSER-BOT", "bot"), + ("CURL/8.0", "curl"), + ("wGeT/1.24", "wget"), + ("Ąęć PYTHON-Requests", "python"), + ("Mozilla/5.0 EdG/110", "edge"), + ("Random ∂x/42", "other"), + ] { + assert_eq!(user_agent_family(agent), expected); + } + } + + #[tokio::test] + async fn batched_worker_keeps_exact_counts_and_bounded_vhosts() { + let shutdown = CancellationToken::new(); + let analytics = Analytics::start(shutdown.clone()); + for id in 0..130 { + analytics.push(Sample { + vhost: if id % 2 == 0 { "left".into() } else { "right".into() }, + route: None, method: "GET".into(), path: "/small".into(), + client_ip: None, status: 200, duration_ms: 1, + request_bytes: 0, response_bytes: 1040, + cache: "BYPASS", source: "reverse_proxy", + user_agent: None, country: None, + }); + } + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while analytics.view(Some("left")).data.requests != 65 + || analytics.view(Some("right")).data.requests != 65 + { + tokio::task::yield_now().await; + } + }).await.expect("analytics should drain two batches"); + let all = analytics.view(None).data; + assert_eq!(all.requests, 130); + assert_eq!(all.response_bytes, 130 * 1040); + assert_eq!(analytics.dropped.load(Ordering::Relaxed), 0); + assert_eq!(analytics.reset(Some("left")), 1); + assert_eq!(analytics.view(None).data.requests, 65); + shutdown.cancel(); + } + }