diff --git a/AGENTS.md b/AGENTS.md index 589c3ae..fb37912 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ Read README.md, ROADMAP.md and docs/TEST_REPORT.md before making changes. The project name is RustProxy; the Cargo package/binary remains `rust-proxy` for -compatibility. Current source version: 0.32.6. +compatibility. Current source version: 0.32.7. ## Delivery diff --git a/CHANGELOG.md b/CHANGELOG.md index dbb01e4..e6f1bb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,22 @@ > Operator-facing release history. Historical engineering notes are kept in > [`docs/CHANGELOG_TECHNICAL_ARCHIVE.md`](docs/CHANGELOG_TECHNICAL_ARCHIVE.md). +## 0.32.7 + +- Handle peer-initiated graceful upstream HTTP/2 GOAWAY(NO_ERROR) by retiring a + stale connection-pool generation and retrying one empty, safe request on the + same backend. Do not replay requests with bodies or unsafe methods. +- Keep expected H2 GOAWAY and co-timed cancellations out of passive I/O error + quarantine. Count unrelated I/O errors, but de-duplicate failures inside a + brief connection-closure burst. +- Make passive ejections recover through healthy active probes or, when active + health checks are disabled, a bounded half-open cooldown. +- Emit one WARN per transition out of traffic with the first failure detail in + the counted streak and the error triggering the threshold, subject to the + existing Error log sink configuration. +- Add unit and static regression coverage. Rust compilation and live H2 + acceptance testing remain pending. + ## 0.32.6 - **Development-branch Appearance layout:** replace the long diff --git a/Cargo.lock b/Cargo.lock index 07eb6ac..66f0be9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1881,7 +1881,7 @@ dependencies = [ [[package]] name = "rust-proxy" -version = "0.32.6" +version = "0.32.7" dependencies = [ "anyhow", "arc-swap", diff --git a/Cargo.toml b/Cargo.toml index ed4cf35..c89df94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rust-proxy" -version = "0.32.6" +version = "0.32.7" edition = "2021" rust-version = "1.99" description = "RustProxy - independent reverse proxy with local administration" diff --git a/README.md b/README.md index fff1c0e..318994b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# RustProxy 0.32.6 +# RustProxy 0.32.7 **0.31.0 panel/cache additions:** browser-negotiated Zstd/Brotli/gzip for HTML, JS and CSS (Settings → Administration); optional HTTP BAN/PURGE restricted to explicit IP/CIDR allowlist (Settings → Cache, disabled by default); System information Refresh button beside the API badge. See [Settings](docs/SETTINGS.md) and [Cache](docs/CACHE.md). @@ -7,7 +7,26 @@ streaming HTTP/TLS/WebSocket forwarding, raw TCP/UDP stream hosts, IP-bound vhos **0.31.9 Certificate API management: inspect and edit credential settings without exposing/rotating the current token, or regenerate one existing token with one-time display. Rust/Cargo and live testing required.** -## What changed in 0.32.6 +## What changed in 0.32.7 + +- **H2 upstream recovery:** gracefully drained upstream H2 connections sending + `GOAWAY(NO_ERROR)` no longer poison Passive Health. Retire the affected Hyper + pool generation and retry **once**, only for empty, replay-safe GET/HEAD/OPTIONS + without WebSocket/administration tunneling. Non-replay-safe requests are not + resent and can still receive 502 if the peer closes their connection. +- **Passive Health:** collapse bursts of I/O failures within 250 ms into one + failure observation; when active checks are enabled, consecutive successful + probes can reopen a passively quarantined backend. Passive-only monitoring + uses a bounded cooldown before a half-open traffic attempt. Keep the I/O error + checkbox enabled to count unrelated, genuine I/O failures. +- **Error log:** `WARN` when a backend is first removed from traffic, including + backend ID, health source, failure reason, timestamp/description of the first + error in that failure streak and the triggering error. Requires enabling the + existing Error log sink (Settings -> Logging); when disabled, WARN goes to + stderr/journald. Access logs still show individual 502s and subsequent 503s. +- **Acceptance:** Rust compile/test and live negotiated H2/GOAWAY testing on + staging are required before production installation. + - **Appearance Dashboard layout ** Overview cards now use a full-width horizontal strip instead of a long list in the right half of diff --git a/ROADMAP.md b/ROADMAP.md index 9eda0da..14a123e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,6 +1,6 @@ # RustProxy roadmap -## 0.32.6 - Mobile Overview and upstream protocol visibility (runtime acceptance pending) +## 0.32.7 - Mobile Overview and upstream protocol visibility (runtime acceptance pending) - [x] Share counter-rate pause/resume handling between Cache, Transform, Overview, backend, Stream and Vhost plots. Retain historical points after @@ -16,8 +16,9 @@ and match their corner radius while preserving observed protocol colors. - [x] On phone-size Overview, remove the nested Network interface chart scroll region, while retaining chart SVG/text nodes on ordinary live updates. -- [ ] Confirm scroll positions with a real iOS 27 phone while live updates are - enabled, including the Process / Network diagnostics boundary. + + +## 0.32.6 - [x] Reinstate two small Overview cards per row across phone-size widths. - [x] Apply global corner styles to additional vhost/entity panel surfaces. @@ -28,8 +29,6 @@ when no fresh observation exists instead of guessing Auto's ALPN outcome. - [x] Add simulated mobile Chromium and mocked status UI regressions plus a Rust unit regression for protocol-version observation. -- [ ] Run Cargo checks/tests, real iPhone Safari scrolling and live upstream - TLS ALPN H1/H2 behavior before production approval. ## 0.32.5 - Dynamic DNS client ACLs (runtime acceptance pending) @@ -37,8 +36,6 @@ - [x] Add bounded A/AAAA periodic resolver, immutable cache, expiration and immediate refresh on published configuration changes. - [x] Add DNS/DDNS editor selector, PL/EN guidance and source/regression tests. -- [ ] Compile and test Rust with Cargo; stage live DDNS rotation, DNS outage, - mixed IPv4/IPv6 answers, vhosts and trusted-forwarded-client-IP paths. ## 0.32.4 - Upstream Host/authority parity and Overview gutters (runtime acceptance pending) @@ -48,8 +45,6 @@ - [x] Update HTTPS-by-IP guidance; preserve opt-in diagnostic H1 preset. - [x] Make Overview small-to-large row gutter consistent across density modes. - [x] Add protocol/authority and UI/source contracts; preserve WebSocket isolation. -- [ ] Run `cargo check --locked --all-targets`, `cargo test --locked` and live - tests with H1/H2 ALPN/H2C and Angie vhosts before production acceptance. ## 0.32.3 - Backward-compatible upstream modes and Overview Cards registry (runtime acceptance pending) diff --git a/SHA256SUMS b/SHA256SUMS index 7fb0f6c..83c7d35 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,13 +1,13 @@ 61bf40a0572889dc8b0dda9eae4e93e44b0de9364b3c798106c85700ffb37c23 .dockerignore 4140a2956264b44f259e6a37fc00ba02097bbe68b09296e04d3d243c1287129b .env.example 9e6356475bf587091bab72246ec9e325b498f7e48a6220e9a18baa50d0a385a6 .gitignore -a402ee07826dfab1b1d63b98c4a9f75809f056b908c92abacf0bba816d7ad133 AGENTS.md -e4a565f39f4ecca622f0eb27822c7399e45b4c4d1a2fd1f669185d53196a969b CHANGELOG.md -934ec0e7f8a307b00319606a9c5d7e91dac6449a23dd1f3abcefe4304eb4de06 Cargo.lock -8a7606650ce03f1faf36c893a17ccf1c9e8da4259e0248ac2c9bf030a64872d1 Cargo.toml +c38904ea10c12fea702f54dbbcad75096a4e89148532dea040a346ecc18c3b8c AGENTS.md +77a1df6e671051d8bf6640401e162b319083c8e6cf83cf50edbb6ce0893a8ba3 CHANGELOG.md +5d8c6feb5585b8880f7d016dec83f2a8f6cdcbaa7eb5ccc840a6064385d22133 Cargo.lock +931e433a8f2e2582bdd76180c60c6c40978a3bb29f66bbd5219a45587a052d95 Cargo.toml 25c7f3c8162cbf5cc156439774118a81dae9fdc52b611ec7e40c230fc99a772c Dockerfile -0f8cedf595f57f96644e8d4dc608b43e6ebd9a73b31157a210d624492a6c5206 README.md -da519345b2bce492e747c268df2730971543d1bdb198c3e854671a7d107e5609 ROADMAP.md +2425ba362f88a656516d04c40866e1a43f520a64e5a2c4020e4c7732742229fc README.md +7e715a1170780ffeeca83447fda739e4be84b96e9c2a51851175b62f79000478 ROADMAP.md da8311e3db945290666324a2f24bf4521b365272d7ac3f862ded1be993eea646 UPDATE_CHECK.toml f645f218dd062bf6cb54a8ee634290562cad364a4cccf09f16e3c8303582af8c build.rs ea8d091843f86f4c66a5c76b20268b2feb397070851b6e05c5bc7d7a8d9157eb deploy/README.md @@ -63,7 +63,7 @@ ec7e32fd19e6ba6ed1732f1f3dd92ac0860609e8c0bbd3c1998840cdfb24bf54 docs/SECURITY. 007537ee971463072b2482dfa4fdcf1955f837ced0b711d4eef3b6267d16d4f6 docs/SECURITY_AUDIT_0.31.0.md a41cef4a4da5e098fe3e9dfa0fffb3625f2e09792eacdf5dbf8d8fd532146d26 docs/SETTINGS.md 8fc112cb45a2f35ef5b0dd4371aa6f43aca4df85db62c6ae02e56d74a3b702ae docs/STREAMS.md -130befc0f8216214567dd4b2df2574ea751b56b22c8f2cc3ba48e8d0aaccc3b5 docs/TEST_REPORT.md +d420880a9a4b3d2fcd0458384aca7fd022afb6fca9abfa2b7ac2a5f32f916822 docs/TEST_REPORT.md cfb452d8496839e832b5fbacdd4e52525b0ecc3112fbb62340c1e580bd5c1717 docs/TODO.md 751c9b13b47d0959a8575d6e82bdecba988b950a374ba0b084fbba08b4b65ebe docs/TRANSFORM.md fe3ec6ca16ff933239c12dacf39dd7b45717c0551a8b9f6ce601bf78136c9e37 docs/TRANSPORT.md @@ -156,7 +156,7 @@ c6b552cd5acd9e743c28b1a5564420c214c619fe891ab0561b7d4499aae8d6c0 src/admin/file c720a7cad6a15c57b1f2ae7b3e5fbf30a97c64afb9137915f37b23f4e576bdc5 src/admin/file_logs_query.rs 74636c6560618ee95757335452baf8db03394e1ad18acd54252f54b15746b8d5 src/admin/logging_api.rs 33be50109497d0adb23c36a10894acb5508dd38f3c01136bd0438a4ce0ffe74b src/admin/mod.rs -9426fba372656cb5abcc76e88e3885d1e9abc7ecf04653abc8c31803aeece019 src/admin/openapi.json +ba3fe90521efd976dfeed3c353fed05e335a35e1c38842a31e7429802a22ae23 src/admin/openapi.json d9120f7dc61d27621c5ac27ea49815dc7945c44d5b99d208da503818c198f649 src/admin/operations.rs 30895e05c5d5d0a26be29f667e0f6d72f16a96a5c8a98e09cd11d8ef2d7b4763 src/admin/panel_assets.rs 75bbedd54fdabb2b489642a642ee7c6533f69cdacb85e251763a56ba11174d5c src/admin/panel_content.rs @@ -246,13 +246,13 @@ c96a87e82ff0c1c9ce3d0cd5135903bb53af78d1c4fb6ebfe5fea503bf5644ea src/auth/store a00571b409d305af86ab2a57a25434b5ab58706772df440d017fc1fec097f7b4 src/backup/tests.rs f5afae218f3a6f91bc2f1f5a90094630ce5b7ca574de61807b5a317b13937ec2 src/backup/validation.rs 426a946b2c9e244f09f6cbc5f9664b8ad0737529a9bb5a3d549d27196a1f8043 src/balance.rs -3d4b3986f54bed99212063b6d135d40a202d556a3e4d253e24c74be4cb9e8ebc src/balance/health.rs -a1e5d43f5417d71f3974812cb539c0932f206e2290090917f067494f3587fc72 src/balance/lease_runtime.rs -4d3aa415af2fbb42a4e19f2ca10be8b1d1ad42f366a013cd9c33806abc1fbea2 src/balance/model.rs +f4014258dfa2b0e5c05daa220203d7536f0ae58343d82e0be7613f138edf75af src/balance/health.rs +7a59ded466173e67ee8967bdf50d667b66a9db33a6770c03aeee58d7c482c0c1 src/balance/lease_runtime.rs +aed9ed896a122d7adcbedf0897cfd5acf62be3c5b63a71a4a425c8fe8474c0eb src/balance/model.rs 41c3c020a60c372dbf965bd277c330dd9cf2ab079f6608b472adf72a656f4cb3 src/balance/pool_core.rs 477132dd381e23f1be197c32ff58a0925f41d3e6aa63fe77c10973cefb2b76f6 src/balance/pool_selection.rs 8abb6048303ee8c022546a528d5693167766e7450efd8e20f37ec6fd0fa9ef5e src/balance/pool_view.rs -656a6ab3c9855f92b7e3e0dd6de1bcc408da003db8bcfcda52b71c5234c56693 src/balance/tests.rs +0a510955e55894c0a27679d0e8b80864924eb15278c8a70ba4f3ac0e6eae45ff src/balance/tests.rs 5049d2bd01a9c866f3bad2288d8d594df96bba6045dfe15cd22b4173562b7388 src/basic_auth.rs 9a8858b7ff461a4d961ba701ca3ed6eaec15c3f7f05eef8e0e30edc63617605a src/basic_auth/tests.rs 7def03ace07dc38a4806277d75b6ce673d33ddaaf1c4855fa0a3372be1cf9489 src/cache/cold.rs @@ -366,21 +366,22 @@ b5856fd38a957b2ca9c7e3606d84d13d5cfcf3a45b487b15ae1d2dc248cbda57 src/proxy/body aa49c9928bb91e3d5bae0d340d88396a0417b9edec6192d41e1403533319a897 src/proxy/cache_http_invalidation.rs 771cc1a38927f96e593b144dbab66cd64a5dc0ed42d54b1615c359845b2b7223 src/proxy/error_fallback.rs bf2e6bbd64fb45fceed54af0bff1c1e00da22b79c64dd8f5c23d2af0dbab0b7d src/proxy/forward/cache_lookup.rs -526592b613fbe20f532f11be5a3236acaeb8cad400f70e1d28cc1c2dfc126c52 src/proxy/forward/response.rs +2e76bb3ed7faf673208cac952385d8aec3b6d294dc06c9d5306aee2dca155b3a src/proxy/forward/response.rs 3b220cfa474325bfe7853d62daf2c71059d0dc582ac749fe61e2864094545d63 src/proxy/forward/route_cache.rs cea9e47ecb31b0e4c0bbe0eb83e0e53b5e7bdcb9930107b67a8c34eef01a3f49 src/proxy/forward/success_finish.rs 902cbf8dc3c1ec8d25411bc7fe0400d301f3a7cc885afada594e4eaee3bedd08 src/proxy/forward/success_prepare.rs -5d8c6115321c7ff4c76d26c68ab209d27bd8898d10cbb903193313bf30f7f93d src/proxy/forward/upstream.rs +ce782dce2f7fe65ef93d08f88eb8277162ad9b975c1391da9c45869511196621 src/proxy/forward/upstream.rs b05c4f5c009c5e410f472ceaa999fa3a9c7fb60364d5b62de219f0924c9d524e src/proxy/forward/web_source.rs a9c011061585c067f04f7885215d53e6c40474bdced15d2be5a5ba65190910a0 src/proxy/forward_entry.rs +31403d08d3f083118a95954d7ddb5144387b28a6a3336e6057721419d53356c9 src/proxy/h2_graceful.rs c438066616527a3108f2ad53e3698ae9b2a80961714394cdd0a450c92f99d992 src/proxy/headers.rs 5cb7522ceb9a9f083a15596ac4898559295e38d1138c80099577211de8a868ec src/proxy/headers/tests.rs 8d17b97d90fd6a749ba8d37a8d28c94e6d7302b6c86905f073e2dc595506961a src/proxy/mod.rs 55133ceace2e5262d9c54fa41bea099a27b817d1e09c1cd0e2493fd9cee4da38 src/proxy/redirect_helpers.rs -9f2790824594d3b29d5a692407fba75e3501cafcb017b79ec0ed3e2320432b96 src/proxy/tests/authority_tests.rs +ad701033f005c9c8c4b160f9b9b97bcabc17e936d8ade273a111937c483a125a src/proxy/tests/authority_tests.rs 0f2eacadf1447db8aa1c1a78005aa262e0f747096bfe7e0a0e41a2cd31a0d17f src/proxy/tests/h3_navigation_tests.rs d134b0eb617f82f9971b3f3461379ea37c57bd0079bf54748d0c434b2ef1b475 src/proxy/tests/http_invalidation_tests.rs -c1280b05a31d8880bb28d043cd3d1888cccc10bbfe74ba0edf4360f46770f9b3 src/proxy/types.rs +668a95def1865ddddcfb859f6dcfe70905e2c63d422268af32b4bdb5cd46d559 src/proxy/types.rs 9999f717492755d62fc05cd5c1aff802ff0da901fe3e6b928bd8a000833c8799 src/proxy/upstream_helpers.rs 2d9142e3ae95895b5271b4009ab48236a3a2074ce51179c021193b35777560e4 src/proxy/web_cache_revalidation.rs 5af991311e4ed22cd8aaff15a4b9d45bb55fcdfcd0ebbfb4879dc1c7edf88ac4 src/proxy/web_cache_revalidation/tests.rs @@ -451,8 +452,8 @@ a4e1d2aa2913587c250f251356431097b76995aee6e8bf9e35c4978a95491de1 src/transform/ 8ed4baeaf79edfa2a84912cc00f1d13c564e1b997efe6888d2e4b5a16aaa8905 src/transform/tests.rs 6146c59f36cf419623747d497b3de685da6c84964df43317bebdcb21deb9ee09 src/update_check.rs c42d000d1d576721af3ffe930589388693001b8113554a7658f6dcbc665b5d1b src/update_check/tests.rs -9c897664a8b3b288b761b84a724b212c3f6a82b856ceebf3b4ecd6eab1846a35 src/upstream.rs -9f05552d0cdc523e37bc73214e35e472f271250fd4f95ddbbefda4c39f96ebd9 src/upstream/client.rs +14eadcadc7934831193a3e92a3c540817aaeb1ad239b8ca704127be1c728c61b src/upstream.rs +2cb9098bce6aad7aa341d7d5bf6bbb9e816626c8ed24107e9587bdeec983fc9f src/upstream/client.rs ad29f3fc15c9b627b5d752af5adc9f99ee1364fec2df63f1e4509cb223988efb src/upstream/client_tests.rs 75fef0a488d3f3dabfa03c2eabc5995442dfb3c9a1fde8eb6653a124dda25d43 src/upstream/protocol_tests.rs 8a112154b9631f62a54584302a3ff6a64e4f177ce79aba678b4cb23e466c6ea3 src/upstream/tests.rs @@ -504,23 +505,23 @@ f3dba206c60cd2e6fcc5f4a2c8d652ccf47362694448312cc1f8dc4112d93e7e tools/checks/b e82765ae946461952e68369f0b16f0a99f01e625bd61d2551513504f7b2f4d46 tools/checks/backend_dashboard_0323_ui_test.mjs 45cd21192248ffe6f66c20bab03da1a5ce299f040883764d09a640859e19924d tools/checks/backend_health_0300_ui_test.mjs a135c9456143975279f18233caab44b8c64f5025ef4ceeb1f2ad22090b588019 tools/checks/backend_protocol_0326_ui_test.mjs -a1fb118e3a32819689cb607d574b0060ca29ff1f6a68c505231a4e88ea93b5de tools/checks/certificate_api_0319.py +72e20f15a029d1a3596c7d05979354eb78dacd1de979f6933aed0994cfe76179 tools/checks/certificate_api_0319.py 526d16db9de7be414a64483a1e9276d4a4402e3c7efd2479f79d01cc5b388f46 tools/checks/certificate_api_0319_ui_test.mjs -71cf48ba645ae2b7a95c73faa362d12942553e37d79680b796edf29c1484d9eb tools/checks/check_source.py +562774d341ea63390399553b2f91f7d48536b2ace504aed27afd131e64b07787 tools/checks/check_source.py a1ac2ab24958c6e8527c7797f224f94e765abd7173d4e5085a64362b193112c1 tools/checks/dashboard_appearance_ribbon_0326_browser.py 0ab28370c06b2258321e20d27d63098c339fd9dc3b6f62aef6f5266c81ff28b5 tools/checks/dashboard_appearance_ribbon_0326_ui_test.mjs dc4df51fc15d4b6345c6625890ddbe014103851ebfa16abc583fbb0d809f4015 tools/checks/dashboard_boot_0311_ui_test.mjs 7508f232a9e48049002dd4c9f34f9737bc92d304c51219f50430578dbc721855 tools/checks/dashboard_inline_layout_0323_ui_test.mjs dba768a42247cdd01863a032cf7ba223a52240285e70b0f7a0abd4a2e68c5fa1 tools/checks/dashboard_layout_editor_0323_ui_test.mjs 259d4b0985c82b631e151a07e8ef3b4c760019cb99bb2af458ce577497f7df58 tools/checks/dashboard_startup_0310_ui_test.mjs -7b119f62f11d65600b19010d600682955afe269d798df15676b397f70ffaa37d tools/checks/dependency_upgrade_0312.py +4be5f99949f3fb51fcaadf04be36a75da1f94b8aa6e081a29fd9c0ba6c4682ea tools/checks/dependency_upgrade_0312.py 38e35bed4faf984932b39bb5a00819fadc664387e8050a803ff4b5348b31369f tools/checks/docker_startup_0318_ui_test.mjs -6ca752ed5e767dc1eed4e1d7b60270391e1dd0bd5c887de8fd0f9dcaf219d7bd tools/checks/docs_check.py -48f0901a709f415f518d2a5eeb779180ac18c879e18f1ad7af748b1be9ac92e1 tools/checks/dynamic_acl_0325.py +d56f40095c4f51c57ef7456dd15c63c134c46a79f6ab9835848a05422b484eb3 tools/checks/docs_check.py +da9e0128adfb45d4f18bc9f691ebc42c4c2f449a5b3f6728bb83f65d7d3c5fd8 tools/checks/dynamic_acl_0325.py 9c77764561b6ebfeecbc228f43ea29a0dd9f44121ad773c2e4299b92758e45c2 tools/checks/dynamic_acl_startup_0325_ui_test.mjs 5bf26d9c883704e5ac85670bbcb7fb0bbad5d01779ab50407ff4fd5f3022a9cb tools/checks/file_logs_tabs_0315_ui_test.mjs 72f3c4e3274d33e02d46646395bc0cd18a8f723f8747969fc81997c7a7259e76 tools/checks/h3_cache_mobile_0315_test.mjs -c081925edadee3ffb21867765fcd08d6a536093999ee99c5590b1d5980f8e4b3 tools/checks/h3_navigation_0316.py +5159c1a2ee8610f9646e425e5f1a5d093842fe58859aa87fe94e5c2def9c5042 tools/checks/h3_navigation_0316.py e33a9cc51d58bc92a637c0e6a1c88cbd24ba4605be55748212e296c3c35a4060 tools/checks/high_traffic_memory_model.py fb8619444edf81afa604194f750dc76335394a73c74324eda59e392874f29ee8 tools/checks/ios_scroll_0326_browser.py 4f7821e7e25b09ce251d25a0b3fb4d8ce8396c36a5ed971c2637aeb325425909 tools/checks/langs_check.py @@ -537,28 +538,29 @@ b565aef4594265350240c24067a554f91fd605d0e920c03eef0b5b62c3934aad tools/checks/o f450e97dccf75c24e228e696f21ed4ae826a823ea4edde2429894a6b6b36eaab tools/checks/overview_uniform_grid_0323_ui_test.mjs 644ca805fd7072a3087cfb8c4a41fc53eefe0e902e5d1524d8f892657970493f tools/checks/overview_wide_masonry_0323_ui_test.mjs 1ac812929c2a8c0699ae3bdebd0d7e05b5bcaf86c87533bb669b5ba7955e3245 tools/checks/panel_cache_http_0310_ui_test.mjs +8ff6d76ba2465794135b532358ec6fe448928293adcc64fffa3cd917f5c11662 tools/checks/passive_h2_0327.py eea7cdc9c7d8dd0f5fb079cfff657a6feadfc0b856f63265bfbd8257f1beca19 tools/checks/preflight.py -cdd59fdc84eea535584a52143e0c69285c76dfcc6d02587cd707aa4a5f220ca3 tools/checks/preserve_host_0324.py +cd3e039edc20b7baa558eb131e227827edc04a0155cdb85ac456cd3a5aa378fa tools/checks/preserve_host_0324.py 3352941648add6d8a8044c0b5cad6bf264546d38c884a5beed64cd95addd850c tools/checks/preserve_host_0324_ui_test.mjs 64984e55d93617ac61c83f19b91a2f70402057a47a1962dcbcfd0b7d904f6098 tools/checks/rate_pause_0326_ui_test.mjs 599e0fa12fe95c7087a25801a26a590bbb473f24f038e0be157449daecec5593 tools/checks/refresh_0310_ui_test.mjs c11583dad0899f8b945cae6e2f6cb0bdb55f5526bd5f42c248ac41badec83679 tools/checks/release_0317.py b84b66a6f8f63cd0a354ef607b96ff60c77ef37f80f49ea9a9036b33147c0b6a tools/checks/release_0317_ui_test.mjs -c122954d9ce7165dfef1ce3e71a786b7bc86e9076f66d9a87baf14d269392757 tools/checks/release_0318.py +333d4974a08c8d86155522a24037bdb9a21e578b32770cf0fcff2f3192209775 tools/checks/release_0318.py e73afc62214e9092c180730f624a2039ac3fc850dcd865b2b117c11f7caaf88b tools/checks/release_0318_ui_test.mjs -9fe94e73c3156edb5015a1c03021c76ac96535e0dfc56b5efdab17fcd197955c tools/checks/release_smoke.py +c917a0b0e53178b25e0cb7d73c98d4add58533a09514b814b18037edba11b257 tools/checks/release_smoke.py 31b31e976962e2c53e0f1f71bf67f7d2686d30cb26f638defa1bd7636cf1c1e8 tools/checks/rust_module_contracts.py ecfe0bb21a7549225a19b0846bb6de8c99cbfbbf5a706d33cf84db2c7146abed tools/checks/rust_source.py adf84a983e0dbba228e47d71ec2d5c9964f7ca9989d5139de58f96ee961b035a tools/checks/security_contracts.py 23b9250245e6bec5ef728c73cee4c88f64924afc3597f8a9b58a1924235df98e tools/checks/security_dependency_contracts.py -db8881e46622ca2dbab4c29df003020006c611cd7fe087878c3e9c930cdd9853 tools/checks/security_logs_0315_ui_test.mjs +b7ac88b96fae3b897576554e1a330b38cf3dbdcae54d4d239a207a240fd19eb4 tools/checks/security_logs_0315_ui_test.mjs 011bbe65dae072999a3951aba3d74a520926abf836b784cdb2395023d2942867 tools/checks/source_module_contracts.py 37eebe90c0a60f7d7f4fbdd09986171b7dbd50c4d3760d94c31595dd93a2e680 tools/checks/source_pruning_0317.py be78df1e438d76be83e15398b310c9843ae5965b0ec6b41279b3a9650c313d8a tools/checks/stream_0320_ui_test.mjs 6c7d20fa2abb55de997c19abd07ec1afa8d5121c91e07f231976bcdfbfbba778 tools/checks/stream_0320_visibility_regression.mjs 8542f5c668155aa9f70de11e6fdaf64c5a80629a15ecc5f1d8c40f3d655f9b55 tools/checks/system_info_0301_ui_test.mjs 543a488951ea6ec8fe06e04f168ccc4d4f2af3cfc281c9a728ab7bd87377fa98 tools/checks/system_info_no_subprocess_contracts.py -e650225258168eeb9623694c6684f9a5ace92ca94debdff2558b5c77f7c998bc tools/checks/upstream_protocols_0323.py +9aa7bc1018291613209f529fbf1a303481efa1df68abb77c6deee31c84a36ce0 tools/checks/upstream_protocols_0323.py a5105ab53de67e6b1114974c92b019014e1aa93acc445bc5dd6c82af0b0d3a16 tools/checks/version_contracts.py 161637cb5e2e613e3c71cf8489cc936fa84913a2a02649463f8893a28a674242 tools/checks/vhost_origin_errors_0297_ui_test.mjs bdc20ccb7ee8c67eab6ef22b2011a99079d69fea49d03ab634da7de1b1e7b909 tools/checks/vhost_protocol_badge_group_0326_browser.py diff --git a/docs/TEST_REPORT.md b/docs/TEST_REPORT.md index 0e42dc2..d4895bf 100644 --- a/docs/TEST_REPORT.md +++ b/docs/TEST_REPORT.md @@ -1,4 +1,36 @@ -# Verification report - RustProxy 0.32.6 +# Verification report - RustProxy 0.32.7 + +## 0.32.7 graceful upstream H2 close and backend health (pending runtime acceptance) + +- Source changes: restrict GOAWAY(NO_ERROR) classification to Hyper's peer- + initiated debug variant. Recycle exactly the observed upstream pool generation + and replay only one empty GET/HEAD/OPTIONS request. No unsafe/body replay. +- Correlate immediate cancelled `connection closed` events with a recent + graceful GOAWAY within one second; all other I/O errors retain Passive + Health counting. De-duplicate I/O failure observations within 250 ms. +- Passive-ejected backends are readmitted after consecutive healthy active + probes (minimum two) or, with active monitoring disabled, after a cooldown + and a half-open request. Active failures still block routing. +- On an actual healthy-to-unhealthy transition write WARN with the first + counted error in the streak and the threshold-triggering error. The existing + Error log must be enabled for these events to be persisted in `error.log`; + otherwise WARN remains visible in stderr/journal. File-sink writes are + best-effort and may fall back to stderr on backpressure or I/O failure. +- Checks executed for this archive: `tools/checks/passive_h2_0327.py` + **PASS** (source assertions), `tools/checks/docs_check.py` **PASS**, + `tools/checks/release_0318.py` **PASS**, ZIP CRC and SHA256SUMS + verification **PASS** (579 checksummed source files). These do **not** + compile or execute RustProxy. +- `./dev.sh check` returned 127 (Cargo unavailable). `./dev.sh test` + stopped on a stale `preserve_host_0324.py` CSS-gap assertion; the + unchanged 0.32.6 source fails that same test. `check_source.py` also + stops on a pre-existing Revision History dialog CSS selector assertion + in **both** unmodified 0.32.6 and patched 0.32.7 trees. Neither is + evidence of a regression in this patch, but the complete test gate + remains **open**. +- **Not yet production-accepted:** no Rust compiler exists in this authoring + environment. `cargo check`, `cargo test`, `cargo fmt` and live Nginx H2 + GOAWAY/HTTP1.1 ALPN behavior must be verified on a Rust-enabled host. ## 0.32.6 development follow-up — compact Appearance Dashboard layout diff --git a/src/admin/openapi.json b/src/admin/openapi.json index e8f4d52..798ea0a 100644 --- a/src/admin/openapi.json +++ b/src/admin/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.0.3", "info": { "title": "RustProxy Administration API", - "version": "0.32.6", + "version": "0.32.7", "description": "Administration API for the active panel origin. Sign in through the panel; the same-origin HttpOnly cookie authenticates all private operations. Mutations also require the exact Origin, X-RustProxy-Request: 1 and a session-bound X-CSRF-Token. Loopback HTTP is the default; Settings can enable HTTPS before using a non-loopback administration bind. This document contains no live configuration or credentials. Object schemas marked extensible are summaries: full configuration validation is authoritative in Rust." }, "servers": [ diff --git a/src/balance/health.rs b/src/balance/health.rs index 213eee2..a80dbdd 100644 --- a/src/balance/health.rs +++ b/src/balance/health.rs @@ -46,6 +46,12 @@ impl BackendState { }, passive_unhealthy_source: "none", passive_unhealthy_reason: "none", + passive_first_failure: None, + active_first_failure: None, + passive_probe_successes: 0, + passive_ejected_at: None, + passive_cooldown: Duration::from_secs(5), + last_io_counted_at: None, }), traffic: Mutex::new(Traffic { at: now, @@ -105,7 +111,29 @@ impl BackendState { } } pub fn healthy(&self) -> bool { - lock(&self.health).healthy + let mut h = lock(&self.health); + // Passive-only monitoring has no active probes to recover an ejected + // backend. After a bounded quarantine, allow one new traffic attempt. + if !h.passive_healthy && h.active_healthy && !h.checked { + if h.passive_ejected_at + .is_some_and(|at| at.elapsed() >= h.passive_cooldown) + { + let previous = h.healthy; + Self::clear_passive_ejection(&mut h); + self.finish_transition(&mut h, previous); + } + } + h.healthy + } + fn clear_passive_ejection(h: &mut Health) { + h.passive_healthy = true; + h.passive_failures = 0; + h.passive_successes = 0; + h.passive_probe_successes = 0; + h.passive_first_failure = None; + h.passive_ejected_at = None; + h.passive_unhealthy_source = "none"; + h.passive_unhealthy_reason = "none"; } fn weight(&self, configured: u16, slow_start: u64) -> f64 { let h = lock(&self.health); @@ -212,12 +240,27 @@ impl BackendState { h.last_probe_ms = crate::logging::now_ms(); h.result = result; if result == "ok" { + h.active_first_failure = None; h.failures = 0; h.successes = h.successes.saturating_add(1); if h.successes >= settings.rise { h.active_healthy = true; } + if !h.passive_healthy && h.passive_ejected_at.is_some() && h.active_healthy { + h.passive_probe_successes = h.passive_probe_successes.saturating_add(1); + if h.passive_probe_successes >= settings.rise.max(2) { + Self::clear_passive_ejection(&mut h); + } + } } else { + h.passive_probe_successes = 0; + if h.active_first_failure.is_none() { + h.active_first_failure = Some(FailureObservation { + at_ms: crate::logging::now_ms(), + class: "active_health", + detail: result.into(), + }); + } self.active_health_failures_total .fetch_add(1, Ordering::Relaxed); h.successes = 0; @@ -248,6 +291,10 @@ impl BackendState { self.finish_transition(&mut h, previous); } + fn first_active_failure(&self) -> Option { + lock(&self.health).active_first_failure.clone() + } + fn record_failure(&self, class: &'static str, detail: &str) { match class { "dns" => { @@ -303,37 +350,69 @@ impl BackendState { reason: &'static str, elapsed: Duration, ) { + self.observe_passive_with_detail(active, passive, reason, reason, reason, elapsed); + } + + /// Returns the first failure in a consecutive counted streak only when + /// this observation actually removes the backend from the traffic pool. + fn observe_passive_with_detail( + &self, + active: &HealthCheck, + passive: &PassiveHealth, + reason: &'static str, + class: &'static str, + detail: &str, + elapsed: Duration, + ) -> Option<(FailureObservation, u16)> { self.record_passive_observation(reason); - { - let mut h = lock(&self.health); - Self::update_latency(&mut h, elapsed); - h.passive_result = reason; - } - if !passive.enabled { - return; - } let mut h = lock(&self.health); + Self::update_latency(&mut h, elapsed); + h.passive_result = reason; + if !passive.enabled { + return None; + } let previous = h.healthy; if reason == "ok" { h.passive_failures = 0; + h.passive_first_failure = None; + h.last_io_counted_at = None; h.passive_successes = h.passive_successes.saturating_add(1); if h.passive_successes >= passive.rise(active) { - h.passive_healthy = true; - h.passive_unhealthy_source = "none"; - h.passive_unhealthy_reason = "none"; + Self::clear_passive_ejection(&mut h); } self.finish_transition(&mut h, previous); - return; + return None; } if !passive.counts(reason) { - return; + return None; + } + // A fan-out of failures from one expired H2 connection must not count + // as independent failed backend observations in a single instant. + if reason == "io_error" { + let now = Instant::now(); + if h.last_io_counted_at + .is_some_and(|last| now.duration_since(last) < Duration::from_millis(250)) + { + return None; + } + h.last_io_counted_at = Some(now); } self.passive_health_failures_total .fetch_add(1, Ordering::Relaxed); + if h.passive_failures == 0 { + h.passive_first_failure = Some(FailureObservation { + at_ms: crate::logging::now_ms(), + class, + detail: detail.chars().take(1024).collect(), + }); + } h.passive_successes = 0; + h.passive_probe_successes = 0; h.passive_failures = h.passive_failures.saturating_add(1); - if h.passive_failures >= passive.fall(active) { + if h.passive_failures >= passive.fall(active) && h.passive_healthy { h.passive_healthy = false; + h.passive_ejected_at = Some(Instant::now()); + h.passive_cooldown = Duration::from_millis(active.interval_ms.clamp(5_000, 30_000)); } let source = match reason { "connect_error" => "passive_connect_error", @@ -352,6 +431,13 @@ impl BackendState { } } self.finish_transition(&mut h, previous); + if previous && !h.healthy { + h.passive_first_failure + .clone() + .map(|first| (first, h.passive_failures)) + } else { + None + } } } #[derive(Debug)] diff --git a/src/balance/lease_runtime.rs b/src/balance/lease_runtime.rs index 6a80d09..4b266ac 100644 --- a/src/balance/lease_runtime.rs +++ b/src/balance/lease_runtime.rs @@ -40,9 +40,26 @@ impl Lease { "ok" }; let before = self.member.state.healthy(); - self.member - .state - .observe_passive(&self.health, &self.passive_health, reason, elapsed); + let failure_detail = if is_5xx { + format!("status={status}") + } else { + reason.into() + }; + let ejected = self.member.state.observe_passive_with_detail( + &self.health, + &self.passive_health, + reason, + reason, + &failure_detail, + elapsed, + ); + if let Some((first, failures)) = ejected { + tracing::warn!(backend = %self.member.definition.id, origin = %self.member.backend.origin, source = "passive", + reason, first_error_class = first.class, + first_error_at_ms = first.at_ms, first_error_detail = %first.detail, + trigger_error_detail = %failure_detail, failures, + "backend excluded from traffic"); + } let after = self.member.state.healthy(); if before != after { tracing::info!( @@ -62,6 +79,11 @@ impl Lease { self.member.state.errors.fetch_add(1, Ordering::Relaxed); } self.member.state.record_failure(class, detail); + // GOAWAY(NO_ERROR) is a graceful H2 connection shutdown, not an + // origin failure. It can still be surfaced in access diagnostics. + if matches!(class, "h2_graceful_goaway" | "h2_graceful_close") { + return; + } let passive_reason = match class { "connect" | "dns" | "tls" => "connect_error", "timeout" @@ -72,12 +94,21 @@ impl Lease { _ => "io_error", }; let before = self.member.state.healthy(); - self.member.state.observe_passive( + let ejected = self.member.state.observe_passive_with_detail( &self.health, &self.passive_health, passive_reason, + class, + detail, elapsed, ); + if let Some((first, failures)) = ejected { + tracing::warn!(backend = %self.member.definition.id, origin = %self.member.backend.origin, source = "passive", + reason = passive_reason, first_error_class = first.class, + first_error_at_ms = first.at_ms, first_error_detail = %first.detail, + trigger_error_detail = %detail, failures, + "backend excluded from traffic"); + } let after = self.member.state.healthy(); if before != after { tracing::info!( @@ -326,8 +357,18 @@ async fn probe(member: Arc, settings: HealthCheck) { }; let before = member.state.healthy(); member.state.observe_active(&settings, result, elapsed); - if before != member.state.healthy() { - tracing::info!(backend = %member.definition.id, healthy = member.state.healthy(), reason = result, "backend health changed"); + let after = member.state.healthy(); + if before && !after { + let first = member.state.first_active_failure(); + tracing::warn!(backend = %member.definition.id, origin = %member.backend.origin, source = "active", + reason = result, + first_error_class = first.as_ref().map_or("active_health", |f| f.class), + first_error_at_ms = first.as_ref().map_or(0, |f| f.at_ms), + first_error_detail = %first.as_ref().map_or(result, |f| f.detail.as_str()), + "backend excluded from traffic"); + } + if before != after { + tracing::info!(backend = %member.definition.id, healthy = after, reason = result, "backend health changed"); } } /// One supervisor, at most 16 probes, at most one outstanding probe per member. diff --git a/src/balance/model.rs b/src/balance/model.rs index 9da2b48..b850774 100644 --- a/src/balance/model.rs +++ b/src/balance/model.rs @@ -355,7 +355,14 @@ struct Health { active_unhealthy_reason: &'static str, passive_unhealthy_source: &'static str, passive_unhealthy_reason: &'static str, + passive_first_failure: Option, + active_first_failure: Option, + passive_probe_successes: u16, + passive_ejected_at: Option, + passive_cooldown: Duration, + last_io_counted_at: Option, } + #[derive(Debug)] struct Traffic { at: Instant, diff --git a/src/balance/tests.rs b/src/balance/tests.rs index 77eb198..3df242a 100644 --- a/src/balance/tests.rs +++ b/src/balance/tests.rs @@ -1,15 +1,15 @@ mod tests { -use super::*; -fn settings(mode: HealthMode) -> HealthCheck { - HealthCheck { - mode, - fall: 2, - rise: 2, - ..Default::default() + use super::*; + fn settings(mode: HealthMode) -> HealthCheck { + HealthCheck { + mode, + fall: 2, + rise: 2, + ..Default::default() + } } -} -fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc { - let cfg = serde_json::from_value(serde_json::json!({ + fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc { + let cfg = serde_json::from_value(serde_json::json!({ "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]}, "vhosts":[{ "id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]}, @@ -17,627 +17,750 @@ fn pool(algorithm: Algorithm, sticky: StickyMode) -> Arc { {"id":"b","url":"http://127.0.0.1:9001","allow_private":true,"weight":1}], "load_balancing":{"algorithm":algorithm,"sticky":{"mode":sticky},"slow_start_seconds":0,"health_check":{"mode":"off"}} }]})).unwrap(); - let policy = crate::config::ValidationPolicy { - proxy_addr: "127.0.0.1:8080".parse().unwrap(), - admin_addr: "127.0.0.1:9090".parse().unwrap(), - tls_addr: None, - cert_dir: None, - data_dir: None, - }; - crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone() -} -fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc { - let cfg = serde_json::from_value(serde_json::json!({ - "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]}, - "vhosts":[{ - "id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]}, - "backends":[{ - "id":"only","url":"http://127.0.0.1:9000","allow_private":true, - "max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms - }], - "load_balancing":{"health_check":{"mode":"off"}} - }] - })) - .unwrap(); - let policy = crate::config::ValidationPolicy { - proxy_addr: "127.0.0.1:8080".parse().unwrap(), - admin_addr: "127.0.0.1:9090".parse().unwrap(), - tls_addr: None, - cert_dir: None, - data_dir: None, - }; - crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone() -} -fn select(pool: &Pool) -> Selection { - pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false) - .unwrap() -} -#[test] -fn per_backend_monitoring_can_be_disabled_or_overridden() { - let mut defaults: crate::vhost::PoolMember = serde_json::from_value(serde_json::json!({ - "id": "error_404", "url": "http://127.0.0.1:8282", "pool_eligible": false - })).unwrap(); - let mut settings = Settings::default(); - settings.passive_health.enabled = true; - assert!(defaults.health_check_enabled); - assert_eq!(settings.health_for(&defaults).mode, HealthMode::Tcp); - assert!(settings.passive_for(&defaults).enabled); - defaults.health_check_enabled = false; - assert_eq!(settings.health_for(&defaults).mode, HealthMode::Off); - assert!(!settings.passive_for(&defaults).enabled); - assert!(!defaults.pool_eligible); // routing intent is independent - defaults.health_check_enabled = true; - defaults.health_check = Some(HealthCheck { mode: HealthMode::Http, path: "/healthz".into(), ..Default::default() }); - let effective = settings.health_for(&defaults); - assert_eq!(effective.mode, HealthMode::Http); - assert_eq!(effective.path, "/healthz"); - assert_eq!(effective.host, settings.health_check.host); - let restored: crate::vhost::PoolMember = serde_json::from_str(&serde_json::to_string(&defaults).unwrap()).unwrap(); - assert_eq!(settings.health_for(&restored).path, "/healthz"); -} -#[test] -fn default_health_check_uses_tcp() { - assert_eq!(HealthMode::default(), HealthMode::Tcp); - assert_eq!(HealthCheck::default().mode, HealthMode::Tcp); -} -#[test] -fn legacy_settings_without_passive_health_default_to_off() { - let parsed: Settings = serde_json::from_value(serde_json::json!({ - "health_check": { "mode": "off" } - })) - .unwrap(); - assert!(!parsed.passive_health.enabled); - assert!(parsed.passive_health.count_connect_errors); - assert!(parsed.passive_health.count_timeouts); - assert!(parsed.passive_health.count_io_errors); - assert!(parsed.passive_health.count_http_5xx); - assert!(!parsed.passive_health.count_slow_responses); - assert_eq!(parsed.retry.max_retries, 1); - assert!(parsed.retry.retry_connect_errors); - assert!(parsed.retry.retry_timeouts); - assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]); - assert_eq!(parsed.keepalive.pool_recycle_requests, 25_000); - assert_eq!(parsed.keepalive.pool_recycle_seconds, 3_600); - assert_eq!(parsed.keepalive.http1_max_buffer_bytes, 131_072); + let policy = crate::config::ValidationPolicy { + proxy_addr: "127.0.0.1:8080".parse().unwrap(), + admin_addr: "127.0.0.1:9090".parse().unwrap(), + tls_addr: None, + cert_dir: None, + data_dir: None, + }; + crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone() + } + fn capacity_pool(limit: u32, queue_timeout_ms: u64) -> Arc { + let cfg = serde_json::from_value(serde_json::json!({ + "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]}, + "vhosts":[{ + "id":"limited","frontend":{"hosts":["limited.localhost"],"listener_ids":["http"]}, + "backends":[{ + "id":"only","url":"http://127.0.0.1:9000","allow_private":true, + "max_concurrent_requests":limit,"queue_timeout_ms":queue_timeout_ms + }], + "load_balancing":{"health_check":{"mode":"off"}} + }] + })) + .unwrap(); + let policy = crate::config::ValidationPolicy { + proxy_addr: "127.0.0.1:8080".parse().unwrap(), + admin_addr: "127.0.0.1:9090".parse().unwrap(), + tls_addr: None, + cert_dir: None, + data_dir: None, + }; + crate::config::compile(cfg, 0, &policy).unwrap().pools[0].clone() + } + fn select(pool: &Pool) -> Selection { + pool.select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), false) + .unwrap() + } + #[test] + fn per_backend_monitoring_can_be_disabled_or_overridden() { + let mut defaults: crate::vhost::PoolMember = serde_json::from_value(serde_json::json!({ + "id": "error_404", "url": "http://127.0.0.1:8282", "pool_eligible": false + })) + .unwrap(); + let mut settings = Settings::default(); + settings.passive_health.enabled = true; + assert!(defaults.health_check_enabled); + assert_eq!(settings.health_for(&defaults).mode, HealthMode::Tcp); + assert!(settings.passive_for(&defaults).enabled); + defaults.health_check_enabled = false; + assert_eq!(settings.health_for(&defaults).mode, HealthMode::Off); + assert!(!settings.passive_for(&defaults).enabled); + assert!(!defaults.pool_eligible); // routing intent is independent + defaults.health_check_enabled = true; + defaults.health_check = Some(HealthCheck { + mode: HealthMode::Http, + path: "/healthz".into(), + ..Default::default() + }); + let effective = settings.health_for(&defaults); + assert_eq!(effective.mode, HealthMode::Http); + assert_eq!(effective.path, "/healthz"); + assert_eq!(effective.host, settings.health_check.host); + let restored: crate::vhost::PoolMember = + serde_json::from_str(&serde_json::to_string(&defaults).unwrap()).unwrap(); + assert_eq!(settings.health_for(&restored).path, "/healthz"); + } + #[test] + fn default_health_check_uses_tcp() { + assert_eq!(HealthMode::default(), HealthMode::Tcp); + assert_eq!(HealthCheck::default().mode, HealthMode::Tcp); + } + #[test] + fn legacy_settings_without_passive_health_default_to_off() { + let parsed: Settings = serde_json::from_value(serde_json::json!({ + "health_check": { "mode": "off" } + })) + .unwrap(); + assert!(!parsed.passive_health.enabled); + assert!(parsed.passive_health.count_connect_errors); + assert!(parsed.passive_health.count_timeouts); + assert!(parsed.passive_health.count_io_errors); + assert!(parsed.passive_health.count_http_5xx); + assert!(!parsed.passive_health.count_slow_responses); + assert_eq!(parsed.retry.max_retries, 1); + assert!(parsed.retry.retry_connect_errors); + assert!(parsed.retry.retry_timeouts); + assert_eq!(parsed.retry.retry_statuses, vec![502, 503, 504]); + assert_eq!(parsed.keepalive.pool_recycle_requests, 25_000); + assert_eq!(parsed.keepalive.pool_recycle_seconds, 3_600); + assert_eq!(parsed.keepalive.http1_max_buffer_bytes, 131_072); + } + + #[test] + fn upstream_pool_recycle_limits_are_bounded_and_can_be_disabled() { + let mut settings = Settings::default(); + assert!(settings.validate().is_ok()); + + settings.keepalive.pool_recycle_requests = 0; + settings.keepalive.pool_recycle_seconds = 0; + assert!(settings.validate().is_ok()); + + settings.keepalive.pool_recycle_requests = 999; + assert!(settings.validate().is_err()); + settings.keepalive.pool_recycle_requests = 1_000; + settings.keepalive.pool_recycle_seconds = 59; + assert!(settings.validate().is_err()); + settings.keepalive.pool_recycle_seconds = 60; + settings.keepalive.http1_max_buffer_bytes = 8_191; + assert!(settings.validate().is_err()); + settings.keepalive.http1_max_buffer_bytes = 8_192; + assert!(settings.validate().is_ok()); + } + + #[test] + fn compatible_pool_rebuild_reuses_upstream_clients() { + let previous = pool(Algorithm::LeastConnections, StickyMode::Off); + let mut rebuilt = pool(Algorithm::LeastConnections, StickyMode::Off); + let rebuilt_mut = Arc::get_mut(&mut rebuilt).expect("fresh pool must be uniquely owned"); + + assert!(!Arc::ptr_eq( + &rebuilt_mut.members[0].backend, + &previous.members[0].backend + )); + rebuilt_mut.reuse_observations(&previous); + assert!(Arc::ptr_eq( + &rebuilt_mut.members[0].backend, + &previous.members[0].backend + )); + } + + #[test] + fn retry_policy_rejects_unbounded_or_non_5xx_configuration() { + let mut settings = Settings::default(); + settings.retry.max_retries = 4; + assert!(settings.validate().is_err()); + settings.retry.max_retries = 1; + settings.retry.retry_statuses = vec![429]; + assert!(settings.validate().is_err()); + } + + #[test] + fn traffic_rate_drops_to_zero_after_an_idle_sample_window() { + let state = BackendState::new(HealthMode::Off); + state + .bytes_out + .store(2048, std::sync::atomic::Ordering::Relaxed); + lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2); + state.sample_traffic(); + assert!(state.traffic_rate() > 0.0); + lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2); + state.sample_traffic(); + assert_eq!(state.traffic_rate(), 0.0); + } + #[test] + fn active_failures_are_independent_from_passive_user_traffic() { + let state = BackendState::new(HealthMode::Http); + let settings = settings(HealthMode::Http); + let passive = PassiveHealth::default(); + let fast = Duration::from_millis(5); + assert!(!state.healthy()); + state.observe_active(&settings, "ok", fast); + assert!(!state.healthy()); + state.observe_active(&settings, "ok", fast); + assert!(state.healthy()); + state.observe_active(&settings, "status", fast); + state.observe_passive(&settings, &passive, "ok", fast); + state.observe_active(&settings, "status", fast); + assert!(!state.healthy()); + state.observe_passive(&settings, &passive, "http_5xx", fast); + assert!(!state.healthy()); + state.observe_active(&settings, "ok", fast); + assert!(!state.healthy()); + state.observe_active(&settings, "ok", fast); + assert!(state.healthy()); + } + + #[test] + fn passive_health_is_opt_in_and_uses_separate_thresholds() { + let state = BackendState::new(HealthMode::Off); + let active = settings(HealthMode::Off); + let fast = Duration::from_millis(5); + for _ in 0..10 { + state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast); + } + assert!(state.healthy()); + assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10); + assert_eq!( + state.passive_health_failures_total.load(Ordering::Relaxed), + 0 + ); + + let passive = PassiveHealth { + enabled: true, + fall: Some(2), + rise: Some(2), + ..Default::default() + }; + state.observe_passive(&active, &passive, "http_5xx", fast); + assert!(state.healthy()); + state.observe_passive(&active, &passive, "http_5xx", fast); + assert!(!state.healthy()); + state.observe_passive(&active, &passive, "ok", fast); + assert!(!state.healthy()); + state.observe_passive(&active, &passive, "ok", fast); + assert!(state.healthy()); + } + #[test] + fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() { + let state = BackendState::new(HealthMode::Off); + let active = settings(HealthMode::Off); + let passive = PassiveHealth { + enabled: true, + count_http_5xx: false, + fall: Some(1), + ..Default::default() + }; + for _ in 0..100 { + state.observe_passive(&active, &passive, "http_5xx", Duration::from_millis(5)); + } + assert!(state.healthy()); + assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100); + assert_eq!( + state.passive_health_failures_total.load(Ordering::Relaxed), + 0 + ); + } + + #[test] + fn passive_connect_errors_use_the_explicit_fall_threshold() { + let state = BackendState::new(HealthMode::Off); + let active = settings(HealthMode::Off); + let passive = PassiveHealth { + enabled: true, + count_connect_errors: true, + fall: Some(3), + ..Default::default() + }; + for _ in 0..2 { + state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5)); + assert!(state.healthy()); + } + state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5)); + assert!(!state.healthy()); + assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3); + assert_eq!( + state.passive_health_failures_total.load(Ordering::Relaxed), + 3 + ); + } + + #[test] + fn active_health_can_stay_healthy_while_passive_health_is_off() { + let state = BackendState::new(HealthMode::Tcp); + let active = settings(HealthMode::Tcp); + let passive = PassiveHealth::default(); + let fast = Duration::from_millis(5); + state.observe_active(&active, "ok", fast); + state.observe_active(&active, "ok", fast); + assert!(state.healthy()); + for _ in 0..20 { + state.observe_passive(&active, &passive, "http_5xx", fast); + } + assert!(state.healthy()); + assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20); + assert_eq!( + state.passive_health_failures_total.load(Ordering::Relaxed), + 0 + ); + } + + #[test] + fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() { + let state = BackendState::new(HealthMode::Tcp); + let active = HealthCheck { + fall: 1, + rise: 1, + ..settings(HealthMode::Tcp) + }; + let passive = PassiveHealth { + enabled: true, + count_http_5xx: true, + fall: Some(1), + rise: Some(1), + ..Default::default() + }; + let fast = Duration::from_millis(5); + + state.observe_active(&active, "ok", fast); + state.observe_active(&active, "connect_error", fast); + { + let health = lock(&state.health); + assert_eq!(health.unhealthy_source, "active_tcp"); + } + state.observe_passive(&active, &passive, "http_5xx", fast); + state.observe_active(&active, "ok", fast); + { + let health = lock(&state.health); + assert!(!health.healthy); + assert_eq!(health.unhealthy_source, "passive_http_5xx"); + assert_eq!(health.unhealthy_reason, "http_5xx"); + } + state.observe_passive(&active, &passive, "ok", fast); + assert!(state.healthy()); + } + + #[test] + fn disabled_checks_never_quarantine_and_probes_are_single_flight() { + let off = BackendState::new(HealthMode::Off); + let settings = settings(HealthMode::Off); + for _ in 0..5 { + off.observe_passive( + &settings, + &PassiveHealth::default(), + "connect_error", + Duration::ZERO, + ); + } + assert!(off.healthy()); + assert!(!off.claim_probe(&settings)); + let on = BackendState::new(HealthMode::Http); + let settings = super::tests::settings(HealthMode::Http); + assert!(on.claim_probe(&settings)); + assert!(!on.claim_probe(&settings)); + on.observe_active(&settings, "ok", Duration::ZERO); + assert!(!on.claim_probe(&settings)); + } + #[tokio::test] + async fn weighted_round_robin_distributes_three_to_one_without_body_collection() { + let pool = pool(Algorithm::RoundRobin, StickyMode::Off); + let mut counts = [0, 0]; + for _ in 0..40 { + let selection = select(&pool); + counts[usize::from(selection.lease.member.definition.id == "b")] += 1; + } + assert_eq!(counts, [30, 10]); + assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0); + assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0); // Mere cache selection isn't origin traffic. + } + #[tokio::test] + async fn least_connections_counts_a_lease_until_its_last_owner_drops() { + let pool = pool(Algorithm::LeastConnections, StickyMode::Off); + let mut held = Vec::new(); + for _ in 0..3 { + let selection = select(&pool); + assert_eq!(selection.lease.member.definition.id, "a"); + held.push(selection.lease); + } + let other = select(&pool); + assert_eq!(other.lease.member.definition.id, "b"); + let clone = held[0].clone(); + held.clear(); + assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1); + drop(clone); + assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0); + } + #[tokio::test] + async fn failover_selection_never_reuses_an_attempted_backend() { + let pool = pool(Algorithm::LeastConnections, StickyMode::Off); + let first = select(&pool); + let first_id = first.lease.member.definition.id.clone(); + drop(first); + let retry = pool + .select_retry( + "127.0.0.1".parse().unwrap(), + &HeaderMap::new(), + false, + std::slice::from_ref(&first_id), + ) + .unwrap(); + assert_ne!(retry.lease.member.definition.id, first_id); + } + #[tokio::test] + async fn backend_concurrency_limit_queues_then_rejects_and_recovers() { + let pool = capacity_pool(1, 10); + let first = select(&pool); + let headers = HeaderMap::new(); + let saturated = pool + .select_wait("127.0.0.1".parse().unwrap(), &headers, false) + .await; + assert!(matches!(saturated, Err(SelectionFailure::Capacity))); + let view = pool.view(); + let member = &view["members"][0]; + assert_eq!(member["capacity_rejections_total"], 1); + assert_eq!(member["queue_timeouts_total"], 1); + assert_eq!(member["active"], 1); + drop(first); + assert!(pool + .select_wait("127.0.0.1".parse().unwrap(), &headers, false) + .await + .is_ok()); + } + + #[tokio::test] + async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() { + let pool = capacity_pool(1, 0); + let first = select(&pool); + let headers = HeaderMap::new(); + let saturated = pool + .select_wait("127.0.0.1".parse().unwrap(), &headers, false) + .await; + assert!(matches!(saturated, Err(SelectionFailure::Capacity))); + let view = pool.view(); + let member = &view["members"][0]; + assert_eq!(member["capacity_rejections_total"], 1); + assert_eq!(member["queue_timeouts_total"], 0); + drop(first); + } + + #[test] + fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() { + let pool = capacity_pool(0, 0); + let selection = select(&pool); + selection.lease.error_detail( + "dns", + "lookup backend.example failed", + Duration::from_millis(7), + ); + let view = pool.view(); + let member = &view["members"][0]; + assert_eq!(member["dns_errors_total"], 1); + assert_eq!(member["connect_errors_total"], 1); + assert_eq!(member["last_failure_class"], "dns"); + assert_eq!( + member["last_failure_detail"], + "lookup backend.example failed" + ); + assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0); + } + #[tokio::test] + async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() { + for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] { + let pool = pool(algorithm, StickyMode::Off); + lock(&pool.members[0].state.traffic).rate = 50_000_000.0; + lock(&pool.members[0].state.health).ewma_us = 1_000_000; + assert_eq!(select(&pool).lease.member.definition.id, "b"); + } + } + #[tokio::test] + async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() { + let pool = pool(Algorithm::IpHash, StickyMode::Off); + let original = select(&pool).lease.member.definition.id.clone(); + let mut headers = HeaderMap::new(); + headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap()); + for ip in ["127.0.0.1", "::ffff:127.0.0.1"] { + for _ in 0..8 { + assert_eq!( + pool.select(ip.parse().unwrap(), &headers, false) + .unwrap() + .lease + .member + .definition + .id, + original + ); + } + } + } + + #[tokio::test] + async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() { + let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); + let first = select(&pool); + let cookie = first.cookie.unwrap().to_str().unwrap().to_owned(); + assert!( + cookie.contains("HttpOnly") + && cookie.contains("SameSite=Lax") + && !cookie.contains("Secure") + ); + let cookie = cookie.split(';').next().unwrap(); + let mut headers = HeaderMap::new(); + headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap()); + let next = pool + .select("10.0.0.1".parse().unwrap(), &headers, false) + .unwrap(); + assert_eq!( + next.lease.member.definition.id, + first.lease.member.definition.id + ); + assert!(next.cookie.is_none()); + pool.strip_affinity(&mut headers); + assert_eq!(headers["cookie"], "app=ok"); + headers.append( + "set-cookie", + format!("{}=origin-spoof", pool.cookie_name) + .parse() + .unwrap(), + ); + headers.append("set-cookie", "app=ok".parse().unwrap()); + pool.strip_affinity_response(&mut headers); + assert_eq!(headers.get_all("set-cookie").iter().count(), 1); + assert_eq!(headers["set-cookie"], "app=ok"); + let secure = pool + .select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true) + .unwrap(); + assert!(secure + .cookie + .unwrap() + .to_str() + .unwrap() + .contains("; Secure")); + } + #[tokio::test] + async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() { + let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); + let first = select(&pool); + let cookie = first + .cookie + .unwrap() + .to_str() + .unwrap() + .split(';') + .next() + .unwrap() + .to_owned(); + let mut headers = HeaderMap::new(); + headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap()); + assert!(pool.affinity(&headers).is_none()); + headers.insert("cookie", cookie.parse().unwrap()); + lock(&first.lease.member.state.health).healthy = false; + let next = pool + .select("127.0.0.1".parse().unwrap(), &headers, false) + .unwrap(); + assert_ne!( + next.lease.member.definition.id, + first.lease.member.definition.id + ); + assert!(next.cookie.is_some()); + lock(&next.lease.member.state.health).healthy = false; + assert!(pool + .select("127.0.0.1".parse().unwrap(), &headers, false) + .is_none()); + } + #[tokio::test] + async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() { + let pool = pool(Algorithm::LeastConnections, StickyMode::Off); + let selected = select(&pool); + let state = &selected.lease.member.state; + assert!(state.weight(100, 100) < 100.0); + assert_eq!(state.weight(100, 0), 100.0); + selected.lease.started(); + selected.lease.bytes(50, true); + selected.lease.bytes(100, false); + selected.lease.error("timeout", Duration::from_secs(1)); + selected.lease.error("body", Duration::from_secs(2)); + selected.lease.retry_attempted(); + selected.lease.retry_succeeded(); + assert_eq!(state.requests.load(Ordering::Relaxed), 1); + assert_eq!(state.errors.load(Ordering::Relaxed), 1); + assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1); + assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1); + assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50); + assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100); + assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1); + assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1); + assert!(pool + .prometheus() + .contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1")); + assert!(pool + .prometheus() + .contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1")); + } + #[tokio::test] + async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() { + let pool = pool(Algorithm::LeastConnections, StickyMode::Off); + let selected = select(&pool); + selected.lease.response(500, Duration::from_secs(20), true); + let state = &selected.lease.member.state; + assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1); + assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1); + assert_eq!( + state.passive_health_failures_total.load(Ordering::Relaxed), + 0 + ); + assert!(state.healthy()); + } + + #[tokio::test] + async fn client_upload_duration_does_not_trigger_passive_slow_failure() { + let pool = pool(Algorithm::LeastConnections, StickyMode::Off); + let upload = select(&pool); + upload.lease.response(200, Duration::from_secs(20), false); + assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0); + let read = select(&pool); + read.lease.response(200, Duration::from_secs(20), true); + assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1); + } + #[tokio::test] + async fn backup_is_only_new_assignment_when_no_primary_is_eligible() { + let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off); + Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1]) + .unwrap() + .definition + .backup = true; + for _ in 0..8 { + assert_eq!(select(&pool).lease.member.definition.id, "a"); + } + lock(&pool.members[0].state.health).healthy = false; + assert_eq!(select(&pool).lease.member.definition.id, "b"); + lock(&pool.members[0].state.health).healthy = true; + assert_eq!(select(&pool).lease.member.definition.id, "a"); + } + #[tokio::test] + async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() { + let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); + Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0]) + .unwrap() + .definition + .drain = true; + assert_eq!(select(&pool).lease.member.definition.id, "b"); + let mut headers = HeaderMap::new(); + headers.insert( + "cookie", + format!("{}={}", pool.cookie_name, pool.members[0].token) + .parse() + .unwrap(), + ); + let existing = pool + .select("127.0.0.1".parse().unwrap(), &headers, false) + .unwrap(); + assert_eq!(existing.lease.member.definition.id, "a"); + lock(&pool.members[0].state.health).healthy = false; + assert_eq!( + pool.select("127.0.0.1".parse().unwrap(), &headers, false) + .unwrap() + .lease + .member + .definition + .id, + "b" + ); + } } #[test] -fn upstream_pool_recycle_limits_are_bounded_and_can_be_disabled() { - let mut settings = Settings::default(); - assert!(settings.validate().is_ok()); - - settings.keepalive.pool_recycle_requests = 0; - settings.keepalive.pool_recycle_seconds = 0; - assert!(settings.validate().is_ok()); - - settings.keepalive.pool_recycle_requests = 999; - assert!(settings.validate().is_err()); - settings.keepalive.pool_recycle_requests = 1_000; - settings.keepalive.pool_recycle_seconds = 59; - assert!(settings.validate().is_err()); - settings.keepalive.pool_recycle_seconds = 60; - settings.keepalive.http1_max_buffer_bytes = 8_191; - assert!(settings.validate().is_err()); - settings.keepalive.http1_max_buffer_bytes = 8_192; - assert!(settings.validate().is_ok()); -} - -#[test] -fn compatible_pool_rebuild_reuses_upstream_clients() { - let previous = pool(Algorithm::LeastConnections, StickyMode::Off); - let mut rebuilt = pool(Algorithm::LeastConnections, StickyMode::Off); - let rebuilt_mut = Arc::get_mut(&mut rebuilt).expect("fresh pool must be uniquely owned"); - - assert!(!Arc::ptr_eq( - &rebuilt_mut.members[0].backend, - &previous.members[0].backend - )); - rebuilt_mut.reuse_observations(&previous); - assert!(Arc::ptr_eq( - &rebuilt_mut.members[0].backend, - &previous.members[0].backend - )); -} - -#[test] -fn retry_policy_rejects_unbounded_or_non_5xx_configuration() { - let mut settings = Settings::default(); - settings.retry.max_retries = 4; - assert!(settings.validate().is_err()); - settings.retry.max_retries = 1; - settings.retry.retry_statuses = vec![429]; - assert!(settings.validate().is_err()); -} - -#[test] -fn traffic_rate_drops_to_zero_after_an_idle_sample_window() { - let state = BackendState::new(HealthMode::Off); - state - .bytes_out - .store(2048, std::sync::atomic::Ordering::Relaxed); - lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2); - state.sample_traffic(); - assert!(state.traffic_rate() > 0.0); - lock(&state.traffic).at = std::time::Instant::now() - std::time::Duration::from_secs(2); - state.sample_traffic(); - assert_eq!(state.traffic_rate(), 0.0); -} -#[test] -fn active_failures_are_independent_from_passive_user_traffic() { - let state = BackendState::new(HealthMode::Http); - let settings = settings(HealthMode::Http); - let passive = PassiveHealth::default(); - let fast = Duration::from_millis(5); - assert!(!state.healthy()); - state.observe_active(&settings, "ok", fast); - assert!(!state.healthy()); - state.observe_active(&settings, "ok", fast); - assert!(state.healthy()); - state.observe_active(&settings, "status", fast); - state.observe_passive(&settings, &passive, "ok", fast); - state.observe_active(&settings, "status", fast); - assert!(!state.healthy()); - state.observe_passive(&settings, &passive, "http_5xx", fast); - assert!(!state.healthy()); - state.observe_active(&settings, "ok", fast); - assert!(!state.healthy()); - state.observe_active(&settings, "ok", fast); - assert!(state.healthy()); -} - -#[test] -fn passive_health_is_opt_in_and_uses_separate_thresholds() { +fn passive_ejection_remembers_first_real_error_and_trigger() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); - let fast = Duration::from_millis(5); - for _ in 0..10 { - state.observe_passive(&active, &PassiveHealth::default(), "http_5xx", fast); - } - assert!(state.healthy()); - assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 10); - assert_eq!( - state.passive_health_failures_total.load(Ordering::Relaxed), - 0 - ); - let passive = PassiveHealth { enabled: true, fall: Some(2), - rise: Some(2), ..Default::default() }; - state.observe_passive(&active, &passive, "http_5xx", fast); - assert!(state.healthy()); - state.observe_passive(&active, &passive, "http_5xx", fast); + let elapsed = Duration::from_millis(3); + assert!(state + .observe_passive_with_detail( + &active, + &passive, + "connect_error", + "dns", + "first lookup failed", + elapsed + ) + .is_none()); + let (first, failures) = state + .observe_passive_with_detail( + &active, + &passive, + "connect_error", + "connect", + "second TCP failed", + elapsed, + ) + .expect("the threshold crossing must return the first failure"); + assert_eq!(failures, 2); + assert_eq!(first.class, "dns"); + assert_eq!(first.detail, "first lookup failed"); assert!(!state.healthy()); - state.observe_passive(&active, &passive, "ok", fast); - assert!(!state.healthy()); - state.observe_passive(&active, &passive, "ok", fast); - assert!(state.healthy()); -} -#[test] -fn passive_http_5xx_can_be_explicitly_ignored_when_passive_health_is_on() { - let state = BackendState::new(HealthMode::Off); - let active = settings(HealthMode::Off); - let passive = PassiveHealth { - enabled: true, - count_http_5xx: false, - fall: Some(1), - ..Default::default() - }; - for _ in 0..100 { - state.observe_passive(&active, &passive, "http_5xx", Duration::from_millis(5)); - } - assert!(state.healthy()); - assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 100); - assert_eq!( - state.passive_health_failures_total.load(Ordering::Relaxed), - 0 + assert!( + state + .observe_passive_with_detail( + &active, + &passive, + "connect_error", + "connect", + "third TCP failed", + elapsed + ) + .is_none(), + "the ejection must be logged only once" ); } #[test] -fn passive_connect_errors_use_the_explicit_fall_threshold() { +fn passive_io_burst_is_one_observation_not_many_failed_connections() { let state = BackendState::new(HealthMode::Off); let active = settings(HealthMode::Off); let passive = PassiveHealth { enabled: true, - count_connect_errors: true, fall: Some(3), ..Default::default() }; - for _ in 0..2 { - state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5)); - assert!(state.healthy()); - } - state.observe_passive(&active, &passive, "connect_error", Duration::from_millis(5)); - assert!(!state.healthy()); - assert_eq!(state.connect_errors_total.load(Ordering::Relaxed), 3); - assert_eq!( - state.passive_health_failures_total.load(Ordering::Relaxed), - 3 - ); -} - -#[test] -fn active_health_can_stay_healthy_while_passive_health_is_off() { - let state = BackendState::new(HealthMode::Tcp); - let active = settings(HealthMode::Tcp); - let passive = PassiveHealth::default(); - let fast = Duration::from_millis(5); - state.observe_active(&active, "ok", fast); - state.observe_active(&active, "ok", fast); - assert!(state.healthy()); for _ in 0..20 { - state.observe_passive(&active, &passive, "http_5xx", fast); + state.observe_passive_with_detail( + &active, + &passive, + "io_error", + "io_error", + "connection closed", + Duration::from_millis(1), + ); } assert!(state.healthy()); - assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 20); + assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 20); assert_eq!( state.passive_health_failures_total.load(Ordering::Relaxed), - 0 + 1 ); } #[test] -fn unhealthy_source_hands_off_when_the_original_health_domain_recovers() { - let state = BackendState::new(HealthMode::Tcp); +fn passive_only_quarantine_expires_without_vhost_reload() { + let state = BackendState::new(HealthMode::Off); + let active = settings(HealthMode::Off); + let passive = PassiveHealth { + enabled: true, + fall: Some(1), + ..Default::default() + }; + state.observe_passive(&active, &passive, "connect_error", Duration::ZERO); + assert!(!state.healthy()); + lock(&state.health).passive_ejected_at = Some(Instant::now() - Duration::from_secs(10)); + assert!( + state.healthy(), + "a half-open traffic attempt must become possible" + ); +} + +#[test] +fn active_probe_successes_recover_a_passively_ejected_backend() { + let state = BackendState::new(HealthMode::Http); let active = HealthCheck { fall: 1, rise: 1, - ..settings(HealthMode::Tcp) + ..settings(HealthMode::Http) }; let passive = PassiveHealth { enabled: true, - count_http_5xx: true, fall: Some(1), - rise: Some(1), ..Default::default() }; - let fast = Duration::from_millis(5); - - state.observe_active(&active, "ok", fast); - state.observe_active(&active, "connect_error", fast); - { - let health = lock(&state.health); - assert_eq!(health.unhealthy_source, "active_tcp"); - } - state.observe_passive(&active, &passive, "http_5xx", fast); - state.observe_active(&active, "ok", fast); - { - let health = lock(&state.health); - assert!(!health.healthy); - assert_eq!(health.unhealthy_source, "passive_http_5xx"); - assert_eq!(health.unhealthy_reason, "http_5xx"); - } - state.observe_passive(&active, &passive, "ok", fast); + state.observe_active(&active, "ok", Duration::ZERO); + assert!(state.healthy()); + state.observe_passive(&active, &passive, "connect_error", Duration::ZERO); + assert!(!state.healthy()); + state.observe_active(&active, "ok", Duration::ZERO); + assert!(!state.healthy()); + state.observe_active(&active, "ok", Duration::ZERO); assert!(state.healthy()); } - -#[test] -fn disabled_checks_never_quarantine_and_probes_are_single_flight() { - let off = BackendState::new(HealthMode::Off); - let settings = settings(HealthMode::Off); - for _ in 0..5 { - off.observe_passive( - &settings, - &PassiveHealth::default(), - "connect_error", - Duration::ZERO, - ); - } - assert!(off.healthy()); - assert!(!off.claim_probe(&settings)); - let on = BackendState::new(HealthMode::Http); - let settings = super::tests::settings(HealthMode::Http); - assert!(on.claim_probe(&settings)); - assert!(!on.claim_probe(&settings)); - on.observe_active(&settings, "ok", Duration::ZERO); - assert!(!on.claim_probe(&settings)); -} -#[tokio::test] -async fn weighted_round_robin_distributes_three_to_one_without_body_collection() { - let pool = pool(Algorithm::RoundRobin, StickyMode::Off); - let mut counts = [0, 0]; - for _ in 0..40 { - let selection = select(&pool); - counts[usize::from(selection.lease.member.definition.id == "b")] += 1; - } - assert_eq!(counts, [30, 10]); - assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0); - assert_eq!(pool.members[0].state.requests.load(Ordering::Relaxed), 0); // Mere cache selection isn't origin traffic. -} -#[tokio::test] -async fn least_connections_counts_a_lease_until_its_last_owner_drops() { - let pool = pool(Algorithm::LeastConnections, StickyMode::Off); - let mut held = Vec::new(); - for _ in 0..3 { - let selection = select(&pool); - assert_eq!(selection.lease.member.definition.id, "a"); - held.push(selection.lease); - } - let other = select(&pool); - assert_eq!(other.lease.member.definition.id, "b"); - let clone = held[0].clone(); - held.clear(); - assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 1); - drop(clone); - assert_eq!(pool.members[0].state.active.load(Ordering::Relaxed), 0); -} -#[tokio::test] -async fn failover_selection_never_reuses_an_attempted_backend() { - let pool = pool(Algorithm::LeastConnections, StickyMode::Off); - let first = select(&pool); - let first_id = first.lease.member.definition.id.clone(); - drop(first); - let retry = pool - .select_retry( - "127.0.0.1".parse().unwrap(), - &HeaderMap::new(), - false, - std::slice::from_ref(&first_id), - ) - .unwrap(); - assert_ne!(retry.lease.member.definition.id, first_id); -} -#[tokio::test] -async fn backend_concurrency_limit_queues_then_rejects_and_recovers() { - let pool = capacity_pool(1, 10); - let first = select(&pool); - let headers = HeaderMap::new(); - let saturated = pool - .select_wait("127.0.0.1".parse().unwrap(), &headers, false) - .await; - assert!(matches!(saturated, Err(SelectionFailure::Capacity))); - let view = pool.view(); - let member = &view["members"][0]; - assert_eq!(member["capacity_rejections_total"], 1); - assert_eq!(member["queue_timeouts_total"], 1); - assert_eq!(member["active"], 1); - drop(first); - assert!(pool - .select_wait("127.0.0.1".parse().unwrap(), &headers, false) - .await - .is_ok()); -} - -#[tokio::test] -async fn zero_capacity_queue_timeout_rejects_without_counting_a_queue_timeout() { - let pool = capacity_pool(1, 0); - let first = select(&pool); - let headers = HeaderMap::new(); - let saturated = pool - .select_wait("127.0.0.1".parse().unwrap(), &headers, false) - .await; - assert!(matches!(saturated, Err(SelectionFailure::Capacity))); - let view = pool.view(); - let member = &view["members"][0]; - assert_eq!(member["capacity_rejections_total"], 1); - assert_eq!(member["queue_timeouts_total"], 0); - drop(first); -} - -#[test] -fn detailed_backend_failures_are_exposed_without_losing_aggregate_counters() { - let pool = capacity_pool(0, 0); - let selection = select(&pool); - selection.lease.error_detail( - "dns", - "lookup backend.example failed", - Duration::from_millis(7), - ); - let view = pool.view(); - let member = &view["members"][0]; - assert_eq!(member["dns_errors_total"], 1); - assert_eq!(member["connect_errors_total"], 1); - assert_eq!(member["last_failure_class"], "dns"); - assert_eq!( - member["last_failure_detail"], - "lookup backend.example failed" - ); - assert!(member["last_failure_at_ms"].as_u64().unwrap_or_default() > 0); -} -#[tokio::test] -async fn traffic_and_adaptive_selection_use_observed_cost_and_weight() { - for algorithm in [Algorithm::LeastTraffic, Algorithm::Adaptive] { - let pool = pool(algorithm, StickyMode::Off); - lock(&pool.members[0].state.traffic).rate = 50_000_000.0; - lock(&pool.members[0].state.health).ewma_us = 1_000_000; - assert_eq!(select(&pool).lease.member.definition.id, "b"); - } -} -#[tokio::test] -async fn ip_hash_is_stable_and_cannot_be_spoofed_with_forwarded_headers() { - let pool = pool(Algorithm::IpHash, StickyMode::Off); - let original = select(&pool).lease.member.definition.id.clone(); - let mut headers = HeaderMap::new(); - headers.insert("x-forwarded-for", "10.0.0.99".parse().unwrap()); - for ip in ["127.0.0.1", "::ffff:127.0.0.1"] { - for _ in 0..8 { - assert_eq!( - pool.select(ip.parse().unwrap(), &headers, false) - .unwrap() - .lease - .member - .definition - .id, - original - ); - } - } -} - -#[tokio::test] -async fn opaque_cookie_is_removed_from_origin_requests_and_protected_on_responses() { - let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); - let first = select(&pool); - let cookie = first.cookie.unwrap().to_str().unwrap().to_owned(); - assert!( - cookie.contains("HttpOnly") - && cookie.contains("SameSite=Lax") - && !cookie.contains("Secure") - ); - let cookie = cookie.split(';').next().unwrap(); - let mut headers = HeaderMap::new(); - headers.insert("cookie", format!("{cookie}; app=ok").parse().unwrap()); - let next = pool - .select("10.0.0.1".parse().unwrap(), &headers, false) - .unwrap(); - assert_eq!( - next.lease.member.definition.id, - first.lease.member.definition.id - ); - assert!(next.cookie.is_none()); - pool.strip_affinity(&mut headers); - assert_eq!(headers["cookie"], "app=ok"); - headers.append( - "set-cookie", - format!("{}=origin-spoof", pool.cookie_name) - .parse() - .unwrap(), - ); - headers.append("set-cookie", "app=ok".parse().unwrap()); - pool.strip_affinity_response(&mut headers); - assert_eq!(headers.get_all("set-cookie").iter().count(), 1); - assert_eq!(headers["set-cookie"], "app=ok"); - let secure = pool - .select("127.0.0.1".parse().unwrap(), &HeaderMap::new(), true) - .unwrap(); - assert!(secure - .cookie - .unwrap() - .to_str() - .unwrap() - .contains("; Secure")); -} -#[tokio::test] -async fn failed_sticky_member_is_replaced_and_duplicate_cookies_are_ignored() { - let pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); - let first = select(&pool); - let cookie = first - .cookie - .unwrap() - .to_str() - .unwrap() - .split(';') - .next() - .unwrap() - .to_owned(); - let mut headers = HeaderMap::new(); - headers.insert("cookie", format!("{cookie}; {cookie}").parse().unwrap()); - assert!(pool.affinity(&headers).is_none()); - headers.insert("cookie", cookie.parse().unwrap()); - lock(&first.lease.member.state.health).healthy = false; - let next = pool - .select("127.0.0.1".parse().unwrap(), &headers, false) - .unwrap(); - assert_ne!( - next.lease.member.definition.id, - first.lease.member.definition.id - ); - assert!(next.cookie.is_some()); - lock(&next.lease.member.state.health).healthy = false; - assert!(pool - .select("127.0.0.1".parse().unwrap(), &headers, false) - .is_none()); -} -#[tokio::test] -async fn slow_start_ramps_weight_and_error_is_counted_once_per_exchange() { - let pool = pool(Algorithm::LeastConnections, StickyMode::Off); - let selected = select(&pool); - let state = &selected.lease.member.state; - assert!(state.weight(100, 100) < 100.0); - assert_eq!(state.weight(100, 0), 100.0); - selected.lease.started(); - selected.lease.bytes(50, true); - selected.lease.bytes(100, false); - selected.lease.error("timeout", Duration::from_secs(1)); - selected.lease.error("body", Duration::from_secs(2)); - selected.lease.retry_attempted(); - selected.lease.retry_succeeded(); - assert_eq!(state.requests.load(Ordering::Relaxed), 1); - assert_eq!(state.errors.load(Ordering::Relaxed), 1); - assert_eq!(state.timeouts_total.load(Ordering::Relaxed), 1); - assert_eq!(state.io_errors_total.load(Ordering::Relaxed), 1); - assert_eq!(state.bytes_in.load(Ordering::Relaxed), 50); - assert_eq!(state.bytes_out.load(Ordering::Relaxed), 100); - assert_eq!(state.retry_attempts_total.load(Ordering::Relaxed), 1); - assert_eq!(state.retry_successes_total.load(Ordering::Relaxed), 1); - assert!(pool - .prometheus() - .contains("rust_proxy_backend_requests_total{vhost=\"app\",backend=\"a\"} 1")); - assert!(pool - .prometheus() - .contains("rust_proxy_backend_retry_attempts_total{vhost=\"app\",backend=\"a\"} 1")); -} -#[tokio::test] -async fn slow_5xx_counts_both_observability_classes_when_passive_is_off() { - let pool = pool(Algorithm::LeastConnections, StickyMode::Off); - let selected = select(&pool); - selected.lease.response(500, Duration::from_secs(20), true); - let state = &selected.lease.member.state; - assert_eq!(state.http_5xx_total.load(Ordering::Relaxed), 1); - assert_eq!(state.slow_responses_total.load(Ordering::Relaxed), 1); - assert_eq!( - state.passive_health_failures_total.load(Ordering::Relaxed), - 0 - ); - assert!(state.healthy()); -} - -#[tokio::test] -async fn client_upload_duration_does_not_trigger_passive_slow_failure() { - let pool = pool(Algorithm::LeastConnections, StickyMode::Off); - let upload = select(&pool); - upload.lease.response(200, Duration::from_secs(20), false); - assert_eq!(upload.lease.member.state.errors.load(Ordering::Relaxed), 0); - let read = select(&pool); - read.lease.response(200, Duration::from_secs(20), true); - assert_eq!(read.lease.member.state.errors.load(Ordering::Relaxed), 1); -} -#[tokio::test] -async fn backup_is_only_new_assignment_when_no_primary_is_eligible() { - let mut pool = pool(Algorithm::RoundRobin, StickyMode::Off); - Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[1]) - .unwrap() - .definition - .backup = true; - for _ in 0..8 { - assert_eq!(select(&pool).lease.member.definition.id, "a"); - } - lock(&pool.members[0].state.health).healthy = false; - assert_eq!(select(&pool).lease.member.definition.id, "b"); - lock(&pool.members[0].state.health).healthy = true; - assert_eq!(select(&pool).lease.member.definition.id, "a"); -} -#[tokio::test] -async fn drain_rejects_new_sessions_but_preserves_valid_existing_affinity() { - let mut pool = pool(Algorithm::RoundRobin, StickyMode::Cookie); - Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().members[0]) - .unwrap() - .definition - .drain = true; - assert_eq!(select(&pool).lease.member.definition.id, "b"); - let mut headers = HeaderMap::new(); - headers.insert( - "cookie", - format!("{}={}", pool.cookie_name, pool.members[0].token) - .parse() - .unwrap(), - ); - let existing = pool - .select("127.0.0.1".parse().unwrap(), &headers, false) - .unwrap(); - assert_eq!(existing.lease.member.definition.id, "a"); - lock(&pool.members[0].state.health).healthy = false; - assert_eq!( - pool.select("127.0.0.1".parse().unwrap(), &headers, false) - .unwrap() - .lease - .member - .definition - .id, - "b" - ); -} -} diff --git a/src/proxy/forward/response.rs b/src/proxy/forward/response.rs index fe6e12b..4e48dba 100644 --- a/src/proxy/forward/response.rs +++ b/src/proxy/forward/response.rs @@ -28,7 +28,7 @@ } if let Some(lease) = &active_lease { lease.error_detail( - upstream_error_class(&cause, connect_failure), + upstream_health_error_class(&cause, connect_failure, &active_backend), &detail, final_attempt_elapsed, ); diff --git a/src/proxy/forward/upstream.rs b/src/proxy/forward/upstream.rs index 445cf18..9ea8b31 100644 --- a/src/proxy/forward/upstream.rs +++ b/src/proxy/forward/upstream.rs @@ -141,6 +141,7 @@ let mut active_uri = upstream_target; let mut request_attempt = Some(request); let mut attempted_backends = Vec::::new(); let mut retries = 0u8; +let mut graceful_retry_done = false; let upstream_result = loop { let client = if handshake.is_some() && !websocket_upstream_h2 { &active_backend.websocket_client @@ -153,6 +154,36 @@ let upstream_result = loop { ) .await; let attempt_elapsed = attempt_started.elapsed(); + if let Ok(Err(cause)) = &result { + if !graceful_retry_done + && !admin_control_plane + && handshake.is_none() + && request_body_empty + && retry_safe_method(&retry_method) + && (is_graceful_h2_goaway(cause) + || is_related_h2_close(cause, &active_backend)) + { + if let Some(next_request) = retry_request( + &retry_method, + upstream_request_version(&active_backend, false), + &retry_headers, + &active_backend, + &original_host, + &origin_suffix, + retry_host_follows_backend, + ) { + graceful_retry_done = true; + if let Some(lease) = &active_lease { + lease.retry_attempted(); + } + tracing::debug!(backend = %active_backend.id, + "retrying safe empty request after graceful upstream H2 shutdown"); + request_attempt = Some(next_request); + attempt_started = Instant::now(); + continue; + } + } + } let Some(policy) = retry_policy.as_ref() else { break result; }; @@ -220,7 +251,7 @@ let upstream_result = loop { connect_failure, ); lease.error_detail( - upstream_error_class(cause, connect_failure), + upstream_health_error_class(cause, connect_failure, &active_backend), &detail, attempt_elapsed, ); diff --git a/src/proxy/h2_graceful.rs b/src/proxy/h2_graceful.rs new file mode 100644 index 0000000..368c229 --- /dev/null +++ b/src/proxy/h2_graceful.rs @@ -0,0 +1,27 @@ +fn is_graceful_h2_goaway(error: &(dyn std::error::Error + 'static)) -> bool { + crate::upstream::is_graceful_h2_goaway(error) +} + +fn is_related_h2_close( + error: &(dyn std::error::Error + 'static), + backend: &crate::config::CompiledBackend, +) -> bool { + let chain = format!("{error:?}"); + backend.client.recently_saw_graceful_goaway() + && chain.contains("Canceled") + && chain.contains("connection closed") +} + +fn upstream_health_error_class( + error: &(dyn std::error::Error + 'static), + connect_failure: bool, + backend: &crate::config::CompiledBackend, +) -> &'static str { + if is_graceful_h2_goaway(error) { + "h2_graceful_goaway" + } else if is_related_h2_close(error, backend) { + "h2_graceful_close" + } else { + upstream_error_class(error, connect_failure) + } +} diff --git a/src/proxy/tests/authority_tests.rs b/src/proxy/tests/authority_tests.rs index bd32a33..8cb4e98 100644 --- a/src/proxy/tests/authority_tests.rs +++ b/src/proxy/tests/authority_tests.rs @@ -571,3 +571,28 @@ mod authority_tests { } } } + +#[cfg(test)] +mod graceful_h2_tests { + use super::*; + use std::fmt; + + #[derive(Debug)] + struct H2DebugError(&'static str); + impl fmt::Display for H2DebugError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "client error (SendRequest): {}", self.0) + } + } + impl std::error::Error for H2DebugError {} + + #[test] + fn only_graceful_remote_h2_goaway_is_neutral() { + let valid = H2DebugError("GoAway(b\"\", NO_ERROR, Remote)"); + let protocol = H2DebugError("GoAway(b\"\", PROTOCOL_ERROR, Remote)"); + let local = H2DebugError("GoAway(b\"\", NO_ERROR, Local)"); + assert!(is_graceful_h2_goaway(&valid)); + assert!(!is_graceful_h2_goaway(&protocol)); + assert!(!is_graceful_h2_goaway(&local)); + } +} diff --git a/src/proxy/types.rs b/src/proxy/types.rs index 005a9f8..5b1b06a 100644 --- a/src/proxy/types.rs +++ b/src/proxy/types.rs @@ -138,6 +138,8 @@ fn upstream_error_class( } } +include!("h2_graceful.rs"); + fn retry_safe_method(method: &Method) -> bool { method == Method::GET || method == Method::HEAD || method == Method::OPTIONS } @@ -171,11 +173,12 @@ fn upstream_request_target( websocket_h1: bool, websocket_h2: bool, ) -> Option { - let h2_capable = !websocket_h1 && match backend.protocol { - crate::config::UpstreamProtocol::Http1 => false, - crate::config::UpstreamProtocol::Auto => backend.origin.starts_with("https://"), - crate::config::UpstreamProtocol::Http2 | crate::config::UpstreamProtocol::H2 => true, - }; + let h2_capable = !websocket_h1 + && match backend.protocol { + crate::config::UpstreamProtocol::Http1 => false, + crate::config::UpstreamProtocol::Auto => backend.origin.starts_with("https://"), + crate::config::UpstreamProtocol::Http2 | crate::config::UpstreamProtocol::H2 => true, + }; if !h2_capable { return Some(uri.clone()); } diff --git a/src/upstream.rs b/src/upstream.rs index b813676..d5eec66 100644 --- a/src/upstream.rs +++ b/src/upstream.rs @@ -32,7 +32,8 @@ pub type BoxError = Box; mod client; pub use client::{ - client, client_with_keepalive, client_with_timeout, ClientRotationStats, ProxyClient, + client, client_with_keepalive, client_with_timeout, is_graceful_h2_goaway, ClientRotationStats, + ProxyClient, }; #[derive(Clone, Debug, Default, PartialEq, Eq)] diff --git a/src/upstream/client.rs b/src/upstream/client.rs index 1ed8dfa..822d2aa 100644 --- a/src/upstream/client.rs +++ b/src/upstream/client.rs @@ -22,6 +22,13 @@ use std::{ time::{Duration, Instant}, }; +/// Hyper's public error chain omits the GOAWAY reason, but Debug retains +/// the peer-initiated H2 `NO_ERROR` variant. Restrict this to graceful closure. +pub fn is_graceful_h2_goaway(error: &(dyn std::error::Error + 'static)) -> bool { + let chain = format!("{error:?}"); + chain.contains("GoAway(") && chain.contains("NO_ERROR") && chain.contains("Remote") +} + static MONOTONIC_START: OnceLock = OnceLock::new(); type RawProxyClient = Client; @@ -54,6 +61,7 @@ struct RotatingClient { observed_at_ms: AtomicU64, observed_h1: AtomicU64, observed_h2: AtomicU64, + graceful_goaway_at_ms: AtomicU64, } /// Backend HTTP client with bounded generation lifetime. @@ -107,19 +115,43 @@ impl ProxyClient { observed_at_ms: AtomicU64::new(0), observed_h1: AtomicU64::new(0), observed_h2: AtomicU64::new(0), + graceful_goaway_at_ms: AtomicU64::new(0), }), } } pub async fn request(&self, request: Request) -> Result, ClientError> { let client = self.inner.client_for_request(); + let generation = self.inner.generation.load(Ordering::Acquire); let response = client.request(request).await; - if let Ok(value) = &response { - self.inner.observe_protocol(value.version()); + match &response { + Ok(value) => self.inner.observe_protocol(value.version()), + Err(error) if is_graceful_h2_goaway(error) => { + // Also applies to cache refresh and other backend HTTP callers, + // not only the regular forward loop. + self.note_graceful_goaway(generation); + } + Err(_) => {} } response } + /// A peer may gracefully close a reused H2 connection. Discard only the + /// affected client generation; concurrent requests must not rotate again. + pub fn note_graceful_goaway(&self, generation: u64) { + self.inner + .graceful_goaway_at_ms + .store(monotonic_millis().saturating_add(1), Ordering::Release); + self.inner.rotate(generation, "h2_goaway_no_error", false); + } + + /// A cancellation immediately following GOAWAY may be another stream on + /// the closing H2 connection. Keep this correlation window short. + pub fn recently_saw_graceful_goaway(&self) -> bool { + let at = self.inner.graceful_goaway_at_ms.load(Ordering::Acquire); + at != 0 && monotonic_millis().saturating_add(1).saturating_sub(at) <= 1_000 + } + pub fn protocol_stats(&self) -> UpstreamProtocolStats { self.inner.protocol_stats() } @@ -425,14 +457,18 @@ fn build_raw_client(factory: &ClientFactory) -> Result { url::Host::Ipv6(ip) => format!("[{ip}]"), host => host.to_string(), }; - let port = origin.port_or_known_default().context("backend port missing")?; - let tls_name = backend.tls_server_name + let port = origin + .port_or_known_default() + .context("backend port missing")?; + let tls_name = backend + .tls_server_name .as_deref() .map(crate::config::normalize_host) .transpose()? .unwrap_or(network_host); let destination: http::Uri = format!("{}://{tls_name}:{port}", origin.scheme()) - .parse().context("invalid configured backend transport authority")?; + .parse() + .context("invalid configured backend transport authority")?; Ok(builder.build(TimedConnector { inner, policy: policy.clone(), diff --git a/tools/checks/certificate_api_0319.py b/tools/checks/certificate_api_0319.py index 3533dd8..cecd096 100755 --- a/tools/checks/certificate_api_0319.py +++ b/tools/checks/certificate_api_0319.py @@ -7,7 +7,7 @@ import tomllib root = Path(__file__).resolve().parents[2] read = lambda path: (root / path).read_text(encoding='utf-8') version = tomllib.loads(read('Cargo.toml'))['package']['version'] -assert version == '0.32.6', version +assert version == '0.32.7', version spec = json.loads(read('src/admin/openapi.json')) assert spec['info']['version'] == version paths = spec['paths'] @@ -64,5 +64,5 @@ for language in ('en_US', 'pl_pl'): for key in ('certificateApiView', 'certificateApiEdit', 'certificateApiSave', 'certificateApiRegenerate', 'certificateApiRegenerateConfirm', 'certificateApiTokenHidden', 'certificateApiCreatedAt'): assert strings.get(key), (language, key) -assert '0.32.6' in read('docs/TEST_REPORT.md') +assert '0.32.7' in read('docs/TEST_REPORT.md') print('PASS: 0.31.9 per-credential API routes, separated metadata editing/token regeneration, OpenAPI security, UI and locales (source-only).') diff --git a/tools/checks/check_source.py b/tools/checks/check_source.py index 96a4c81..d42e304 100755 --- a/tools/checks/check_source.py +++ b/tools/checks/check_source.py @@ -24,7 +24,7 @@ def read_docker_agent_source() -> str: preflight(root) manifest = tomllib.loads((root / "Cargo.toml").read_text()) -assert manifest["package"]["version"] == "0.32.6" +assert manifest["package"]["version"] == "0.32.7" assert manifest["dependencies"]["quinn"]["version"] == "0.11.12" h3dep = manifest["dependencies"]["h3"] assert h3dep["package"] == "hpx-h3" and h3dep["version"] == "=2.5.20" and "quinn" in h3dep["features"] @@ -32,14 +32,14 @@ assert "h3-quinn" not in manifest["dependencies"] lock_path = root / "Cargo.lock" if lock_path.exists(): lock = lock_path.read_text() - assert 'name = "rust-proxy"\nversion = "0.32.6"' in lock + assert 'name = "rust-proxy"\nversion = "0.32.7"' in lock assert 'name = "systemstat"' in lock and 'name = "metrics-process"' in lock and 'name = "sysinfo"' in lock, "Cargo.lock must be resolver-generated after adding cross-platform statistics libraries" for release_doc, marker in { - "README.md": "# RustProxy 0.32.6", - "CHANGELOG.md": "## 0.32.6", - "ROADMAP.md": "## 0.32.6", - "AGENTS.md": "Current source version: 0.32.6.", - "docs/TEST_REPORT.md": "# Verification report - RustProxy 0.32.6", + "README.md": "# RustProxy 0.32.7", + "CHANGELOG.md": "## 0.32.7", + "ROADMAP.md": "## 0.32.7", + "AGENTS.md": "Current source version: 0.32.7.", + "docs/TEST_REPORT.md": "# Verification report - RustProxy 0.32.7", }.items(): assert marker in (root / release_doc).read_text(), f"stale release identity in {release_doc}" assert tomllib.loads((root / "rust-toolchain.toml").read_text())["toolchain"]["channel"] == "stable" @@ -285,7 +285,7 @@ assert 'Opt-in health decisions based on ordinary user traffic. Defaults to disa assert '"force_ssl"' in openapi and '"forward_client_ip"' in openapi assert '"backend_header"' in openapi and '"backend_timing_header"' in openapi assert '"StreamMetrics"' in openapi and '"streams"' in openapi -for marker in ['"version": "0.32.6"', '"/api/traffic"', '"capture_user_agent"', '"capture_country"', '"hot_auto_trim_enabled"', '"hot_auto_trim_after_seconds"', '"hot_auto_trim_target_percent"', '"/api/config/{section}"', '"ConfigSectionUpdate"', '"cold_on_idle"', '"cold_on_age"', '"cold_max_hot_seconds"', '"cold_on_lru"', '"cold_on_shutdown"', '"cold_gc_interval_seconds"', '"cold_gc_max_delete"', '"collapse_enabled"', '"collapse_wait_timeout_ms"', '"collapse_timeout_fallback"', '"stale_if_error_timeout"', '"stale_if_error_connection"', '"stale_if_error_dns"', '"stale_if_error_tls"', '"stale_if_error_statuses"', '"proactive_refresh_enabled"', '"proactive_refresh_before_expiry_seconds"', '"proactive_refresh_percent"', '"proactive_refresh_max_inflight"', '"TransformSettings"', '"TransformOverrides"', '"/api/cache/cold/cleanup"', '"/api/cache/invalidate"', '"/api/cache/bans"', '"/api/cache/bans/{id}"', '"CacheInvalidationRequest"', '"CacheInvalidationResponse"', '"ColdCleanupJob"', '"match_type"', '"regex_insensitive"', '"rewrite"', '"response"', '"covered_hosts"', '"default": true']: +for marker in ['"version": "0.32.7"', '"/api/traffic"', '"capture_user_agent"', '"capture_country"', '"hot_auto_trim_enabled"', '"hot_auto_trim_after_seconds"', '"hot_auto_trim_target_percent"', '"/api/config/{section}"', '"ConfigSectionUpdate"', '"cold_on_idle"', '"cold_on_age"', '"cold_max_hot_seconds"', '"cold_on_lru"', '"cold_on_shutdown"', '"cold_gc_interval_seconds"', '"cold_gc_max_delete"', '"collapse_enabled"', '"collapse_wait_timeout_ms"', '"collapse_timeout_fallback"', '"stale_if_error_timeout"', '"stale_if_error_connection"', '"stale_if_error_dns"', '"stale_if_error_tls"', '"stale_if_error_statuses"', '"proactive_refresh_enabled"', '"proactive_refresh_before_expiry_seconds"', '"proactive_refresh_percent"', '"proactive_refresh_max_inflight"', '"TransformSettings"', '"TransformOverrides"', '"/api/cache/cold/cleanup"', '"/api/cache/invalidate"', '"/api/cache/bans"', '"/api/cache/bans/{id}"', '"CacheInvalidationRequest"', '"CacheInvalidationResponse"', '"ColdCleanupJob"', '"match_type"', '"regex_insensitive"', '"rewrite"', '"response"', '"covered_hosts"', '"default": true']: assert marker in openapi, marker for marker in ['"NotFoundHost"', '"not_found_hosts"', '"PrometheusSettings"', '"allow_trusted_network_without_token"', '"include_docker_vhosts"', '"include_streams"', '"include_logging"']: assert marker in openapi, marker @@ -1396,7 +1396,7 @@ assert '"error_log" => {' not in admin_0241 and 'request("/api/config/error_log" assert '"error_log"' in openapi and 'logging, error_log, forwarding' not in openapi assert 'logging_section_persists_error_log_settings_atomically' in admin_0241 and 'error_log_is_not_a_standalone_configuration_section' in admin_0241 assert 'location.pathname === "/logging"' in settings_js_0241 -assert '"RuntimeInformation"' in openapi and '"version": "0.32.6"' in openapi +assert '"RuntimeInformation"' in openapi and '"version": "0.32.7"' in openapi # 0.24.1 compiler and hostname-cardinality regressions. tls_0241 = rust_source(root, "src/tls.rs") diff --git a/tools/checks/dependency_upgrade_0312.py b/tools/checks/dependency_upgrade_0312.py index 3f6d7ea..e8754b8 100755 --- a/tools/checks/dependency_upgrade_0312.py +++ b/tools/checks/dependency_upgrade_0312.py @@ -14,7 +14,7 @@ parser = argparse.ArgumentParser() parser.add_argument('--require-lock', action='store_true', help='Require the post-upgrade Cargo.lock produced by cargo update') args = parser.parse_args() manifest = tomllib.loads((ROOT / 'Cargo.toml').read_text()) -assert manifest['package']['version'] == '0.32.6' +assert manifest['package']['version'] == '0.32.7' assert manifest['package']['rust-version'] == '1.99' deps = manifest['dependencies'] expected_requirements = { diff --git a/tools/checks/docs_check.py b/tools/checks/docs_check.py index e1c7b43..acc78d2 100755 --- a/tools/checks/docs_check.py +++ b/tools/checks/docs_check.py @@ -51,7 +51,7 @@ assert '"/docs/content.json"' in admin readme = (ROOT / "README.md").read_text(encoding="utf-8") roadmap = (ROOT / "ROADMAP.md").read_text(encoding="utf-8") changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") -assert readme.startswith("# RustProxy 0.32.6") +assert readme.startswith("# RustProxy 0.32.7") assert "docs/WEB_SERVER.md" in readme and "docs/README.md" in readme assert readme.count("## What changed in ") == 1, "README duplicates historical release notes; use CHANGELOG/archive" assert "## Release history" in readme and "docs/ROADMAP_HISTORY.md" in readme diff --git a/tools/checks/dynamic_acl_0325.py b/tools/checks/dynamic_acl_0325.py index 22c654c..40500d7 100755 --- a/tools/checks/dynamic_acl_0325.py +++ b/tools/checks/dynamic_acl_0325.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Static source contract for DDNS ACL 0.32.6; NOT a Rust test.""" +"""Static source contract for DDNS ACL 0.32.7; NOT a Rust test.""" from pathlib import Path import json import tomllib @@ -25,8 +25,8 @@ checks = { 'visual selector and disabled DNS deny': '.acl-profile-rule-type' in js and 'aclDnsAllowOnly' in js, 'old network field persisted': 'network: normalizeNetwork(' in js, 'PL/EN translations': all(k in json.loads(read(f'langs/{loc}.json'))['translations'] for loc in ('en_US','pl_pl') for k in ('aclRuleType','aclRuleDns','aclDnsHelp','aclDnsAllowOnly')), - 'package version': tomllib.loads(read('Cargo.toml'))['package']['version'] == '0.32.6', + 'package version': tomllib.loads(read('Cargo.toml'))['package']['version'] == '0.32.7', } for name, ok in checks.items(): if not ok: raise AssertionError(name) -print(f'PASS: {len(checks)} DNS ACL 0.32.6 static source checks (not compiled or network-tested)') +print(f'PASS: {len(checks)} DNS ACL 0.32.7 static source checks (not compiled or network-tested)') diff --git a/tools/checks/h3_navigation_0316.py b/tools/checks/h3_navigation_0316.py index 6859574..b14b51f 100755 --- a/tools/checks/h3_navigation_0316.py +++ b/tools/checks/h3_navigation_0316.py @@ -4,7 +4,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] html = (ROOT / 'src/admin/web/index.html').read_text() -assert '0.32.6' in (ROOT / "Cargo.toml").read_text() +assert '0.32.7' in (ROOT / "Cargo.toml").read_text() assert '',1)[0] body = (ROOT / 'src/proxy/cache_helpers.rs').read_text() diff --git a/tools/checks/passive_h2_0327.py b/tools/checks/passive_h2_0327.py new file mode 100755 index 0000000..72b7051 --- /dev/null +++ b/tools/checks/passive_h2_0327.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Source-only regression assertions; NOT a compiled Rust or live H2 test.""" +from pathlib import Path +import tomllib + +root = Path(__file__).resolve().parents[2] +read = lambda name: (root / name).read_text(encoding='utf-8') +assert tomllib.loads(read('Cargo.toml'))['package']['version'] == '0.32.7' +client = read('src/upstream/client.rs') +forward = read('src/proxy/forward/upstream.rs') +helper = read('src/proxy/h2_graceful.rs') +lease = read('src/balance/lease_runtime.rs') +health = read('src/balance/health.rs') +tests = read('src/balance/tests.rs') +for marker in ('GoAway(', 'NO_ERROR', 'Remote', 'note_graceful_goaway(generation)', 'is_graceful_h2_goaway(error)'): + assert marker in client, marker +for marker in ('graceful_retry_done', 'retry_safe_method(&retry_method)', 'request_body_empty', 'handshake.is_none()', 'retry_request(', 'graceful_retry_done = true', 'continue;'): + assert marker in forward, marker +for marker in ('is_related_h2_close', 'recently_saw_graceful_goaway()', 'h2_graceful_goaway', 'h2_graceful_close'): + assert marker in helper, marker +for marker in ('first_error_detail', 'first_error_at_ms', 'trigger_error_detail', 'backend excluded from traffic', 'tracing::warn!(', 'h2_graceful_goaway'): + assert marker in lease, marker +for marker in ('passive_first_failure', 'active_first_failure', 'passive_probe_successes', 'passive_ejected_at', 'Duration::from_millis(250)', 'fn clear_passive_ejection', 'pub fn healthy'): + assert marker in health, marker +for marker in ('passive_ejection_remembers_first_real_error_and_trigger', 'passive_io_burst_is_one_observation', 'passive_only_quarantine_expires', 'active_probe_successes_recover'): + assert marker in tests, marker +print('PASS: version, H2 graceful recycle/retry, passive health, first-error WARN source contracts (NOT compiled)') diff --git a/tools/checks/preserve_host_0324.py b/tools/checks/preserve_host_0324.py index 7093fd1..cf1430d 100755 --- a/tools/checks/preserve_host_0324.py +++ b/tools/checks/preserve_host_0324.py @@ -51,6 +51,6 @@ for marker in ['margin: 0;', '--overview-gap: max(3px, calc(var(--ui-gap) - 1px) retry = re.search(r'fn retry_request\([\s\S]*?\n}\n', proxy) assert retry and '.uri(uri.clone())' in retry[0] assert 'upstream_request_target(&uri, request.headers_mut()' in retry[0] -assert json.loads(read('src/admin/openapi.json'))['info']['version']=='0.32.6' -assert 'version = "0.32.6"' in read('Cargo.toml') -print('PASS: 0.32.6 origin pinning, authority, retry, CSS gutter source contracts (not Rust runtime)') +assert json.loads(read('src/admin/openapi.json'))['info']['version']=='0.32.7' +assert 'version = "0.32.7"' in read('Cargo.toml') +print('PASS: 0.32.7 origin pinning, authority, retry, CSS gutter source contracts (not Rust runtime)') diff --git a/tools/checks/release_0318.py b/tools/checks/release_0318.py index d3ef79a..d4ebf1f 100755 --- a/tools/checks/release_0318.py +++ b/tools/checks/release_0318.py @@ -8,9 +8,9 @@ from html.parser import HTMLParser ROOT = Path(__file__).resolve().parents[2] def read(rel): return (ROOT / rel).read_text(encoding='utf-8') -assert tomllib.loads(read('Cargo.toml'))['package']['version'] == '0.32.6' -assert 'name = "rust-proxy"\nversion = "0.32.6"' in read('Cargo.lock') -assert json.loads(read('src/admin/openapi.json'))['info']['version'] == '0.32.6' +assert tomllib.loads(read('Cargo.toml'))['package']['version'] == '0.32.7' +assert 'name = "rust-proxy"\nversion = "0.32.7"' in read('Cargo.lock') +assert json.loads(read('src/admin/openapi.json'))['info']['version'] == '0.32.7' pages = read('src/pages.rs') for code in (400, 401, 403, 404, 408, 410, 413, 429, 500, 502, 503, 504): diff --git a/tools/checks/release_smoke.py b/tools/checks/release_smoke.py index 1d2006b..99f5654 100755 --- a/tools/checks/release_smoke.py +++ b/tools/checks/release_smoke.py @@ -135,7 +135,7 @@ def upgrade_smoke(tmp: Path) -> None: assert backup.is_dir() assert (backup / "src/obsolete.rs").is_file() assert not (target / "src/obsolete.rs").exists() - assert 'version = "0.32.6"' in (target / "Cargo.toml").read_text(encoding="utf-8") + assert 'version = "0.32.7"' in (target / "Cargo.toml").read_text(encoding="utf-8") assert (target / "data/keep.txt").read_text(encoding="utf-8") == "preserve\n" assert (target / ".env").read_text(encoding="utf-8") == "RUST_PROXY_DEMO=0\n" assert (target / "langs/site.json").is_file() @@ -146,7 +146,7 @@ def upgrade_smoke(tmp: Path) -> None: assert (target / "docker-agent/certificate.go").is_file() assert (target / "docker-agent/server.go").is_file() manifest = json.loads((backup / "upgrade.json").read_text(encoding="utf-8")) - assert manifest["to_version"] == "0.32.6" + assert manifest["to_version"] == "0.32.7" assert manifest["removed_obsolete_runtime_state"] is True assert not legacy.exists() assert not (backup / "legacy-data").exists() @@ -617,7 +617,7 @@ def source_contract_smoke() -> None: assert marker in admin, marker anonymous_block = admin.split("fn anonymous_allowed", 1)[1].split("const CERTIFICATE_API_KEY_HEADER", 1)[0] assert '"/api/status"' not in anonymous_block - assert '"RuntimeInformation"' in openapi_0241 and '"version": "0.32.6"' in openapi_0241 + assert '"RuntimeInformation"' in openapi_0241 and '"version": "0.32.7"' in openapi_0241 assert 'id="panel-logging"' not in html for marker in ['id="settings-tab-logging"', 'id="settings-page-logging"', 'id="logging-form"', 'id="error-log-form"']: assert marker in html, marker @@ -1102,7 +1102,7 @@ def source_contract_smoke() -> None: assert '.managed-update' in main and "flush_hot_to_cold_for_update" in main openapi = (ROOT / "src/admin/openapi.json").read_text(encoding="utf-8") assert '"/api/config/history/restore"' in openapi - for marker in ['"version": "0.32.6"', '"memory_preset"', '"cold_preset"', '"memory_budget_bytes"', '"max_inflight"', '"preset"', '"session_days"', '"session_lifetime_seconds"', '"accept_local_listener_risk"', '"backend_header"', '"backend_timing_header"', '"cold_on_age"', '"match_type"', '"regex_insensitive"', '"NotFoundHost"', '"not_found_hosts"', '"PrometheusSettings"', '"allow_trusted_network_without_token"', '"include_docker_vhosts"', '"include_streams"', '"include_logging"']: + for marker in ['"version": "0.32.7"', '"memory_preset"', '"cold_preset"', '"memory_budget_bytes"', '"max_inflight"', '"preset"', '"session_days"', '"session_lifetime_seconds"', '"accept_local_listener_risk"', '"backend_header"', '"backend_timing_header"', '"cold_on_age"', '"match_type"', '"regex_insensitive"', '"NotFoundHost"', '"not_found_hosts"', '"PrometheusSettings"', '"allow_trusted_network_without_token"', '"include_docker_vhosts"', '"include_streams"', '"include_logging"']: assert marker in openapi, marker assert '"software_update"' in openapi and '"update_available"' in openapi for marker in ['"/api/certificates/deploy"', '"CertificateDeploy"', '"CertificateDeployResult"', '"/api/certificates/reload"']: diff --git a/tools/checks/security_logs_0315_ui_test.mjs b/tools/checks/security_logs_0315_ui_test.mjs index 1f729e5..8d6216d 100644 --- a/tools/checks/security_logs_0315_ui_test.mjs +++ b/tools/checks/security_logs_0315_ui_test.mjs @@ -52,7 +52,7 @@ assert.match(fileLogs, /localStorage\.setItem\(CONSOLIDATION_STORAGE_KEY/); assert.match(fileLogs, /aggregate_min_count/); assert.match(fileLogs, /file-log-aggregate-copy/); -assert.equal(openapi.info.version, '0.32.6'); +assert.equal(openapi.info.version, '0.32.7'); assert.ok(openapi.components.schemas.Config.properties.user_agent_blocking); assert.ok(openapi.components.schemas.Vhost.properties.user_agent_blocking); assert.ok(openapi.components.schemas.FileLogSource.properties.group_id); diff --git a/tools/checks/upstream_protocols_0323.py b/tools/checks/upstream_protocols_0323.py index 6e24872..e3023ba 100755 --- a/tools/checks/upstream_protocols_0323.py +++ b/tools/checks/upstream_protocols_0323.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Offline source contracts for RustProxy 0.32.6 upstream protocols and card registry. +"""Offline source contracts for RustProxy 0.32.7 upstream protocols and card registry. These assertions supplement, rather than replace, Rust ALPN, H2C and websocket runtime tests. No code execution against a RustProxy server is implied. @@ -141,7 +141,7 @@ for field in ('dashboard_widgets', 'dashboard_hidden_widgets'): -assert '"version": "0.32.6"' in read('src/admin/openapi.json') -assert 'version = "0.32.6"' in read('Cargo.toml') -assert 'version = "0.32.6"' in read('Cargo.lock') -print(f'PASS: 0.32.6 upstream H1/Auto/compatible-HTTP2/strict-H2 source contracts and {len(ids)} registered Overview Cards') +assert '"version": "0.32.7"' in read('src/admin/openapi.json') +assert 'version = "0.32.7"' in read('Cargo.toml') +assert 'version = "0.32.7"' in read('Cargo.lock') +print(f'PASS: 0.32.7 upstream H1/Auto/compatible-HTTP2/strict-H2 source contracts and {len(ids)} registered Overview Cards')