v0.35.2

src/balance/health.rs

@@ -450,6 +450,7 @@ pub struct Member {
     pub backend: Arc<CompiledBackend>,
     pub definition: PoolMember,
     pub state: Arc<BackendState>,
+    health_probe: Option<Arc<CompiledBackend>>,
     health_settings: Arc<HealthCheck>,
     passive_settings: Arc<PassiveHealth>,
     token: String,

src/balance/lease_runtime.rs

@@ -291,12 +291,36 @@ impl Pool {
         text
     }
 }
+const PROBE_BODY_LIMIT: usize = 65_536;
+
+async fn probe_body_contains(mut body: hyper::body::Incoming, expected: &[u8]) -> bool {
+    use http_body_util::BodyExt;
+    let mut seen = Vec::new();
+    while seen.len() < PROBE_BODY_LIMIT {
+        match body.frame().await {
+            Some(Ok(frame)) => {
+                if let Ok(data) = frame.into_data() {
+                    let room = PROBE_BODY_LIMIT - seen.len();
+                    seen.extend_from_slice(&data[..data.len().min(room)]);
+                    if seen.windows(expected.len()).any(|window| window == expected) {
+                        return true;
+                    }
+                }
+            }
+            Some(Err(_)) => return false,
+            None => break,
+        }
+    }
+    seen.windows(expected.len()).any(|window| window == expected)
+}
+
 async fn probe(member: Arc<Member>, settings: Arc<HealthCheck>) {
     let start = Instant::now();
+    let target = member.health_probe.as_deref().unwrap_or(&member.backend);
     let result = tokio::time::timeout(Duration::from_millis(settings.timeout_ms), async {
         match settings.mode {
             HealthMode::Tcp => {
-                crate::upstream::tcp_probe(&member.backend.origin, &member.backend.address_policy)
+                crate::upstream::tcp_probe(&target.origin, &target.address_policy)
                     .await
                     .map(|_| "ok")
                     .unwrap_or("connect")
@@ -305,14 +329,14 @@ async fn probe(member: Arc<Member>, settings: Arc<HealthCheck>) {
                 let request = Request::builder()
                     .uri(format!(
                         "{}{}",
-                        member.backend.request_origin, settings.path
+                        target.request_origin, settings.path
                     ))
                     .header(
                         "host",
                         settings
                             .host
                             .as_deref()
-                            .unwrap_or_else(|| member.backend.host_header.to_str().unwrap_or("")),
+                            .unwrap_or_else(|| target.host_header.to_str().unwrap_or("")),
                     )
                     .header("connection", "close")
                     .header(
@@ -321,14 +345,20 @@ async fn probe(member: Arc<Member>, settings: Arc<HealthCheck>) {
                     )
                     .body(Body::empty());
                 match request {
-                    Ok(request) => match member.backend.websocket_client.request(request).await {
+                    Ok(request) => match target.websocket_client.request(request).await {
                         Ok(response) => {
-                            if (settings.expected_status_min..=settings.expected_status_max)
+                            if !(settings.expected_status_min..=settings.expected_status_max)
                                 .contains(&response.status().as_u16())
                             {
-                                "ok"
-                            } else {
                                 "http_status"
+                            } else if let Some(expected) = settings.expect_body.as_deref() {
+                                if probe_body_contains(response.into_body(), expected.as_bytes()).await {
+                                    "ok"
+                                } else {
+                                    "http_body"
+                                }
+                            } else {
+                                "ok"
                             }
                         }
                         Err(_) => "connect",

src/balance/model.rs

@@ -99,6 +99,8 @@ pub struct HealthCheck {
     pub mode: HealthMode,
     pub path: String,
     pub host: Option<String>,
+    pub port: Option<u16>,
+    pub expect_body: Option<String>,
     pub interval_ms: u64,
     pub timeout_ms: u64,
     pub slow_response_ms: u64,
@@ -113,6 +115,8 @@ impl Default for HealthCheck {
             mode: HealthMode::Tcp,
             path: "/".into(),
             host: None,
+            port: None,
+            expect_body: None,
             interval_ms: 5000,
             timeout_ms: 2000,
             slow_response_ms: 1500,
@@ -261,6 +265,13 @@ impl Settings {
                 && !h.path.bytes().any(|b| b <= 32 || b == 127),
             "invalid health HTTP path"
         );
+        ensure!(h.port != Some(0), "health check port must be 1..65535");
+        if let Some(expected) = &h.expect_body {
+            ensure!(
+                !expected.is_empty() && expected.len() <= 256 && !expected.chars().any(char::is_control),
+                "health check expected body text must be 1..256 printable characters"
+            );
+        }
         if let Some(host) = &h.host {
             crate::config::request_host(host)?;
             let _: HeaderValue = host.parse()?;

src/balance/pool_core.rs

@@ -24,6 +24,7 @@ impl Pool {
                 let passive_settings = Arc::new(settings.passive_for(&definition));
                 Arc::new(Member {
                     state: Arc::new(BackendState::new(health_settings.mode)),
+                    health_probe: backend.health_probe.clone(),
                     definition,
                     backend,
                     health_settings,

src/balance/tests.rs

@@ -832,4 +832,133 @@ mod tests {
         assert_eq!(state.last_status.load(Ordering::Relaxed), 503);
         assert_eq!(lock(&state.last_failure).detail, "status=503");
     }
+
+    fn health_port_config(backend_port: u16, health: serde_json::Value) -> crate::config::Config {
+        serde_json::from_value(serde_json::json!({
+            "listeners":{"endpoints":[{"id":"http","binding":{"address":"127.0.0.1:8080"}}]},
+            "vhosts":[{
+            "id":"app","frontend":{"hosts":["app.localhost"],"listener_ids":["http"]},
+            "backends":[{"id":"a","url":format!("http://127.0.0.1:{backend_port}"),"allow_private":true}],
+            "load_balancing":{"health_check":health}
+        }]})).unwrap()
+    }
+    fn health_port_policy() -> crate::config::ValidationPolicy {
+        crate::config::ValidationPolicy {
+            proxy_addr: "127.0.0.1:8080".parse().unwrap(),
+            admin_addr: "127.0.0.1:9090".parse().unwrap(),
+            tls_addr: None,
+            cert_dir: None,
+            data_dir: None,
+        }
+    }
+    #[test]
+    fn health_check_port_compiles_separate_probe_target() {
+        let policy = health_port_policy();
+        let snapshot = crate::config::compile(
+            health_port_config(9000, serde_json::json!({"mode":"tcp","port":8079})), 0, &policy).unwrap();
+        let backend = &snapshot.pools[0].members[0].backend;
+        assert_eq!(backend.origin, "http://127.0.0.1:9000");
+        assert_eq!(backend.health_probe.as_ref().unwrap().origin, "http://127.0.0.1:8079");
+        for health in [
+            serde_json::json!({"mode":"tcp","port":9000}),
+            serde_json::json!({"mode":"tcp"}),
+            serde_json::json!({"mode":"off","port":8079}),
+        ] {
+            let snapshot = crate::config::compile(health_port_config(9000, health), 0, &policy).unwrap();
+            assert!(snapshot.pools[0].members[0].backend.health_probe.is_none());
+        }
+        assert!(crate::config::compile(
+            health_port_config(9000, serde_json::json!({"mode":"tcp","port":0})), 0, &policy).is_err());
+        assert!(crate::config::compile(
+            health_port_config(9000, serde_json::json!({"mode":"http","port":9090})), 0, &policy).is_err());
+    }
+    #[tokio::test]
+    async fn health_check_probes_configured_port_instead_of_traffic_port() {
+        let traffic = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let traffic_port = traffic.local_addr().unwrap().port();
+        let closed_port = {
+            let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+            listener.local_addr().unwrap().port()
+        };
+        let policy = health_port_policy();
+        let config = health_port_config(traffic_port, serde_json::json!({
+            "mode":"tcp","port":closed_port,"fall":1,"rise":1,"interval_ms":250,"timeout_ms":200,"slow_response_ms":0}));
+        let snapshot = crate::config::compile(config, 0, &policy).unwrap();
+        let member = snapshot.pools[0].members[0].clone();
+        let settings = member.health_settings.clone();
+        probe(member.clone(), settings.clone()).await;
+        assert!(!member.state.healthy(), "probe must use the closed health port");
+
+        let health = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let health_port = health.local_addr().unwrap().port();
+        drop(traffic);
+        let config = health_port_config(traffic_port, serde_json::json!({
+            "mode":"tcp","port":health_port,"fall":1,"rise":1,"interval_ms":250,"timeout_ms":200,"slow_response_ms":0}));
+        let snapshot = crate::config::compile(config, 0, &policy).unwrap();
+        let member = snapshot.pools[0].members[0].clone();
+        let settings = member.health_settings.clone();
+        probe(member.clone(), settings).await;
+        assert!(member.state.healthy(), "probe must succeed on the open health port while traffic port is closed");
+        tokio::time::timeout(Duration::from_secs(1), health.accept())
+            .await
+            .expect("health port must receive the probe connection")
+            .unwrap();
+    }
+
+    async fn one_shot_http(body: &'static str) -> (u16, tokio::task::JoinHandle<()>) {
+        use tokio::io::{AsyncReadExt, AsyncWriteExt};
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let port = listener.local_addr().unwrap().port();
+        let task = tokio::spawn(async move {
+            while let Ok((mut socket, _)) = listener.accept().await {
+                let mut buffer = [0u8; 2048];
+                let _ = socket.read(&mut buffer).await;
+                let response = format!(
+                    "HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+                    body.len(), body
+                );
+                let _ = socket.write_all(response.as_bytes()).await;
+            }
+        });
+        (port, task)
+    }
+    #[tokio::test]
+    async fn http_health_check_requires_expected_body_text() {
+        let policy = health_port_policy();
+        for (body, healthy) in [("{\"status\":\"ok\"}", true), ("{\"status\":\"degraded\"}", false)] {
+            let (port, server) = one_shot_http(body).await;
+            let config = health_port_config(port, serde_json::json!({
+                "mode":"http","path":"/health","expect_body":"\"status\":\"ok\"",
+                "fall":1,"rise":1,"interval_ms":1000,"timeout_ms":800,"slow_response_ms":0}));
+            let snapshot = crate::config::compile(config, 0, &policy).unwrap();
+            let member = snapshot.pools[0].members[0].clone();
+            probe(member.clone(), member.health_settings.clone()).await;
+            assert_eq!(member.state.healthy(), healthy, "body {body}");
+            if !healthy {
+                assert_eq!(lock(&member.state.health).result, "http_body");
+            }
+            server.abort();
+        }
+        for invalid in ["", "line\nbreak"] {
+            assert!(crate::config::compile(
+                health_port_config(9000, serde_json::json!({"mode":"http","expect_body":invalid})), 0, &policy).is_err());
+        }
+    }
+
+    #[tokio::test]
+    async fn health_check_port_survives_live_reload_with_reused_clients() {
+        let policy = health_port_policy();
+        let defaults = (policy.proxy_addr, policy.tls_addr);
+        let first = crate::config::compile_reusing(
+            health_port_config(9000, serde_json::json!({"mode":"tcp"})), 0, &policy, defaults, None).unwrap();
+        assert!(first.pools[0].members[0].health_probe.is_none());
+        let second = crate::config::compile_reusing(
+            health_port_config(9000, serde_json::json!({"mode":"tcp","port":8079})), 1, &policy, defaults, Some(&first)).unwrap();
+        let member = &second.pools[0].members[0];
+        assert!(Arc::ptr_eq(&member.backend, &first.pools[0].members[0].backend), "traffic clients are reused");
+        assert_eq!(member.health_probe.as_ref().unwrap().origin, "http://127.0.0.1:8079");
+        let third = crate::config::compile_reusing(
+            health_port_config(9000, serde_json::json!({"mode":"tcp"})), 2, &policy, defaults, Some(&second)).unwrap();
+        assert!(third.pools[0].members[0].health_probe.is_none());
+    }
 }